Edwin Eames
af9c2f99e7
FastAPI-based scraper for commodity ticker prices (HO, CL, RB futures) and competitor oil pricing from NewEnglandOil. Includes cron-driven scraping, PostgreSQL storage, and REST endpoints for price retrieval. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
27 lines
768 B
Python
27 lines
768 B
Python
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
url = "https://www.newenglandoil.com/massachusetts/zone10.asp?x=0"
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
|
}
|
|
|
|
try:
|
|
response = requests.get(url, headers=headers, timeout=15)
|
|
print(f"Status Code: {response.status_code}")
|
|
|
|
with open("debug_page.html", "w", encoding="utf-8") as f:
|
|
f.write(response.text)
|
|
|
|
soup = BeautifulSoup(response.content, 'lxml')
|
|
tables = soup.find_all('table')
|
|
print(f"Tables found: {len(tables)}")
|
|
|
|
if not tables:
|
|
print("Preview of content:")
|
|
print(response.text[:500])
|
|
|
|
except Exception as e:
|
|
print(f"Error: {e}")
|