Edwin Eames
af9c2f99e7
FastAPI-based scraper for commodity ticker prices (HO, CL, RB futures) and competitor oil pricing from NewEnglandOil. Includes cron-driven scraping, PostgreSQL storage, and REST endpoints for price retrieval. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
26 lines
707 B
Python
26 lines
707 B
Python
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
url = "https://www.newenglandoil.com/massachusetts/zone10.asp?x=0"
|
|
# Use the UA from config.py
|
|
headers = {
|
|
"User-Agent": "Unraid-EamcoScraper/1.0 (eeames214@gmail.com)"
|
|
}
|
|
|
|
try:
|
|
print(f"Testing with UA: {headers['User-Agent']}")
|
|
response = requests.get(url, headers=headers, timeout=15)
|
|
print(f"Status Code: {response.status_code}")
|
|
|
|
soup = BeautifulSoup(response.content, 'lxml')
|
|
tables = soup.find_all('table')
|
|
print(f"Tables found: {len(tables)}")
|
|
|
|
if not tables:
|
|
print("No tables found. Dumping start of response:")
|
|
print(response.text[:500])
|
|
|
|
except Exception as e:
|
|
print(f"Error: {e}")
|