feat: initial commit for oil price scraper service
FastAPI-based scraper for commodity ticker prices (HO, CL, RB futures) and competitor oil pricing from NewEnglandOil. Includes cron-driven scraping, PostgreSQL storage, and REST endpoints for price retrieval. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
26
debug_scraper.py
Normal file
26
debug_scraper.py
Normal file
@@ -0,0 +1,26 @@
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
url = "https://www.newenglandoil.com/massachusetts/zone10.asp?x=0"
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=headers, timeout=15)
|
||||
print(f"Status Code: {response.status_code}")
|
||||
|
||||
with open("debug_page.html", "w", encoding="utf-8") as f:
|
||||
f.write(response.text)
|
||||
|
||||
soup = BeautifulSoup(response.content, 'lxml')
|
||||
tables = soup.find_all('table')
|
||||
print(f"Tables found: {len(tables)}")
|
||||
|
||||
if not tables:
|
||||
print("Preview of content:")
|
||||
print(response.text[:500])
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
Reference in New Issue
Block a user