feat: initial commit for oil price scraper service
FastAPI-based scraper for commodity ticker prices (HO, CL, RB futures) and competitor oil pricing from NewEnglandOil. Includes cron-driven scraping, PostgreSQL storage, and REST endpoints for price retrieval. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
25
debug_scraper_ua.py
Normal file
25
debug_scraper_ua.py
Normal file
@@ -0,0 +1,25 @@
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
url = "https://www.newenglandoil.com/massachusetts/zone10.asp?x=0"
|
||||
# Use the UA from config.py
|
||||
headers = {
|
||||
"User-Agent": "Unraid-EamcoScraper/1.0 (eeames214@gmail.com)"
|
||||
}
|
||||
|
||||
try:
|
||||
print(f"Testing with UA: {headers['User-Agent']}")
|
||||
response = requests.get(url, headers=headers, timeout=15)
|
||||
print(f"Status Code: {response.status_code}")
|
||||
|
||||
soup = BeautifulSoup(response.content, 'lxml')
|
||||
tables = soup.find_all('table')
|
||||
print(f"Tables found: {len(tables)}")
|
||||
|
||||
if not tables:
|
||||
print("No tables found. Dumping start of response:")
|
||||
print(response.text[:500])
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
Reference in New Issue
Block a user