Edwin Eames
1592e6d685
- Add newenglandoil/ package as the primary scraper (replaces fuel_scraper) - Add cheapestoil/ package as a secondary market price scraper - Add app.py entry point for direct execution - Update run.py: new scrape_cheapest(), migrate command, --state filter, --refresh-metadata flag for overwriting existing phone/URL data - Update models.py with latest schema fields - Update requirements.txt dependencies - Update Dockerfile and docker-compose.yml for new structure - Remove deprecated fuel_scraper module, test.py, and log file Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
51 lines
1.5 KiB
Python
51 lines
1.5 KiB
Python
"""
|
|
Configuration for the CheapestOil scraper.
|
|
"""
|
|
|
|
API_URL = "https://www.cheapestoil.com/heating-oil-prices/api"
|
|
|
|
# Seconds between requests to be polite
|
|
SCRAPE_DELAY = 2
|
|
|
|
# State abbreviation -> list of county names on cheapestoil.com
|
|
# None means state-level only (no county filter)
|
|
STATE_COUNTIES = {
|
|
"MA": [
|
|
"Barnstable", "Berkshire", "Bristol", "Essex", "Franklin",
|
|
"Hampden", "Hampshire", "Middlesex", "Norfolk", "Plymouth",
|
|
"Suffolk", "Worcester",
|
|
],
|
|
"CT": [
|
|
"Fairfield", "Hartford", "Litchfield", "Middlesex",
|
|
"New Haven", "New London", "Tolland", "Windham",
|
|
],
|
|
"ME": [
|
|
"Cumberland", "York", "Penobscot", "Kennebec", "Androscoggin",
|
|
"Aroostook", "Oxford", "Hancock", "Somerset", "Knox",
|
|
"Waldo", "Sagadahoc", "Lincoln", "Washington", "Franklin",
|
|
"Piscataquis",
|
|
],
|
|
"NH": [
|
|
"Belknap", "Carroll", "Cheshire", "Coos", "Grafton",
|
|
"Hillsborough", "Merrimack", "Rockingham", "Strafford", "Sullivan",
|
|
],
|
|
"RI": [
|
|
"Bristol", "Kent", "Newport", "Providence", "Washington",
|
|
],
|
|
"VT": [
|
|
"Addison", "Bennington", "Caledonia", "Chittenden", "Essex",
|
|
"Franklin", "Grand Isle", "Lamoille", "Orange", "Orleans",
|
|
"Rutland", "Washington", "Windham", "Windsor",
|
|
],
|
|
}
|
|
|
|
# State abbreviation -> API state name (as used in cheapestoil.com params)
|
|
STATE_API_NAMES = {
|
|
"MA": "Massachusetts",
|
|
"CT": "Connecticut",
|
|
"ME": "Maine",
|
|
"NH": "NewHampshire",
|
|
"RI": "RhodeIsland",
|
|
"VT": "Vermont",
|
|
}
|