refactor: replace fuel_scraper with newenglandoil + cheapestoil scrapers
- Add newenglandoil/ package as the primary scraper (replaces fuel_scraper) - Add cheapestoil/ package as a secondary market price scraper - Add app.py entry point for direct execution - Update run.py: new scrape_cheapest(), migrate command, --state filter, --refresh-metadata flag for overwriting existing phone/URL data - Update models.py with latest schema fields - Update requirements.txt dependencies - Update Dockerfile and docker-compose.yml for new structure - Remove deprecated fuel_scraper module, test.py, and log file Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
50
cheapestoil/config.py
Normal file
50
cheapestoil/config.py
Normal file
@@ -0,0 +1,50 @@
|
||||
"""
|
||||
Configuration for the CheapestOil scraper.
|
||||
"""
|
||||
|
||||
API_URL = "https://www.cheapestoil.com/heating-oil-prices/api"
|
||||
|
||||
# Seconds between requests to be polite
|
||||
SCRAPE_DELAY = 2
|
||||
|
||||
# State abbreviation -> list of county names on cheapestoil.com
|
||||
# None means state-level only (no county filter)
|
||||
STATE_COUNTIES = {
|
||||
"MA": [
|
||||
"Barnstable", "Berkshire", "Bristol", "Essex", "Franklin",
|
||||
"Hampden", "Hampshire", "Middlesex", "Norfolk", "Plymouth",
|
||||
"Suffolk", "Worcester",
|
||||
],
|
||||
"CT": [
|
||||
"Fairfield", "Hartford", "Litchfield", "Middlesex",
|
||||
"New Haven", "New London", "Tolland", "Windham",
|
||||
],
|
||||
"ME": [
|
||||
"Cumberland", "York", "Penobscot", "Kennebec", "Androscoggin",
|
||||
"Aroostook", "Oxford", "Hancock", "Somerset", "Knox",
|
||||
"Waldo", "Sagadahoc", "Lincoln", "Washington", "Franklin",
|
||||
"Piscataquis",
|
||||
],
|
||||
"NH": [
|
||||
"Belknap", "Carroll", "Cheshire", "Coos", "Grafton",
|
||||
"Hillsborough", "Merrimack", "Rockingham", "Strafford", "Sullivan",
|
||||
],
|
||||
"RI": [
|
||||
"Bristol", "Kent", "Newport", "Providence", "Washington",
|
||||
],
|
||||
"VT": [
|
||||
"Addison", "Bennington", "Caledonia", "Chittenden", "Essex",
|
||||
"Franklin", "Grand Isle", "Lamoille", "Orange", "Orleans",
|
||||
"Rutland", "Washington", "Windham", "Windsor",
|
||||
],
|
||||
}
|
||||
|
||||
# State abbreviation -> API state name (as used in cheapestoil.com params)
|
||||
STATE_API_NAMES = {
|
||||
"MA": "Massachusetts",
|
||||
"CT": "Connecticut",
|
||||
"ME": "Maine",
|
||||
"NH": "NewHampshire",
|
||||
"RI": "RhodeIsland",
|
||||
"VT": "Vermont",
|
||||
}
|
||||
Reference in New Issue
Block a user