""" Configuration module for the fuel scraper. Contains site definitions, zone-to-county mapping, and logging setup. """ import logging # --- SITES CONFIGURATION --- SITES_CONFIG = [ { "site_name": "NewEnglandOil", "base_url": "https://www.newenglandoil.com", "url_template": "{base_url}/{state_slug}/{zone_slug}.asp?type={oil_type}", "oil_type": 0, "locations": { "connecticut": [ "zone1", "zone2", "zone3", "zone4", "zone5", "zone6", "zone7", "zone8", "zone9", "zone10" ], "massachusetts": [ "zone1", "zone2", "zone3", "zone4", "zone5", "zone6", "zone7", "zone8", "zone9", "zone10", "zone11", "zone12", "zone13", "zone14", "zone15" ], "newhampshire": [ "zone1", "zone2", "zone3", "zone4", "zone5", "zone6" ], "rhodeisland": [ "zone1", "zone2", "zone3", "zone4" ], } }, { "site_name": "MaineOil", "base_url": "https://www.maineoil.com", "url_template": "{base_url}/{zone_slug}.asp?type={oil_type}", "oil_type": 0, "locations": { "maine": [ "zone1", "zone2", "zone3", "zone4", "zone5", "zone6", "zone7" ] } } ] # --- ZONE-TO-COUNTY MAPPING --- # Maps (state_key, zone_number) -> (state_abbrev, county_name) # state_key matches the keys in SITES_CONFIG locations (lowercase, no spaces) # county_name must match the county.name in the database exactly ZONE_COUNTY_MAP = { # Connecticut (10 zones -> 8 counties) ("connecticut", 1): ("CT", "New London"), # Southeast CT ("connecticut", 2): ("CT", "Windham"), # Northeast CT ("connecticut", 3): ("CT", "New Haven"), # New Haven, Bridgeport ("connecticut", 4): ("CT", "Middlesex"), # Southeast Central CT ("connecticut", 5): ("CT", "New Haven"), # Southwest Central CT ("connecticut", 6): ("CT", "Hartford"), # Greater Hartford ("connecticut", 7): ("CT", "Litchfield"), # West CT ("connecticut", 8): ("CT", "Fairfield"), # Southwest CT ("connecticut", 9): ("CT", "Tolland"), # Northeast Central CT ("connecticut", 10): ("CT", "Litchfield"), # Northwest CT # Massachusetts (15 zones -> 14 counties) ("massachusetts", 1): ("MA", "Suffolk"), # South Boston ("massachusetts", 2): ("MA", "Middlesex"), # North Boston ("massachusetts", 3): ("MA", "Norfolk"), # Southwest of Boston ("massachusetts", 4): ("MA", "Plymouth"), # South of Boston ("massachusetts", 5): ("MA", "Middlesex"), # West of Boston ("massachusetts", 6): ("MA", "Bristol"), # Southern Massachusetts ("massachusetts", 7): ("MA", "Barnstable"), # Cape Cod & Islands ("massachusetts", 8): ("MA", "Essex"), # Northwest of Boston ("massachusetts", 9): ("MA", "Essex"), # North of Boston ("massachusetts", 10): ("MA", "Worcester"), # Central Massachusetts ("massachusetts", 11): ("MA", "Worcester"), # East Central Massachusetts ("massachusetts", 12): ("MA", "Hampshire"), # West Central Massachusetts ("massachusetts", 13): ("MA", "Hampden"), # Springfield Area ("massachusetts", 14): ("MA", "Franklin"), # Northwestern Massachusetts ("massachusetts", 15): ("MA", "Berkshire"), # Western Massachusetts # New Hampshire (6 zones -> 10 counties) ("newhampshire", 1): ("NH", "Coos"), # Northern NH ("newhampshire", 2): ("NH", "Strafford"), # Eastern NH ("newhampshire", 3): ("NH", "Merrimack"), # Central NH ("newhampshire", 4): ("NH", "Grafton"), # West Central NH ("newhampshire", 5): ("NH", "Cheshire"), # Southwest NH ("newhampshire", 6): ("NH", "Hillsborough"), # South Central NH # Rhode Island (4 zones -> 5 counties) ("rhodeisland", 1): ("RI", "Newport"), # Southeast RI ("rhodeisland", 2): ("RI", "Providence"), # Northern RI ("rhodeisland", 3): ("RI", "Washington"), # Southwest RI ("rhodeisland", 4): ("RI", "Kent"), # Central RI # Maine (7 zones -> 16 counties, via MaineOil.com) ("maine", 1): ("ME", "Cumberland"), # Greater Portland ("maine", 2): ("ME", "Kennebec"), # Augusta/Waterville ("maine", 3): ("ME", "Androscoggin"), # Auburn/Lewiston/Western ("maine", 4): ("ME", "York"), # Southern Maine ("maine", 5): ("ME", "Knox"), # Mid-Coast ("maine", 6): ("ME", "Penobscot"), # Bangor West ("maine", 7): ("ME", "Washington"), # Downeast } # --- LOGGING CONFIGURATION --- LOG_FILE = "oil_scraper.log" def setup_logging(): """Configure logging for the scraper.""" logging.basicConfig( filename=LOG_FILE, level=logging.INFO, format='%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s' )