Edwin Eames
8f45f4c209
Add ZONE_COUNTY_MAP for all 5 scraped states (42 zone-to-county entries). Scraper now resolves county_id at startup and assigns it to each record. Upsert logic deduplicates by (name, state, county_id) to prevent duplicates when multiple zones map to the same county. Also adds County model for DB lookups and fixes Rhode Island zone count (4, not 5). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
60 lines
2.5 KiB
Python
60 lines
2.5 KiB
Python
# models.py
|
|
from sqlalchemy import Column, Integer, String, Float, Date, Boolean, BigInteger, ForeignKey, DateTime # Added DateTime
|
|
from sqlalchemy.orm import relationship
|
|
from datetime import date, datetime # Import datetime as well
|
|
|
|
# Import Base from our database.py
|
|
from database import Base
|
|
|
|
# --- OilPrice Model ---
|
|
class OilPrice(Base):
|
|
__tablename__ = "oil_prices"
|
|
|
|
id = Column(Integer, primary_key=True, index=True, autoincrement=True)
|
|
state = Column(String(100), index=True)
|
|
zone = Column(Integer, index=True)
|
|
name = Column(String(255), index=True)
|
|
price = Column(Float, nullable=True)
|
|
date = Column(String(20)) # This is the 'Date Posted' from the website
|
|
|
|
# --- UPDATED scrapetimestamp ---
|
|
# To store both date and time of scraping
|
|
scrapetimestamp = Column(DateTime, default=datetime.utcnow, index=True)
|
|
# 'datetime.utcnow' will automatically provide the current UTC date and time
|
|
# when a new record is created and this field is not explicitly set.
|
|
|
|
company_id = Column(Integer, ForeignKey("company.id"), nullable=True)
|
|
county_id = Column(Integer, nullable=True)
|
|
|
|
def __repr__(self):
|
|
return (f"<OilPrice(id={self.id}, state='{self.state}', zone='{self.zone}', "
|
|
f"name='{self.name}', price={self.price}, date='{self.date}', "
|
|
f"county_id={self.county_id}, scraped_at='{self.scrapetimestamp}')>")
|
|
|
|
# --- County Model (read-only, for lookups) ---
|
|
class County(Base):
|
|
__tablename__ = "county"
|
|
|
|
id = Column(Integer, primary_key=True)
|
|
name = Column(String(255))
|
|
state = Column(String(2))
|
|
|
|
# --- Company Model (remains the same) ---
|
|
class Company(Base):
|
|
__tablename__ = "company"
|
|
# ... (fields as before) ...
|
|
id = Column(Integer, primary_key=True, index=True, autoincrement=True)
|
|
active = Column(Boolean, nullable=False, default=True)
|
|
created = Column(Date, nullable=False, default=date.today) # This 'created' is for the company record
|
|
name = Column(String(255), nullable=False, index=True, unique=True)
|
|
address = Column(String(500), nullable=True)
|
|
town = Column(String(100), nullable=True)
|
|
state = Column(String(50), nullable=True)
|
|
phone = Column(String(20), nullable=True)
|
|
owner_name = Column(String(255), nullable=True)
|
|
owner_phone_number = Column(String(20), nullable=True)
|
|
email = Column(String(255), nullable=True, unique=True)
|
|
user_id = Column(Integer, ForeignKey("users.id"), nullable=True, index=True)
|
|
|
|
def __repr__(self):
|
|
return f"<Company(id={self.id}, name='{self.name}', active={self.active})>" |