refactor: replace fuel_scraper with newenglandoil + cheapestoil scrapers
- Add newenglandoil/ package as the primary scraper (replaces fuel_scraper) - Add cheapestoil/ package as a secondary market price scraper - Add app.py entry point for direct execution - Update run.py: new scrape_cheapest(), migrate command, --state filter, --refresh-metadata flag for overwriting existing phone/URL data - Update models.py with latest schema fields - Update requirements.txt dependencies - Update Dockerfile and docker-compose.yml for new structure - Remove deprecated fuel_scraper module, test.py, and log file Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
16
models.py
16
models.py
@@ -25,6 +25,8 @@ class OilPrice(Base):
|
||||
|
||||
company_id = Column(Integer, ForeignKey("company.id"), nullable=True)
|
||||
county_id = Column(Integer, nullable=True)
|
||||
phone = Column(String(20), nullable=True)
|
||||
url = Column(String(500), nullable=True)
|
||||
|
||||
def __repr__(self):
|
||||
return (f"<OilPrice(id={self.id}, state='{self.state}', zone='{self.zone}', "
|
||||
@@ -57,4 +59,16 @@ class Company(Base):
|
||||
user_id = Column(Integer, ForeignKey("users.id"), nullable=True, index=True)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Company(id={self.id}, name='{self.name}', active={self.active})>"
|
||||
return f"<Company(id={self.id}, name='{self.name}', active={self.active})>"
|
||||
|
||||
# --- StatsPrice Model ---
|
||||
class StatsPrice(Base):
|
||||
__tablename__ = "stats_prices"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True, autoincrement=True)
|
||||
state = Column(String(2), nullable=False)
|
||||
price = Column(Float, nullable=False)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<StatsPrice(state='{self.state}', price={self.price})>"
|
||||
Reference in New Issue
Block a user