Edwin Eames
af9c2f99e7
FastAPI-based scraper for commodity ticker prices (HO, CL, RB futures) and competitor oil pricing from NewEnglandOil. Includes cron-driven scraping, PostgreSQL storage, and REST endpoints for price retrieval. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
100 lines
3.8 KiB
Python
100 lines
3.8 KiB
Python
"""
|
|
SQLAlchemy models for eamco_scraper.
|
|
|
|
This module defines the database models for storing scraped oil price data.
|
|
"""
|
|
|
|
from datetime import datetime
|
|
from sqlalchemy import Column, Integer, String, Numeric, Date, DateTime, Index
|
|
from sqlalchemy.ext.declarative import declarative_base
|
|
|
|
Base = declarative_base()
|
|
|
|
|
|
class CompanyPrice(Base):
|
|
"""
|
|
Model for storing oil company pricing data.
|
|
|
|
This table stores historical pricing data from oil companies.
|
|
Each scrape creates new records (no updates) to enable price trend analysis.
|
|
|
|
Attributes:
|
|
id: Primary key
|
|
company_name: Name of the oil company
|
|
town: Town where the company operates
|
|
price_decimal: Price per gallon (e.g., 2.599)
|
|
scrape_date: Date when the price was listed on the website
|
|
zone: Geographic zone (default: 'zone10' for Central Massachusetts)
|
|
created_at: Timestamp when the record was inserted into database
|
|
"""
|
|
|
|
__tablename__ = "company_prices"
|
|
|
|
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
company_name = Column(String(255), nullable=False, index=True)
|
|
town = Column(String(100), nullable=True)
|
|
price_decimal = Column(Numeric(6, 3), nullable=False)
|
|
scrape_date = Column(Date, nullable=False, index=True)
|
|
zone = Column(String(50), nullable=False, default="zone10", index=True)
|
|
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
|
|
|
def __repr__(self):
|
|
return f"<CompanyPrice(company='{self.company_name}', price={self.price_decimal}, date={self.scrape_date})>"
|
|
|
|
def to_dict(self):
|
|
"""Convert model instance to dictionary for JSON serialization."""
|
|
return {
|
|
"id": self.id,
|
|
"company_name": self.company_name,
|
|
"town": self.town,
|
|
"price_decimal": float(self.price_decimal) if self.price_decimal else None,
|
|
"scrape_date": self.scrape_date.isoformat() if self.scrape_date else None,
|
|
"zone": self.zone,
|
|
"created_at": self.created_at.isoformat() if self.created_at else None,
|
|
}
|
|
|
|
|
|
# Create composite indexes for common queries
|
|
Index('idx_company_prices_company_date', CompanyPrice.company_name, CompanyPrice.scrape_date)
|
|
Index('idx_company_prices_zone_date', CompanyPrice.zone, CompanyPrice.scrape_date)
|
|
|
|
|
|
class TickerPrice(Base):
|
|
"""
|
|
Model for storing ticker prices (Stocks/Commodities).
|
|
|
|
Attributes:
|
|
id: Primary key
|
|
symbol: Ticker symbol (e.g., HO=F, CL=F, RB=F)
|
|
price_decimal: Current price
|
|
currency: Currency code (e.g., USD)
|
|
change_decimal: Price change amount
|
|
percent_change_decimal: Price change percentage
|
|
timestamp: Time of scrape
|
|
"""
|
|
|
|
__tablename__ = "ticker_prices"
|
|
|
|
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
symbol = Column(String(20), nullable=False, index=True)
|
|
price_decimal = Column(Numeric(10, 4), nullable=False)
|
|
currency = Column(String(10), nullable=True)
|
|
change_decimal = Column(Numeric(10, 4), nullable=True)
|
|
percent_change_decimal = Column(Numeric(10, 4), nullable=True)
|
|
timestamp = Column(DateTime, nullable=False, default=datetime.utcnow, index=True)
|
|
|
|
def __repr__(self):
|
|
return f"<TickerPrice(symbol='{self.symbol}', price={self.price_decimal}, time={self.timestamp})>"
|
|
|
|
def to_dict(self):
|
|
return {
|
|
"id": self.id,
|
|
"symbol": self.symbol,
|
|
"price": float(self.price_decimal) if self.price_decimal is not None else None,
|
|
"currency": self.currency,
|
|
"change": float(self.change_decimal) if self.change_decimal is not None else None,
|
|
"percent_change": float(self.percent_change_decimal) if self.percent_change_decimal is not None else None,
|
|
"timestamp": self.timestamp.isoformat() if self.timestamp else None,
|
|
}
|
|
|