feat: initial commit for oil price scraper service

FastAPI-based scraper for commodity ticker prices (HO, CL, RB futures)
and competitor oil pricing from NewEnglandOil. Includes cron-driven
scraping, PostgreSQL storage, and REST endpoints for price retrieval.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-08 17:57:44 -05:00
commit af9c2f99e7
25 changed files with 1566 additions and 0 deletions

99
app/models.py Normal file
View File

@@ -0,0 +1,99 @@
"""
SQLAlchemy models for eamco_scraper.
This module defines the database models for storing scraped oil price data.
"""
from datetime import datetime
from sqlalchemy import Column, Integer, String, Numeric, Date, DateTime, Index
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class CompanyPrice(Base):
"""
Model for storing oil company pricing data.
This table stores historical pricing data from oil companies.
Each scrape creates new records (no updates) to enable price trend analysis.
Attributes:
id: Primary key
company_name: Name of the oil company
town: Town where the company operates
price_decimal: Price per gallon (e.g., 2.599)
scrape_date: Date when the price was listed on the website
zone: Geographic zone (default: 'zone10' for Central Massachusetts)
created_at: Timestamp when the record was inserted into database
"""
__tablename__ = "company_prices"
id = Column(Integer, primary_key=True, autoincrement=True)
company_name = Column(String(255), nullable=False, index=True)
town = Column(String(100), nullable=True)
price_decimal = Column(Numeric(6, 3), nullable=False)
scrape_date = Column(Date, nullable=False, index=True)
zone = Column(String(50), nullable=False, default="zone10", index=True)
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
def __repr__(self):
return f"<CompanyPrice(company='{self.company_name}', price={self.price_decimal}, date={self.scrape_date})>"
def to_dict(self):
"""Convert model instance to dictionary for JSON serialization."""
return {
"id": self.id,
"company_name": self.company_name,
"town": self.town,
"price_decimal": float(self.price_decimal) if self.price_decimal else None,
"scrape_date": self.scrape_date.isoformat() if self.scrape_date else None,
"zone": self.zone,
"created_at": self.created_at.isoformat() if self.created_at else None,
}
# Create composite indexes for common queries
Index('idx_company_prices_company_date', CompanyPrice.company_name, CompanyPrice.scrape_date)
Index('idx_company_prices_zone_date', CompanyPrice.zone, CompanyPrice.scrape_date)
class TickerPrice(Base):
"""
Model for storing ticker prices (Stocks/Commodities).
Attributes:
id: Primary key
symbol: Ticker symbol (e.g., HO=F, CL=F, RB=F)
price_decimal: Current price
currency: Currency code (e.g., USD)
change_decimal: Price change amount
percent_change_decimal: Price change percentage
timestamp: Time of scrape
"""
__tablename__ = "ticker_prices"
id = Column(Integer, primary_key=True, autoincrement=True)
symbol = Column(String(20), nullable=False, index=True)
price_decimal = Column(Numeric(10, 4), nullable=False)
currency = Column(String(10), nullable=True)
change_decimal = Column(Numeric(10, 4), nullable=True)
percent_change_decimal = Column(Numeric(10, 4), nullable=True)
timestamp = Column(DateTime, nullable=False, default=datetime.utcnow, index=True)
def __repr__(self):
return f"<TickerPrice(symbol='{self.symbol}', price={self.price_decimal}, time={self.timestamp})>"
def to_dict(self):
return {
"id": self.id,
"symbol": self.symbol,
"price": float(self.price_decimal) if self.price_decimal is not None else None,
"currency": self.currency,
"change": float(self.change_decimal) if self.change_decimal is not None else None,
"percent_change": float(self.percent_change_decimal) if self.percent_change_decimal is not None else None,
"timestamp": self.timestamp.isoformat() if self.timestamp else None,
}