feat: initial commit for oil price scraper service
FastAPI-based scraper for commodity ticker prices (HO, CL, RB futures) and competitor oil pricing from NewEnglandOil. Includes cron-driven scraping, PostgreSQL storage, and REST endpoints for price retrieval. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
99
app/models.py
Normal file
99
app/models.py
Normal file
@@ -0,0 +1,99 @@
|
||||
"""
|
||||
SQLAlchemy models for eamco_scraper.
|
||||
|
||||
This module defines the database models for storing scraped oil price data.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, Integer, String, Numeric, Date, DateTime, Index
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
class CompanyPrice(Base):
|
||||
"""
|
||||
Model for storing oil company pricing data.
|
||||
|
||||
This table stores historical pricing data from oil companies.
|
||||
Each scrape creates new records (no updates) to enable price trend analysis.
|
||||
|
||||
Attributes:
|
||||
id: Primary key
|
||||
company_name: Name of the oil company
|
||||
town: Town where the company operates
|
||||
price_decimal: Price per gallon (e.g., 2.599)
|
||||
scrape_date: Date when the price was listed on the website
|
||||
zone: Geographic zone (default: 'zone10' for Central Massachusetts)
|
||||
created_at: Timestamp when the record was inserted into database
|
||||
"""
|
||||
|
||||
__tablename__ = "company_prices"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
company_name = Column(String(255), nullable=False, index=True)
|
||||
town = Column(String(100), nullable=True)
|
||||
price_decimal = Column(Numeric(6, 3), nullable=False)
|
||||
scrape_date = Column(Date, nullable=False, index=True)
|
||||
zone = Column(String(50), nullable=False, default="zone10", index=True)
|
||||
created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<CompanyPrice(company='{self.company_name}', price={self.price_decimal}, date={self.scrape_date})>"
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert model instance to dictionary for JSON serialization."""
|
||||
return {
|
||||
"id": self.id,
|
||||
"company_name": self.company_name,
|
||||
"town": self.town,
|
||||
"price_decimal": float(self.price_decimal) if self.price_decimal else None,
|
||||
"scrape_date": self.scrape_date.isoformat() if self.scrape_date else None,
|
||||
"zone": self.zone,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
# Create composite indexes for common queries
|
||||
Index('idx_company_prices_company_date', CompanyPrice.company_name, CompanyPrice.scrape_date)
|
||||
Index('idx_company_prices_zone_date', CompanyPrice.zone, CompanyPrice.scrape_date)
|
||||
|
||||
|
||||
class TickerPrice(Base):
|
||||
"""
|
||||
Model for storing ticker prices (Stocks/Commodities).
|
||||
|
||||
Attributes:
|
||||
id: Primary key
|
||||
symbol: Ticker symbol (e.g., HO=F, CL=F, RB=F)
|
||||
price_decimal: Current price
|
||||
currency: Currency code (e.g., USD)
|
||||
change_decimal: Price change amount
|
||||
percent_change_decimal: Price change percentage
|
||||
timestamp: Time of scrape
|
||||
"""
|
||||
|
||||
__tablename__ = "ticker_prices"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
symbol = Column(String(20), nullable=False, index=True)
|
||||
price_decimal = Column(Numeric(10, 4), nullable=False)
|
||||
currency = Column(String(10), nullable=True)
|
||||
change_decimal = Column(Numeric(10, 4), nullable=True)
|
||||
percent_change_decimal = Column(Numeric(10, 4), nullable=True)
|
||||
timestamp = Column(DateTime, nullable=False, default=datetime.utcnow, index=True)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<TickerPrice(symbol='{self.symbol}', price={self.price_decimal}, time={self.timestamp})>"
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"id": self.id,
|
||||
"symbol": self.symbol,
|
||||
"price": float(self.price_decimal) if self.price_decimal is not None else None,
|
||||
"currency": self.currency,
|
||||
"change": float(self.change_decimal) if self.change_decimal is not None else None,
|
||||
"percent_change": float(self.percent_change_decimal) if self.percent_change_decimal is not None else None,
|
||||
"timestamp": self.timestamp.isoformat() if self.timestamp else None,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user