feat: initial commit for oil price scraper service
FastAPI-based scraper for commodity ticker prices (HO, CL, RB futures) and competitor oil pricing from NewEnglandOil. Includes cron-driven scraping, PostgreSQL storage, and REST endpoints for price retrieval. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
0
app/priceticker/__init__.py
Normal file
0
app/priceticker/__init__.py
Normal file
72
app/priceticker/router.py
Normal file
72
app/priceticker/router.py
Normal file
@@ -0,0 +1,72 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import List
|
||||
import logging
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import TickerPrice
|
||||
from app.priceticker.scraper import fetch_ticker_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/scraper/priceticker",
|
||||
tags=["Price Ticker"]
|
||||
)
|
||||
|
||||
@router.post("/update")
|
||||
async def update_prices(db: Session = Depends(get_db)):
|
||||
"""
|
||||
Trigger an immediate update of stock/commodity prices.
|
||||
"""
|
||||
logger.info("Triggering ticker update...")
|
||||
data = fetch_ticker_data()
|
||||
|
||||
if not data:
|
||||
raise HTTPException(status_code=500, detail="Failed to fetch ticker data")
|
||||
|
||||
try:
|
||||
saved_records = []
|
||||
for item in data:
|
||||
record = TickerPrice(
|
||||
symbol=item["symbol"],
|
||||
price_decimal=item["price"],
|
||||
currency=item["currency"],
|
||||
change_decimal=item["change"],
|
||||
percent_change_decimal=item["percent_change"],
|
||||
timestamp=item["timestamp"]
|
||||
)
|
||||
db.add(record)
|
||||
saved_records.append(record)
|
||||
|
||||
db.commit()
|
||||
return {"status": "success", "updated": len(saved_records)}
|
||||
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.error(f"Database error saving tickers: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Database error: {str(e)}")
|
||||
|
||||
@router.get("/latest")
|
||||
async def get_latest_prices(db: Session = Depends(get_db)):
|
||||
"""
|
||||
Get the most recent price for each ticker symbol.
|
||||
"""
|
||||
# Subquery to find the latest timestamp for each symbol
|
||||
# This is a bit complex in pure ORM, so we might do it simply for now:
|
||||
# 1. Get list of distinct symbols we care about
|
||||
# 2. Query latest for each
|
||||
|
||||
results = []
|
||||
# We know the symbols we care about: HO=F, CL=F, RB=F
|
||||
TARGET_SYMBOLS = ["HO=F", "CL=F", "RB=F"]
|
||||
|
||||
for symbol in TARGET_SYMBOLS:
|
||||
latest = db.query(TickerPrice).filter(
|
||||
TickerPrice.symbol == symbol
|
||||
).order_by(TickerPrice.timestamp.desc()).first()
|
||||
|
||||
if latest:
|
||||
results.append(latest.to_dict())
|
||||
|
||||
return results
|
||||
65
app/priceticker/scraper.py
Normal file
65
app/priceticker/scraper.py
Normal file
@@ -0,0 +1,65 @@
|
||||
import logging
|
||||
import yfinance as yf
|
||||
from decimal import Decimal
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Ticker mapping
|
||||
# HO=F : Heating Oil
|
||||
# CL=F : Crude Oil
|
||||
# RB=F : RBOB Gasoline
|
||||
TICKERS = ["HO=F", "CL=F", "RB=F"]
|
||||
|
||||
def fetch_ticker_data() -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Fetch current data for oil tickers from Yahoo Finance.
|
||||
|
||||
Returns:
|
||||
List of dictionaries containing ticker data.
|
||||
"""
|
||||
results = []
|
||||
|
||||
try:
|
||||
# Fetch data for all tickers at once
|
||||
tickers_str = " ".join(TICKERS)
|
||||
data = yf.Tickers(tickers_str)
|
||||
|
||||
for symbol in TICKERS:
|
||||
try:
|
||||
ticker = data.tickers[symbol]
|
||||
# Fast info usually contains the latest price
|
||||
info = ticker.fast_info
|
||||
|
||||
# Fallback to history if fast_info is missing crucial data (simplified here)
|
||||
# But fast_info is generally faster and sufficient for last_price
|
||||
|
||||
last_price = info.last_price
|
||||
previous_close = info.previous_close
|
||||
|
||||
if last_price is None:
|
||||
logger.warning(f"No price found for {symbol}")
|
||||
continue
|
||||
|
||||
change = last_price - previous_close
|
||||
percent_change = (change / previous_close) * 100 if previous_close else 0
|
||||
|
||||
results.append({
|
||||
"symbol": symbol,
|
||||
"price": Decimal(str(last_price)),
|
||||
"currency": info.currency,
|
||||
"change": Decimal(str(change)),
|
||||
"percent_change": Decimal(str(percent_change)),
|
||||
"timestamp": datetime.utcnow()
|
||||
})
|
||||
|
||||
logger.info(f"Fetched {symbol}: {last_price} ({percent_change:.2f}%)")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing {symbol}: {e}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching ticker data: {e}")
|
||||
|
||||
return results
|
||||
Reference in New Issue
Block a user