feat: initial commit for oil price scraper service

FastAPI-based scraper for commodity ticker prices (HO, CL, RB futures)
and competitor oil pricing from NewEnglandOil. Includes cron-driven
scraping, PostgreSQL storage, and REST endpoints for price retrieval.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-08 17:57:44 -05:00
commit af9c2f99e7
25 changed files with 1566 additions and 0 deletions

75
test.sh Executable file
View File

@@ -0,0 +1,75 @@
#!/bin/bash
# Test script for eamco_scraper service
echo "=== eamco_scraper Test Script ==="
echo ""
# Check if service directory exists
if [ ! -d "/mnt/code/oil/eamco/eamco_scraper" ]; then
echo "❌ Service directory not found"
exit 1
fi
echo "✅ Service directory exists"
echo ""
# Test 1: Build Docker image
echo "Test 1: Building Docker image..."
cd /mnt/code/oil/eamco/eamco_deploy
docker-compose -f docker-compose.local.yml build scraper_local
if [ $? -eq 0 ]; then
echo "✅ Docker build successful"
else
echo "❌ Docker build failed"
exit 1
fi
echo ""
# Test 2: Start the service
echo "Test 2: Starting service..."
docker-compose -f docker-compose.local.yml up -d scraper_local
if [ $? -eq 0 ]; then
echo "✅ Service started"
else
echo "❌ Service failed to start"
exit 1
fi
echo ""
echo "Waiting 5 seconds for service to initialize..."
sleep 5
# Test 3: Health check
echo "Test 3: Testing health endpoint..."
HEALTH_RESPONSE=$(curl -s http://localhost:9619/health)
if echo "$HEALTH_RESPONSE" | grep -q "status"; then
echo "✅ Health endpoint responding"
echo "Response: $HEALTH_RESPONSE"
else
echo "❌ Health endpoint not responding"
fi
echo ""
# Test 4: Scraper endpoint
echo "Test 4: Testing scraper endpoint..."
echo "This will scrape live data from New England Oil..."
SCRAPER_RESPONSE=$(curl -s http://localhost:9619/scraper/newenglandoil/latestprice)
if echo "$SCRAPER_RESPONSE" | grep -q "status"; then
echo "✅ Scraper endpoint responding"
echo "Response preview:"
echo "$SCRAPER_RESPONSE" | head -n 20
else
echo "❌ Scraper endpoint not responding"
fi
echo ""
echo "=== Test Complete ==="
echo ""
echo "To view logs: docker-compose -f docker-compose.local.yml logs scraper_local"
echo "To stop service: docker-compose -f docker-compose.local.yml down scraper_local"