diff --git a/.github/MONITORING_GUIDE.md b/.github/MONITORING_GUIDE.md index e408436..7c93c95 100644 --- a/.github/MONITORING_GUIDE.md +++ b/.github/MONITORING_GUIDE.md @@ -1,565 +1,14 @@ -# SDK Monitoring & Alerting Guide +# Python SDK monitoring -This guide explains how to monitor OilPriceAPI Python SDK health and catch issues like the v1.4.1 timeout bug before they affect users. +Current SDK monitoring has two layers: -## Overview +1. `Live API Tests` exercises keyless and keyed integration paths on pushes and + pull requests. +2. `Scheduled SDK Synthetic` runs hourly, checks the latest-price and bounded + history customer paths, and retains a privacy-safe JSON receipt for 30 days. -**Problem**: The v1.4.1 historical timeout bug wasn't detected until a customer reported it. +The operational design and response runbook live in +[`docs/SYNTHETIC_MONITORING.md`](../docs/SYNTHETIC_MONITORING.md). -**Solution**: Proactive monitoring of SDK health metrics to detect issues before customer impact. - -## Monitoring Architecture - -``` -┌─────────────────┐ -│ SDK Usage │ -│ (PyPI Stats) │ -└────────┬────────┘ - │ - v -┌─────────────────┐ ┌──────────────┐ ┌─────────────┐ -│ Synthetic │────>│ Prometheus │────>│ Grafana │ -│ Monitoring │ │ (Metrics) │ │ (Dashboards)│ -└─────────────────┘ └──────┬───────┘ └─────────────┘ - │ - v - ┌──────────────┐ - │ Alertmanager│ - │ (PagerDuty) │ - └──────────────┘ -``` - -## Metrics to Monitor - -### 1. SDK Health Metrics - -#### Download Statistics (PyPI) -- **Metric**: `sdk_downloads_total` -- **Source**: PyPI JSON API -- **Alert**: Downloads drop >50% week-over-week -- **Why**: Indicates major issue causing users to stop upgrading - -#### Version Adoption Rate -- **Metric**: `sdk_version_distribution` -- **Source**: PyPI stats / telemetry (if enabled) -- **Alert**: Latest version adoption <10% after 7 days -- **Why**: Users not upgrading = possible issue with new version - -### 2. Synthetic Monitoring - -#### Historical Query Tests (The Critical Test) -- **Metric**: `sdk_historical_query_duration_seconds` -- **Test**: Run 1-week, 1-month, 1-year queries every 15 minutes -- **Alert**: - - 1-week query >30s - - 1-month query >60s - - 1-year query >120s or timeout -- **Why**: Would have caught the v1.4.1 timeout bug - -#### Endpoint Selection Verification -- **Metric**: `sdk_endpoint_selection_correct` -- **Test**: Verify SDK selects correct endpoint for date range -- **Alert**: Wrong endpoint selected -- **Why**: Would have caught v1.4.1 hardcoded endpoint bug - -### 3. Error Tracking - -#### SDK Exceptions -- **Tool**: Sentry / Application Insights -- **Track**: Timeout errors, authentication failures, connection errors -- **Alert**: Error rate >1% of requests -- **Why**: Early warning of SDK or API issues - -### 4. API Response Times - -#### Backend Performance -- **Metric**: `api_response_duration_seconds` -- **Track**: P50, P95, P99 response times by endpoint -- **Alert**: P95 >30s for past_week endpoint -- **Why**: Backend slowness affects SDK performance - -## Implementation - -### 1. Synthetic Monitoring Script - -Create `/scripts/synthetic_monitor.py`: - -```python -""" -Synthetic monitoring for OilPriceAPI SDK. - -Runs continuous health checks and reports metrics to Prometheus. -""" - -import time -from datetime import datetime, timedelta -from prometheus_client import start_http_server, Gauge, Counter, Histogram -from oilpriceapi import OilPriceAPI - -# Metrics -QUERY_DURATION = Histogram( - 'sdk_historical_query_duration_seconds', - 'Historical query duration', - ['query_type', 'commodity'] -) - -QUERY_SUCCESS = Counter( - 'sdk_historical_query_success_total', - 'Successful historical queries', - ['query_type'] -) - -QUERY_FAILURE = Counter( - 'sdk_historical_query_failure_total', - 'Failed historical queries', - ['query_type', 'error_type'] -) - -ENDPOINT_CORRECTNESS = Gauge( - 'sdk_endpoint_selection_correct', - 'Whether SDK selected correct endpoint', - ['query_type'] -) - -def test_1_week_query(client): - """Test 1-week historical query.""" - start_time = time.time() - try: - end_date = datetime.now() - start_date = end_date - timedelta(days=7) - - history = client.historical.get( - commodity="WTI_USD", - start_date=start_date.strftime("%Y-%m-%d"), - end_date=end_date.strftime("%Y-%m-%d"), - interval="daily" - ) - - duration = time.time() - start_time - QUERY_DURATION.labels(query_type='1_week', commodity='WTI_USD').observe(duration) - QUERY_SUCCESS.labels(query_type='1_week').inc() - - # Alert if too slow (would catch v1.4.1 bug) - if duration < 30: - ENDPOINT_CORRECTNESS.labels(query_type='1_week').set(1) - return True - else: - ENDPOINT_CORRECTNESS.labels(query_type='1_week').set(0) - print(f"WARNING: 1-week query took {duration}s (expected <30s)") - return False - - except Exception as e: - QUERY_FAILURE.labels(query_type='1_week', error_type=type(e).__name__).inc() - ENDPOINT_CORRECTNESS.labels(query_type='1_week').set(0) - print(f"ERROR: 1-week query failed: {e}") - return False - - -def test_1_year_query(client): - """Test 1-year historical query.""" - start_time = time.time() - try: - history = client.historical.get( - commodity="WTI_USD", - start_date="2024-01-01", - end_date="2024-12-31", - interval="daily" - ) - - duration = time.time() - start_time - QUERY_DURATION.labels(query_type='1_year', commodity='WTI_USD').observe(duration) - QUERY_SUCCESS.labels(query_type='1_year').inc() - - # Alert if timeout (would catch v1.4.1 bug) - if duration < 120: - ENDPOINT_CORRECTNESS.labels(query_type='1_year').set(1) - return True - else: - ENDPOINT_CORRECTNESS.labels(query_type='1_year').set(0) - print(f"WARNING: 1-year query took {duration}s (expected <120s)") - return False - - except Exception as e: - QUERY_FAILURE.labels(query_type='1_year', error_type=type(e).__name__).inc() - ENDPOINT_CORRECTNESS.labels(query_type='1_year').set(0) - print(f"ERROR: 1-year query failed: {e}") - return False - - -def run_synthetic_tests(client): - """Run all synthetic tests.""" - print(f"[{datetime.now()}] Running synthetic tests...") - - test_1_week_query(client) - test_1_year_query(client) - - print(f"[{datetime.now()}] Tests complete") - - -def main(): - """Main monitoring loop.""" - import os - - api_key = os.getenv('OILPRICEAPI_KEY') - if not api_key: - print("ERROR: OILPRICEAPI_KEY environment variable not set") - return - - # Start Prometheus metrics server - port = int(os.getenv('METRICS_PORT', '8000')) - start_http_server(port) - print(f"Metrics server started on port {port}") - - # Create SDK client - client = OilPriceAPI(api_key=api_key) - - # Run tests every 15 minutes - interval = int(os.getenv('TEST_INTERVAL', '900')) # 15 minutes - - while True: - try: - run_synthetic_tests(client) - except Exception as e: - print(f"ERROR in monitoring loop: {e}") - - time.sleep(interval) - - -if __name__ == '__main__': - main() -``` - -### 2. Docker Container for Monitoring - -`Dockerfile.monitor`: - -```dockerfile -FROM python:3.11-slim - -WORKDIR /app - -# Install SDK and dependencies -RUN pip install oilpriceapi prometheus_client - -# Copy monitoring script -COPY scripts/synthetic_monitor.py . - -# Expose Prometheus metrics port -EXPOSE 8000 - -# Run monitoring -CMD ["python", "synthetic_monitor.py"] -``` - -Run with: -```bash -docker build -f Dockerfile.monitor -t oilpriceapi-monitor . -docker run -e OILPRICEAPI_KEY=$OILPRICEAPI_KEY -p 8000:8000 oilpriceapi-monitor -``` - -### 3. Prometheus Configuration - -`prometheus.yml`: - -```yaml -global: - scrape_interval: 60s - evaluation_interval: 60s - -scrape_configs: - - job_name: 'oilpriceapi_sdk_monitor' - static_configs: - - targets: ['localhost:8000'] - metrics_path: /metrics - - - job_name: 'pypi_stats' - scrape_interval: 3600s # Every hour - static_configs: - - targets: ['pypi-exporter:9101'] - -rule_files: - - 'alert_rules.yml' - -alerting: - alertmanagers: - - static_configs: - - targets: ['alertmanager:9093'] -``` - -### 4. Alert Rules - -`alert_rules.yml`: - -```yaml -groups: - - name: sdk_health - interval: 60s - rules: - # CRITICAL: Would have caught v1.4.1 bug - - alert: HistoricalQuery1WeekSlow - expr: sdk_historical_query_duration_seconds{query_type="1_week"} > 30 - for: 5m - labels: - severity: critical - component: sdk - annotations: - summary: "SDK 1-week queries are slow" - description: "1-week historical queries taking >30s (v1.4.1 bug)" - runbook: "Check endpoint selection logic in historical.py" - - - alert: HistoricalQuery1YearTimeout - expr: sdk_historical_query_duration_seconds{query_type="1_year"} > 120 - for: 5m - labels: - severity: critical - component: sdk - annotations: - summary: "SDK 1-year queries timing out" - description: "1-year queries taking >120s or timing out (v1.4.1 bug)" - runbook: "Check timeout configuration and endpoint selection" - - - alert: HistoricalQueryFailures - expr: rate(sdk_historical_query_failure_total[5m]) > 0.1 - for: 10m - labels: - severity: warning - component: sdk - annotations: - summary: "High SDK query failure rate" - description: "More than 10% of queries failing" - - - alert: EndpointSelectionWrong - expr: sdk_endpoint_selection_correct{query_type="1_week"} == 0 - for: 5m - labels: - severity: critical - component: sdk - annotations: - summary: "SDK selecting wrong endpoint" - description: "SDK not using optimized endpoint for date range (v1.4.1 bug)" - runbook: "Check _get_optimal_endpoint() in historical.py" - - - name: sdk_adoption - interval: 3600s - rules: - - alert: LowVersionAdoption - expr: | - (sdk_version_downloads{version=~".*latest.*"} / - sum(sdk_version_downloads)) < 0.10 - for: 7d - labels: - severity: warning - component: releases - annotations: - summary: "Low adoption of latest SDK version" - description: "Less than 10% of downloads are latest version after 7 days" - - - alert: DownloadsDrop - expr: | - (sdk_downloads_total offset 7d) - sdk_downloads_total > - (sdk_downloads_total offset 7d) * 0.5 - for: 1d - labels: - severity: warning - component: releases - annotations: - summary: "SDK downloads dropped significantly" - description: "Downloads down >50% week-over-week" -``` - -### 5. Grafana Dashboard - -Import this JSON dashboard for monitoring SDK health: - -**Key Panels:** -1. Historical Query Duration (1-week, 1-month, 1-year) -2. Query Success Rate -3. Endpoint Selection Correctness -4. PyPI Download Trends -5. Version Distribution - -See `grafana_dashboard.json` in this directory. - -### 6. PagerDuty Integration - -Configure Alertmanager to page on critical alerts: - -`alertmanager.yml`: - -```yaml -global: - resolve_timeout: 5m - -route: - receiver: 'default' - group_by: ['alertname', 'severity'] - group_wait: 10s - group_interval: 10s - repeat_interval: 12h - routes: - - match: - severity: critical - receiver: 'pagerduty' - - match: - severity: warning - receiver: 'slack' - -receivers: - - name: 'default' - email_configs: - - to: 'sdk-team@oilpriceapi.com' - - - name: 'pagerduty' - pagerduty_configs: - - service_key: '' - description: '{{ .CommonAnnotations.summary }}' - - - name: 'slack' - slack_configs: - - api_url: '' - channel: '#sdk-alerts' - text: '{{ .CommonAnnotations.description }}' -``` - -## Deployment - -### Quick Start (Docker Compose) - -```yaml -version: '3' -services: - sdk-monitor: - build: - context: . - dockerfile: Dockerfile.monitor - environment: - - OILPRICEAPI_KEY=${OILPRICEAPI_KEY} - - METRICS_PORT=8000 - - TEST_INTERVAL=900 - ports: - - "8000:8000" - restart: unless-stopped - - prometheus: - image: prom/prometheus - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml - - ./alert_rules.yml:/etc/prometheus/alert_rules.yml - ports: - - "9090:9090" - restart: unless-stopped - - grafana: - image: grafana/grafana - environment: - - GF_SECURITY_ADMIN_PASSWORD=secret - ports: - - "3000:3000" - volumes: - - grafana_data:/var/lib/grafana - restart: unless-stopped - - alertmanager: - image: prom/alertmanager - volumes: - - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml - ports: - - "9093:9093" - restart: unless-stopped - -volumes: - grafana_data: -``` - -Run with: -```bash -docker-compose up -d -``` - -Access: -- Grafana: http://localhost:3000 -- Prometheus: http://localhost:9090 -- Metrics: http://localhost:8000/metrics - -## Testing the Monitoring - -### 1. Verify metrics are being collected: -```bash -curl http://localhost:8000/metrics | grep sdk_historical -``` - -### 2. Trigger test alert (simulate v1.4.1 bug): -```bash -# Temporarily break endpoint selection to trigger alert -# This would simulate the v1.4.1 bug -``` - -### 3. Check alert fired in Prometheus: -- Go to http://localhost:9090/alerts -- Should see "HistoricalQuery1WeekSlow" or "EndpointSelectionWrong" - -### 4. Verify PagerDuty received alert: -- Check PagerDuty incidents -- Should receive page for critical alerts - -## Cost Considerations - -### Free Tier Options: -- **Prometheus**: Self-hosted (free) -- **Grafana**: Self-hosted (free) or Cloud free tier -- **Synthetic Monitoring**: ~$5-10/month for small VM -- **Total**: ~$10/month for complete monitoring - -### Paid Options: -- **Datadog**: ~$15/host/month (includes everything) -- **New Relic**: ~$25/month (includes synthetics) -- **Grafana Cloud**: Free tier, then $8/month - -## Maintenance - -### Weekly: -- Review synthetic test results -- Check for new alert patterns -- Update baseline thresholds if needed - -### Monthly: -- Review PyPI download trends -- Analyze version adoption rates -- Update dashboard based on new insights - -### Per Release: -- Add synthetic tests for new features -- Update alert thresholds if performance improves -- Document new metrics in runbooks - -## What This Would Have Prevented - -### v1.4.1 Historical Timeout Bug - -**Detection:** Within 15 minutes of release via synthetic monitoring - -**Alerts Triggered:** -1. `HistoricalQuery1WeekSlow` - 7-day query took 67s (expected <30s) -2. `HistoricalQuery1YearTimeout` - 1-year query timed out at 30s -3. `EndpointSelectionWrong` - SDK using wrong endpoint - -**Response:** -1. PagerDuty pages on-call engineer -2. Check synthetic test logs -3. Identify endpoint selection bug -4. Rollback v1.4.1, release v1.4.2 fix -5. Customer never affected - -## Related Issues - -- [#20](https://github.com/OilpriceAPI/python-sdk/issues/20) - Integration tests -- [#21](https://github.com/OilpriceAPI/python-sdk/issues/21) - Performance baselines -- [#22](https://github.com/OilpriceAPI/python-sdk/issues/22) - Pre-release validation -- [#23](https://github.com/OilpriceAPI/python-sdk/issues/23) - Monitoring & alerting (this document) - -## Resources - -- [Prometheus Documentation](https://prometheus.io/docs/) -- [Grafana Dashboards](https://grafana.com/grafana/dashboards/) -- [PyPI Stats API](https://pypistats.org/api/) -- [Best Practices for SDK Monitoring](https://example.com) +API-service availability, infrastructure metrics, and incident alerting belong +to the API service rather than this SDK repository. diff --git a/.github/workflows/weekly-health.yml b/.github/workflows/weekly-health.yml index f03053d..ac5c7e6 100644 --- a/.github/workflows/weekly-health.yml +++ b/.github/workflows/weekly-health.yml @@ -1,21 +1,24 @@ -name: Weekly SDK Health Check +name: Scheduled SDK Synthetic on: schedule: - - cron: "0 14 * * 1" # Monday 6am PST / 2pm UTC + - cron: "17 * * * *" workflow_dispatch: {} +permissions: + contents: read + +concurrency: + group: python-sdk-synthetic + cancel-in-progress: false + jobs: health-check: - name: Integration Health Check + name: Latest + bounded history runs-on: ubuntu-latest + timeout-minutes: 10 env: - # Both names on purpose: the integration/contract conftests read - # OILPRICEAPI_KEY, while the live futures/subscriptions/well-production - # tests read OILPRICEAPI_TEST_KEY. Exporting only one silently skipped - # the other half of the suite (#48). OILPRICEAPI_KEY: ${{ secrets.OILPRICEAPI_TEST_KEY }} - OILPRICEAPI_TEST_KEY: ${{ secrets.OILPRICEAPI_TEST_KEY }} steps: - uses: actions/checkout@v7 @@ -30,31 +33,31 @@ jobs: python -m pip install --upgrade pip pip install -e '.[dev]' - # Tier 1 (#48): keyless demo smoke — no secret, no gate, cannot - # silently skip even if the repo secret disappears. - name: Keyless demo smoke (always runs) run: pytest tests/integration/test_demo_contract.py -m live --no-cov -v --timeout=60 - # Guard at the shell level so the job passes loudly (::warning::) instead - # of failing or silently skipping every test if the secret is empty. - - name: Run integration tests + - name: Require monitor credential run: | if [ -z "$OILPRICEAPI_KEY" ]; then - echo "::warning::OILPRICEAPI_TEST_KEY secret is empty/unset - live integration tests skipped" - exit 0 + echo "::error::OILPRICEAPI_TEST_KEY is empty or unset" + exit 1 fi - pytest tests/ -m 'integration' -v --no-cov --timeout=60 - - name: Run contract tests - run: | - if [ -z "$OILPRICEAPI_KEY" ]; then - echo "::warning::OILPRICEAPI_TEST_KEY secret is empty/unset - contract tests skipped" - exit 0 - fi - pytest tests/ -m 'contract' -v --no-cov --timeout=60 + - name: Run bounded customer-path synthetic + run: python scripts/synthetic_monitor.py --output artifacts/sdk-health.json - - name: Check for dependency vulnerabilities + - name: Publish receipt in job summary + if: always() && hashFiles('artifacts/sdk-health.json') != '' run: | - pip install pip-audit - pip-audit --strict - continue-on-error: true + echo '### Python SDK synthetic receipt' >> "$GITHUB_STEP_SUMMARY" + echo '```json' >> "$GITHUB_STEP_SUMMARY" + cat artifacts/sdk-health.json >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + + - name: Upload 30-day receipt + if: always() && hashFiles('artifacts/sdk-health.json') != '' + uses: actions/upload-artifact@v7 + with: + name: sdk-health-${{ github.run_id }} + path: artifacts/sdk-health.json + retention-days: 30 diff --git a/Dockerfile.monitor b/Dockerfile.monitor deleted file mode 100644 index 8dbddbf..0000000 --- a/Dockerfile.monitor +++ /dev/null @@ -1,22 +0,0 @@ -FROM python:3.11-slim - -WORKDIR /app - -# Install SDK and dependencies -COPY pyproject.toml ./ -COPY oilpriceapi/ ./oilpriceapi/ -RUN pip install --no-cache-dir prometheus_client && \ - pip install --no-cache-dir -e . - -# Copy monitoring script -COPY scripts/synthetic_monitor.py ./ - -# Expose Prometheus metrics port -EXPOSE 8000 - -# Health check -HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ - CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/metrics')" - -# Run monitoring -CMD ["python", "synthetic_monitor.py"] diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml deleted file mode 100644 index 6fe2751..0000000 --- a/docker-compose.monitoring.yml +++ /dev/null @@ -1,103 +0,0 @@ -version: '3.8' - -services: - # SDK Synthetic Monitor - sdk-monitor: - build: - context: . - dockerfile: Dockerfile.monitor - environment: - - OILPRICEAPI_KEY=${OILPRICEAPI_KEY} - - METRICS_PORT=8000 - - TEST_INTERVAL=900 # 15 minutes - ports: - - "8000:8000" - restart: unless-stopped - networks: - - monitoring - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/metrics"] - interval: 30s - timeout: 10s - retries: 3 - - # Prometheus (metrics storage) - prometheus: - image: prom/prometheus:v2.48.0 - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--storage.tsdb.retention.time=90d' - - '--web.console.libraries=/etc/prometheus/console_libraries' - - '--web.console.templates=/etc/prometheus/consoles' - volumes: - - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml - - ./monitoring/alert_rules.yml:/etc/prometheus/alert_rules.yml - - prometheus_data:/prometheus - ports: - - "9090:9090" - restart: unless-stopped - networks: - - monitoring - depends_on: - - sdk-monitor - - # Grafana (dashboards) - grafana: - image: grafana/grafana:10.2.2 - environment: - - GF_SECURITY_ADMIN_USER=admin - - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:-admin} - - GF_USERS_ALLOW_SIGN_UP=false - - GF_SERVER_ROOT_URL=http://localhost:3000 - volumes: - - grafana_data:/var/lib/grafana - - ./monitoring/grafana/provisioning:/etc/grafana/provisioning - - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards - ports: - - "3000:3000" - restart: unless-stopped - networks: - - monitoring - depends_on: - - prometheus - - # Alertmanager (alert routing) - alertmanager: - image: prom/alertmanager:v0.26.0 - command: - - '--config.file=/etc/alertmanager/alertmanager.yml' - - '--storage.path=/alertmanager' - volumes: - - ./monitoring/alertmanager.yml:/etc/alertmanager/alertmanager.yml - - alertmanager_data:/alertmanager - ports: - - "9093:9093" - restart: unless-stopped - networks: - - monitoring - depends_on: - - prometheus - - # PyPI Stats Exporter (optional) - pypi-exporter: - build: - context: . - dockerfile: Dockerfile.pypi-exporter - environment: - - PACKAGE_NAME=oilpriceapi - - METRICS_PORT=9101 - ports: - - "9101:9101" - restart: unless-stopped - networks: - - monitoring - -networks: - monitoring: - driver: bridge - -volumes: - prometheus_data: - grafana_data: - alertmanager_data: diff --git a/docs/SYNTHETIC_MONITORING.md b/docs/SYNTHETIC_MONITORING.md new file mode 100644 index 0000000..9bf4c25 --- /dev/null +++ b/docs/SYNTHETIC_MONITORING.md @@ -0,0 +1,41 @@ +# Python SDK synthetic monitoring + +The `Scheduled SDK Synthetic` workflow runs once an hour and validates the two +smallest customer-critical paths through the installed SDK: + +1. a latest `BRENT_CRUDE_USD` price with a finite numeric value, currency, unit, + and source timestamp; +2. a five-record daily history request with finite numeric values. + +A keyless demo-contract test runs first. The keyed check fails loudly when the +monitor credential is missing; it never silently skips. + +## Receipts and alerts + +Each run writes `sdk-health.json`, publishes it in the GitHub Actions job +summary, and retains it as an artifact for 30 days. Receipts contain the SDK +version, check names, durations, and structural assertions. They exclude the API +key, response values, request URLs, response bodies, and exception messages. + +GitHub Actions run history is the dashboard. A failed scheduled workflow is the +alert and uses the repository notification settings; the workflow deliberately +does not create recurring GitHub issues. + +## Response runbook + +1. Open the failed `Scheduled SDK Synthetic` run and read its JSON receipt. +2. If `configuration` failed, restore or rotate `OILPRICEAPI_TEST_KEY`, then run + the workflow manually. +3. If only the keyless demo check failed, verify public API availability and + response-envelope drift. +4. If `latest_price` or `bounded_history` failed, compare with the latest + `Live API Tests` run and reproduce with a non-customer test credential. +5. Treat repeated time-budget failures as a latency regression. Treat missing + fields, empty history, or non-finite values as a contract regression. +6. Record any product incident in the owning API repository. Keep SDK parsing, + retry, or compatibility fixes in this repository. + +The schedule is hourly rather than every five minutes. Two authenticated checks +per hour provide continuous SDK-contract coverage without spending thousands of +CI minutes or consuming unnecessary API quota. Push and pull-request live tests +provide additional coverage between scheduled runs. diff --git a/mkdocs.yml b/mkdocs.yml index c8df318..5afe1b3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -16,6 +16,7 @@ theme: nav: - Home: index.md - DataFrames and Pagination: DATAFRAMES.md + - Synthetic Monitoring: SYNTHETIC_MONITORING.md - Performance Guide: PERFORMANCE_GUIDE.md - API Reference: - Client: reference/client.md diff --git a/monitoring/README.md b/monitoring/README.md index 918655c..7f3f3f0 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -1,482 +1,11 @@ -# Synthetic Monitoring Deployment Guide +# Synthetic monitoring -Complete guide to deploying SDK synthetic monitoring that would have caught the v1.4.1 bug. +The supported monitor is the repository's hourly `Scheduled SDK Synthetic` +GitHub Actions workflow. ---- +See [the current design, receipts, and response runbook](../docs/SYNTHETIC_MONITORING.md). -## Quick Start - -```bash -# 1. Set API key -export OILPRICEAPI_KEY=your_key_here - -# 2. Start monitoring stack -docker-compose -f docker-compose.monitoring.yml up -d - -# 3. Access dashboards -open http://localhost:3000 # Grafana (admin/admin) -open http://localhost:9090 # Prometheus -open http://localhost:8000/metrics # Metrics endpoint -``` - -That's it! Monitoring is now running. - ---- - -## What Gets Monitored - -### SDK Health Checks (Every 15 Minutes) - -1. **1-day historical query** - - Expected: <10s - - Alert if: >10s - -2. **1-week historical query** ⭐ - - Expected: <30s - - Alert if: >30s - - **Would have caught v1.4.1 bug** (took 67s) - -3. **1-month historical query** - - Expected: <60s - - Alert if: >60s - -4. **1-year historical query** ⭐ - - Expected: <120s - - Alert if: >120s or timeout - - **Would have caught v1.4.1 bug** (timed out at 30s) - -### Metrics Collected - -- `sdk_historical_query_duration_seconds` - Query latency -- `sdk_historical_query_success_total` - Success count -- `sdk_historical_query_failure_total` - Failure count -- `sdk_endpoint_selection_correct` - Correct endpoint used -- `sdk_historical_records_returned` - Record count -- `sdk_monitor_last_test_timestamp` - Last test run time - ---- - -## Architecture - -``` -┌─────────────────┐ -│ SDK Monitor │ Runs tests every 15min -│ (Docker) │ Exposes Prometheus metrics -└────────┬────────┘ - │ - v -┌─────────────────┐ -│ Prometheus │ Scrapes metrics every 60s -│ (Docker) │ Stores time-series data -└────────┬────────┘ - │ - ├──────> Grafana (Dashboards) - │ - └──────> Alertmanager (PagerDuty/Slack) -``` - ---- - -## Configuration - -### Environment Variables - -Create `.env` file: - -```bash -# Required -OILPRICEAPI_KEY=your_api_key_here - -# Optional -GRAFANA_PASSWORD=secure_password -TEST_INTERVAL=900 # 15 minutes (default) -METRICS_PORT=8000 # Metrics port (default) -``` - -### Prometheus Configuration - -`monitoring/prometheus.yml`: - -```yaml -global: - scrape_interval: 60s - evaluation_interval: 60s - -scrape_configs: - - job_name: 'sdk_monitor' - static_configs: - - targets: ['sdk-monitor:8000'] - - - job_name: 'pypi_stats' - scrape_interval: 3600s - static_configs: - - targets: ['pypi-exporter:9101'] - -rule_files: - - 'alert_rules.yml' - -alerting: - alertmanagers: - - static_configs: - - targets: ['alertmanager:9093'] -``` - -### Alert Rules - -`monitoring/alert_rules.yml`: - -```yaml -groups: - - name: sdk_health - interval: 60s - rules: - # Would have caught v1.4.1 bug - - alert: HistoricalQuery1WeekSlow - expr: sdk_historical_query_duration_seconds{query_type="1_week"} > 30 - for: 5m - labels: - severity: critical - annotations: - summary: "1-week queries >30s (v1.4.1 bug pattern)" - description: "Duration: {{ $value }}s (expected <30s)" - - - alert: HistoricalQuery1YearTimeout - expr: sdk_historical_query_duration_seconds{query_type="1_year"} > 120 - for: 5m - labels: - severity: critical - annotations: - summary: "1-year queries timing out" - description: "Duration: {{ $value }}s (v1.4.1 bug pattern)" - - - alert: SDKMonitorDown - expr: up{job="sdk_monitor"} == 0 - for: 5m - labels: - severity: warning - annotations: - summary: "SDK monitor is down" -``` - -### Alertmanager Configuration - -`monitoring/alertmanager.yml`: - -```yaml -global: - resolve_timeout: 5m - -route: - receiver: 'default' - group_by: ['alertname', 'severity'] - group_wait: 30s - group_interval: 5m - repeat_interval: 12h - - routes: - - match: - severity: critical - receiver: 'pagerduty' - - - match: - severity: warning - receiver: 'slack' - -receivers: - - name: 'default' - email_configs: - - to: 'team@oilpriceapi.com' - - - name: 'pagerduty' - pagerduty_configs: - - service_key: 'YOUR_PAGERDUTY_KEY' - description: '{{ .CommonAnnotations.summary }}' - - - name: 'slack' - slack_configs: - - api_url: 'YOUR_SLACK_WEBHOOK_URL' - channel: '#sdk-alerts' - text: '{{ .CommonAnnotations.description }}' -``` - ---- - -## Deployment - -### Production Deployment - -```bash -# 1. Clone repository -git clone https://github.com/OilpriceAPI/python-sdk -cd python-sdk - -# 2. Create .env file -cat > .env << EOF -OILPRICEAPI_KEY=your_production_key -GRAFANA_PASSWORD=secure_password -EOF - -# 3. Create monitoring directory -mkdir -p monitoring/grafana/{provisioning,dashboards} - -# 4. Copy configuration files -cp monitoring/prometheus.yml.example monitoring/prometheus.yml -cp monitoring/alert_rules.yml.example monitoring/alert_rules.yml -cp monitoring/alertmanager.yml.example monitoring/alertmanager.yml - -# 5. Configure alerts (edit files above) -vim monitoring/alertmanager.yml # Add PagerDuty/Slack keys - -# 6. Start monitoring -docker-compose -f docker-compose.monitoring.yml up -d - -# 7. Verify all services running -docker-compose -f docker-compose.monitoring.yml ps -``` - -### Verify Deployment - -```bash -# Check monitor is running -curl http://localhost:8000/metrics | grep sdk_ - -# Check Prometheus scraping -curl http://localhost:9090/api/v1/query?query=up - -# Check Grafana -open http://localhost:3000 # Login: admin/admin (or your password) -``` - ---- - -## Grafana Dashboards - -### Import SDK Health Dashboard - -1. Go to http://localhost:3000 -2. Login (admin/admin or your password) -3. Click "+" → "Import" -4. Upload `monitoring/grafana/dashboards/sdk-health.json` - -### Dashboard Panels - -**Row 1: Query Performance** -- 1-Week Query Duration (should be <30s) -- 1-Month Query Duration (should be <60s) -- 1-Year Query Duration (should be <120s) -- Query Success Rate (should be >99%) - -**Row 2: Error Tracking** -- Error Rate by Type -- Timeout Errors (should be 0) -- Failed Queries Over Time - -**Row 3: Endpoint Selection** -- Endpoint Correctness (should be 1.0) -- Records Returned by Query Type -- Last Test Timestamp - ---- - -## Troubleshooting - -### Monitor Container Won't Start - -```bash -# Check logs -docker-compose -f docker-compose.monitoring.yml logs sdk-monitor - -# Common issues: -# 1. No API key set -export OILPRICEAPI_KEY=your_key - -# 2. Port already in use -# Change port in docker-compose.monitoring.yml - -# 3. Package installation failed -docker-compose -f docker-compose.monitoring.yml build --no-cache sdk-monitor -``` - -### No Metrics in Prometheus - -```bash -# Check monitor is exposing metrics -curl http://localhost:8000/metrics - -# Check Prometheus can reach monitor -docker-compose -f docker-compose.monitoring.yml exec prometheus \ - wget -O- http://sdk-monitor:8000/metrics - -# Check Prometheus targets -open http://localhost:9090/targets -``` - -### Alerts Not Firing - -```bash -# Check alert rules are loaded -open http://localhost:9090/alerts - -# Check Alertmanager config -docker-compose -f docker-compose.monitoring.yml exec alertmanager \ - amtool check-config /etc/alertmanager/alertmanager.yml - -# Test alert routing -docker-compose -f docker-compose.monitoring.yml exec alertmanager \ - amtool alert add test severity=critical -``` - ---- - -## Maintenance - -### Daily - -- Check Grafana dashboard for anomalies -- Verify latest test timestamp is recent -- Review any fired alerts - -### Weekly - -- Review query duration trends -- Check for performance regressions -- Update alert thresholds if needed - -### Monthly - -- Review and cleanup old metrics (auto-retention: 90 days) -- Update SDK monitor to latest version -- Review alert fatigue (too many false positives?) - ---- - -## Cost Estimates - -### Self-Hosted (Docker Compose) - -**Requirements:** -- VM: 2 CPU, 4GB RAM, 20GB disk -- Providers: DigitalOcean, AWS, GCP - -**Costs:** -- DigitalOcean Droplet ($24/month) -- AWS t3.medium (~$30/month) -- GCP e2-medium (~$25/month) - -**Total: ~$25/month** - -### Managed Services - -**Grafana Cloud:** -- Free tier: 10k series, 14-day retention -- Pro: $8/month for 100k series - -**Datadog:** -- ~$15/host/month -- Includes everything - -**New Relic:** -- ~$25/month -- Includes synthetics - ---- - -## Scaling - -### For Multiple Commodities - -Edit `scripts/synthetic_monitor.py`: - -```python -# Test multiple commodities -COMMODITIES = ["WTI_USD", "BRENT_CRUDE_USD", "NATURAL_GAS_USD"] - -for commodity in COMMODITIES: - test_1_week_query(client, commodity) - test_1_year_query(client, commodity) -``` - -### For Multiple Intervals - -```python -# Test different intervals -INTERVALS = ["hourly", "daily", "weekly"] - -for interval in INTERVALS: - test_query_with_interval(client, interval) -``` - -### For Different Regions - -Run multiple monitors in different regions: - -```yaml -# docker-compose.monitoring.yml -services: - sdk-monitor-us: - build: . - environment: - - OILPRICEAPI_KEY=${US_API_KEY} - - REGION=us-east-1 - - sdk-monitor-eu: - build: . - environment: - - OILPRICEAPI_KEY=${EU_API_KEY} - - REGION=eu-west-1 -``` - ---- - -## Success Metrics - -### Monitoring is Working If: - -✅ All services show "up" in Prometheus targets -✅ Latest test timestamp is within last 20 minutes -✅ Query durations within expected ranges -✅ Zero timeout errors -✅ Endpoint selection correctness = 1.0 - -### Would Have Detected v1.4.1 If: - -✅ 1-week query duration alert fires (>30s) -✅ 1-year query timeout alert fires -✅ Endpoint selection correctness drops to 0 -✅ PagerDuty/Slack notification sent -✅ Team investigates within 1 hour - -**Result: Bug caught in <1 hour instead of 8 hours** - ---- - -## Related - -- [Monitoring Guide](../.github/MONITORING_GUIDE.md) - Architecture details -- [Synthetic Monitor Script](../scripts/synthetic_monitor.py) - Monitor source code -- [GitHub Issue #28](https://github.com/OilpriceAPI/python-sdk/issues/28) - ---- - -## Quick Commands - -```bash -# Start monitoring -docker-compose -f docker-compose.monitoring.yml up -d - -# Stop monitoring -docker-compose -f docker-compose.monitoring.yml down - -# View logs -docker-compose -f docker-compose.monitoring.yml logs -f sdk-monitor - -# Restart monitor -docker-compose -f docker-compose.monitoring.yml restart sdk-monitor - -# Check metrics -curl http://localhost:8000/metrics | grep sdk_ - -# Test alert -docker-compose -f docker-compose.monitoring.yml exec alertmanager \ - amtool alert add test severity=critical summary="Test alert" -``` +The old local Prometheus/Grafana compose example was removed because its +referenced configuration and dashboard files did not exist, it exposed a +long-running process that printed part of the API key, and it was never the +deployed monitoring path. diff --git a/scripts/synthetic_monitor.py b/scripts/synthetic_monitor.py index 4495c9a..f8701f6 100644 --- a/scripts/synthetic_monitor.py +++ b/scripts/synthetic_monitor.py @@ -1,376 +1,202 @@ -""" -Synthetic monitoring for OilPriceAPI SDK. - -Runs continuous health checks and reports metrics to Prometheus. -This would have caught the v1.4.1 historical timeout bug. - -Usage: - export OILPRICEAPI_KEY=your_key - python scripts/synthetic_monitor.py +#!/usr/bin/env python3 +"""Run a bounded, privacy-safe production smoke through the Python SDK. - # With custom settings - export METRICS_PORT=8000 - export TEST_INTERVAL=900 # 15 minutes - python scripts/synthetic_monitor.py +The script is intentionally one-shot so a scheduler owns cadence, retries, and +alerting. It emits a machine-readable receipt and never prints credentials, +response bodies, request URLs, or exception messages. """ +from __future__ import annotations + +import argparse +import json +import math import os -import sys import time -from datetime import datetime, timedelta - -try: - from prometheus_client import start_http_server, Gauge, Counter, Histogram - from oilpriceapi import OilPriceAPI -except ImportError: - print("ERROR: Required dependencies not installed") - print("Run: pip install oilpriceapi prometheus_client") - sys.exit(1) - - -# Prometheus Metrics -QUERY_DURATION = Histogram( - 'sdk_historical_query_duration_seconds', - 'Historical query duration in seconds', - ['query_type', 'commodity'], - buckets=[1, 5, 10, 30, 60, 120, 300] -) - -QUERY_SUCCESS = Counter( - 'sdk_historical_query_success_total', - 'Total successful historical queries', - ['query_type', 'commodity'] -) - -QUERY_FAILURE = Counter( - 'sdk_historical_query_failure_total', - 'Total failed historical queries', - ['query_type', 'commodity', 'error_type'] -) - -ENDPOINT_CORRECTNESS = Gauge( - 'sdk_endpoint_selection_correct', - 'Whether SDK selected correct endpoint (1=correct, 0=wrong)', - ['query_type'] -) - -RECORD_COUNT = Gauge( - 'sdk_historical_records_returned', - 'Number of records returned by historical query', - ['query_type', 'commodity'] -) - -LAST_TEST_TIMESTAMP = Gauge( - 'sdk_monitor_last_test_timestamp', - 'Unix timestamp of last test run' -) - - -def test_1_day_query(client, commodity="WTI_USD"): - """Test 1-day historical query.""" - query_type = "1_day" - print(f" Testing {query_type} query for {commodity}...") - - start_time = time.time() - try: - end_date = datetime.now() - start_date = end_date - timedelta(days=1) - - history = client.historical.get( - commodity=commodity, - start_date=start_date.strftime("%Y-%m-%d"), - end_date=end_date.strftime("%Y-%m-%d"), - interval="hourly" - ) - - duration = time.time() - start_time - record_count = len(history.data) - - QUERY_DURATION.labels(query_type=query_type, commodity=commodity).observe(duration) - QUERY_SUCCESS.labels(query_type=query_type, commodity=commodity).inc() - RECORD_COUNT.labels(query_type=query_type, commodity=commodity).set(record_count) - - # Expected: <10s for 1-day query - is_correct = duration < 10 - ENDPOINT_CORRECTNESS.labels(query_type=query_type).set(1 if is_correct else 0) - - status = "✓" if is_correct else "✗" - print(f" {status} Completed in {duration:.2f}s ({record_count} records)") - - if not is_correct: - print(f" WARNING: Expected <10s, got {duration:.2f}s") - - return is_correct - - except Exception as e: - duration = time.time() - start_time - error_type = type(e).__name__ - - QUERY_FAILURE.labels(query_type=query_type, commodity=commodity, error_type=error_type).inc() - ENDPOINT_CORRECTNESS.labels(query_type=query_type).set(0) - - print(f" ✗ Failed after {duration:.2f}s: {error_type}: {str(e)[:100]}") - return False - - -def test_1_week_query(client, commodity="WTI_USD"): - """ - Test 1-week historical query. - - This test would have caught the v1.4.1 bug: - - Bug: Took 67s (using wrong endpoint) - - Expected: <30s (using /v1/prices/past_week) - """ - query_type = "1_week" - print(f" Testing {query_type} query for {commodity}...") - - start_time = time.time() - try: - end_date = datetime.now() - start_date = end_date - timedelta(days=7) - - history = client.historical.get( - commodity=commodity, - start_date=start_date.strftime("%Y-%m-%d"), - end_date=end_date.strftime("%Y-%m-%d"), - interval="daily" - ) - - duration = time.time() - start_time - record_count = len(history.data) - - QUERY_DURATION.labels(query_type=query_type, commodity=commodity).observe(duration) - QUERY_SUCCESS.labels(query_type=query_type, commodity=commodity).inc() - RECORD_COUNT.labels(query_type=query_type, commodity=commodity).set(record_count) - - # Expected: <30s for 1-week query (would catch v1.4.1 bug) - is_correct = duration < 30 - ENDPOINT_CORRECTNESS.labels(query_type=query_type).set(1 if is_correct else 0) - - status = "✓" if is_correct else "✗" - print(f" {status} Completed in {duration:.2f}s ({record_count} records)") - - if not is_correct: - print(f" 🚨 CRITICAL: Expected <30s, got {duration:.2f}s") - print(f" This matches the v1.4.1 bug pattern!") - - return is_correct - - except Exception as e: - duration = time.time() - start_time - error_type = type(e).__name__ - - QUERY_FAILURE.labels(query_type=query_type, commodity=commodity, error_type=error_type).inc() - ENDPOINT_CORRECTNESS.labels(query_type=query_type).set(0) +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Dict, Optional + +from oilpriceapi import OilPriceAPI +from oilpriceapi.version import SDK_VERSION + +Receipt = Dict[str, Any] +Check = Callable[[OilPriceAPI], Dict[str, Any]] + + +def _latest_price_check(client: OilPriceAPI) -> Dict[str, Any]: + price = client.prices.get("BRENT_CRUDE_USD") + if ( + isinstance(price.value, bool) + or not isinstance(price.value, (int, float)) + or not math.isfinite(price.value) + ): + raise ValueError("latest price is not finite") + if not isinstance(price.currency, str) or not price.currency: + raise ValueError("latest price currency is missing") + if not isinstance(price.unit, str) or not price.unit: + raise ValueError("latest price unit is missing") + if price.timestamp is None: + raise ValueError("latest price timestamp is missing") + return { + "commodity": price.commodity, + "currency_present": True, + "numeric_value": True, + "source_timestamp_present": True, + "unit_present": True, + } - print(f" ✗ Failed after {duration:.2f}s: {error_type}: {str(e)[:100]}") - return False +def _historical_check(client: OilPriceAPI) -> Dict[str, Any]: + history = client.historical.get( + commodity="BRENT_CRUDE_USD", + interval="daily", + per_page=5, + ) + if not history.data: + raise ValueError("historical response is empty") + if any( + isinstance(record.value, bool) + or not isinstance(record.value, (int, float)) + or not math.isfinite(record.value) + for record in history.data + ): + raise ValueError("historical response contains a non-finite value") + return { + "commodity": history.data[0].commodity, + "nonempty": True, + "records_checked": len(history.data), + } -def test_1_month_query(client, commodity="WTI_USD"): - """Test 1-month historical query.""" - query_type = "1_month" - print(f" Testing {query_type} query for {commodity}...") - start_time = time.time() +def _run_check( + name: str, + check: Check, + client: OilPriceAPI, + *, + monotonic: Callable[[], float], + budget_seconds: float, +) -> Receipt: + started = monotonic() try: - end_date = datetime.now() - start_date = end_date - timedelta(days=30) - - history = client.historical.get( - commodity=commodity, - start_date=start_date.strftime("%Y-%m-%d"), - end_date=end_date.strftime("%Y-%m-%d"), - interval="daily" - ) - - duration = time.time() - start_time - record_count = len(history.data) - - QUERY_DURATION.labels(query_type=query_type, commodity=commodity).observe(duration) - QUERY_SUCCESS.labels(query_type=query_type, commodity=commodity).inc() - RECORD_COUNT.labels(query_type=query_type, commodity=commodity).set(record_count) - - # Expected: <60s for 1-month query - is_correct = duration < 60 - ENDPOINT_CORRECTNESS.labels(query_type=query_type).set(1 if is_correct else 0) - - status = "✓" if is_correct else "✗" - print(f" {status} Completed in {duration:.2f}s ({record_count} records)") - - if not is_correct: - print(f" WARNING: Expected <60s, got {duration:.2f}s") - - return is_correct - - except Exception as e: - duration = time.time() - start_time - error_type = type(e).__name__ - - QUERY_FAILURE.labels(query_type=query_type, commodity=commodity, error_type=error_type).inc() - ENDPOINT_CORRECTNESS.labels(query_type=query_type).set(0) - - print(f" ✗ Failed after {duration:.2f}s: {error_type}: {str(e)[:100]}") - return False - - -def test_1_year_query(client, commodity="WTI_USD"): - """ - Test 1-year historical query. - - This test would have caught the v1.4.1 bug: - - Bug: Timed out at 30s - - Expected: Complete in <120s - """ - query_type = "1_year" - print(f" Testing {query_type} query for {commodity}...") - - start_time = time.time() + details = check(client) + duration = monotonic() - started + if duration > budget_seconds: + return { + "name": name, + "status": "fail", + "duration_seconds": round(duration, 3), + "error_type": "TimeBudgetExceeded", + } + return { + "name": name, + "status": "pass", + "duration_seconds": round(duration, 3), + "details": details, + } + except Exception as exc: # noqa: BLE001 - capture SDK/network failures safely + return { + "name": name, + "status": "fail", + "duration_seconds": round(monotonic() - started, 3), + "error_type": type(exc).__name__, + } + + +def run_synthetic_checks( + api_key: str, + *, + base_url: Optional[str] = None, + client_factory: Callable[..., OilPriceAPI] = OilPriceAPI, + monotonic: Callable[[], float] = time.monotonic, +) -> Receipt: + """Run the latest-price and bounded-history checks once.""" + checked_at = datetime.now(timezone.utc).isoformat() try: - # Use fixed date range for consistency - history = client.historical.get( - commodity=commodity, - start_date="2024-01-01", - end_date="2024-12-31", - interval="daily" - ) - - duration = time.time() - start_time - record_count = len(history.data) - - QUERY_DURATION.labels(query_type=query_type, commodity=commodity).observe(duration) - QUERY_SUCCESS.labels(query_type=query_type, commodity=commodity).inc() - RECORD_COUNT.labels(query_type=query_type, commodity=commodity).set(record_count) - - # Expected: <120s for 1-year query (would catch v1.4.1 timeout bug) - is_correct = duration < 120 - ENDPOINT_CORRECTNESS.labels(query_type=query_type).set(1 if is_correct else 0) - - status = "✓" if is_correct else "✗" - print(f" {status} Completed in {duration:.2f}s ({record_count} records)") - - if not is_correct: - print(f" 🚨 CRITICAL: Expected <120s, got {duration:.2f}s") - print(f" This matches the v1.4.1 timeout bug!") - - return is_correct - - except Exception as e: - duration = time.time() - start_time - error_type = type(e).__name__ - - QUERY_FAILURE.labels(query_type=query_type, commodity=commodity, error_type=error_type).inc() - ENDPOINT_CORRECTNESS.labels(query_type=query_type).set(0) - - print(f" ✗ Failed after {duration:.2f}s: {error_type}: {str(e)[:100]}") - - if "timeout" in str(e).lower() or error_type == "TimeoutError": - print(f" 🚨 CRITICAL: TIMEOUT DETECTED - THIS IS THE v1.4.1 BUG!") - - return False - - -def run_synthetic_tests(client): - """Run all synthetic tests.""" - print(f"\n[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Running synthetic tests...") - - results = { - "1_day": test_1_day_query(client), - "1_week": test_1_week_query(client), - "1_month": test_1_month_query(client), - "1_year": test_1_year_query(client), + with client_factory( + api_key=api_key, + base_url=base_url, + timeout=20, + max_retries=1, + ) as client: + checks = [ + _run_check( + "latest_price", + _latest_price_check, + client, + monotonic=monotonic, + budget_seconds=30, + ), + _run_check( + "bounded_history", + _historical_check, + client, + monotonic=monotonic, + budget_seconds=30, + ), + ] + except Exception as exc: # noqa: BLE001 - sanitize initialization failures + checks = [ + { + "name": "client_initialization", + "status": "fail", + "duration_seconds": 0.0, + "error_type": type(exc).__name__, + } + ] + + status = "pass" if all(check["status"] == "pass" for check in checks) else "fail" + return { + "schema_version": 1, + "status": status, + "checked_at": checked_at, + "sdk_version": SDK_VERSION, + "checks": checks, } - LAST_TEST_TIMESTAMP.set(time.time()) - # Summary - passed = sum(1 for v in results.values() if v) - total = len(results) +def _write_receipt(receipt: Receipt, output: Optional[str]) -> None: + rendered = json.dumps(receipt, indent=2, sort_keys=True) + "\n" + if output: + destination = Path(output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(rendered, encoding="utf-8") + print(rendered, end="") - print(f"\n Summary: {passed}/{total} tests passed") - if passed < total: - print(f" ⚠️ {total - passed} test(s) failed - check logs above") +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output", + help="Optional path for the JSON receipt; the receipt is always printed.", + ) + args = parser.parse_args() - return passed == total - - -def main(): - """Main monitoring loop.""" - print("="*60) - print("OilPriceAPI SDK Synthetic Monitor") - print("="*60) - print("") - print("This monitor would have caught the v1.4.1 timeout bug.") - print("") - - # Get configuration from environment - api_key = os.getenv('OILPRICEAPI_KEY') + api_key = os.environ.get("OILPRICEAPI_KEY") if not api_key: - print("ERROR: OILPRICEAPI_KEY environment variable not set") - print("") - print("Set it with:") - print(" export OILPRICEAPI_KEY=your_key") - return 1 - - metrics_port = int(os.getenv('METRICS_PORT', '8000')) - test_interval = int(os.getenv('TEST_INTERVAL', '900')) # 15 minutes default - - print(f"Configuration:") - print(f" API Key: {api_key[:8]}...{api_key[-4:]}") - print(f" Metrics Port: {metrics_port}") - print(f" Test Interval: {test_interval}s ({test_interval/60:.0f} minutes)") - print("") - - # Start Prometheus metrics server - try: - start_http_server(metrics_port) - print(f"✓ Metrics server started on http://0.0.0.0:{metrics_port}/metrics") - except OSError as e: - print(f"ERROR: Failed to start metrics server on port {metrics_port}: {e}") - print(f"Try a different port: METRICS_PORT=8001 python {sys.argv[0]}") - return 1 - - # Create SDK client - try: - client = OilPriceAPI(api_key=api_key) - print(f"✓ SDK client initialized") - except Exception as e: - print(f"ERROR: Failed to initialize SDK client: {e}") - return 1 - - print("") - print("Starting monitoring loop (Ctrl+C to stop)...") - print("") - - # Run tests immediately on start - try: - run_synthetic_tests(client) - except Exception as e: - print(f"ERROR in initial test run: {e}") - - # Monitoring loop - test_count = 1 - while True: - try: - time.sleep(test_interval) - test_count += 1 - print(f"\n{'='*60}") - print(f"Test run #{test_count}") - print(f"{'='*60}") - run_synthetic_tests(client) - - except KeyboardInterrupt: - print("\n\nMonitoring stopped by user (Ctrl+C)") - break - - except Exception as e: - print(f"\nERROR in monitoring loop: {e}") - print("Continuing monitoring...") - time.sleep(60) # Wait a minute before retrying + receipt: Receipt = { + "schema_version": 1, + "status": "fail", + "checked_at": datetime.now(timezone.utc).isoformat(), + "sdk_version": SDK_VERSION, + "checks": [ + { + "name": "configuration", + "status": "fail", + "duration_seconds": 0.0, + "error_type": "MissingAPIKey", + } + ], + } + else: + receipt = run_synthetic_checks( + api_key, + base_url=os.environ.get("OILPRICEAPI_BASE_URL"), + ) - return 0 + _write_receipt(receipt, args.output) + return 0 if receipt["status"] == "pass" else 1 -if __name__ == '__main__': - sys.exit(main()) +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/test_synthetic_monitor.py b/tests/unit/test_synthetic_monitor.py new file mode 100644 index 0000000..02ac697 --- /dev/null +++ b/tests/unit/test_synthetic_monitor.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import importlib.util +import json +from datetime import datetime, timezone +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import Any, List + + +def _load_monitor() -> ModuleType: + path = Path(__file__).parents[2] / "scripts" / "synthetic_monitor.py" + spec = importlib.util.spec_from_file_location("synthetic_monitor", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +monitor = _load_monitor() + + +class FakeClient: + def __init__(self, **_: Any) -> None: + record = SimpleNamespace( + commodity="BRENT_CRUDE_USD", + value=96.25, + currency="USD", + unit="barrel", + timestamp=datetime(2026, 7, 25, tzinfo=timezone.utc), + ) + self.prices = SimpleNamespace(get=lambda _code: record) + self.historical = SimpleNamespace( + get=lambda **_kwargs: SimpleNamespace(data=[record, record]) + ) + + def __enter__(self) -> "FakeClient": + return self + + def __exit__(self, *_args: Any) -> None: + return None + + +def _clock(values: List[float]): + iterator = iter(values) + return lambda: next(iterator) + + +def test_receipt_passes_for_valid_latest_and_history() -> None: + receipt = monitor.run_synthetic_checks( + "not-a-real-key", + client_factory=FakeClient, + monotonic=_clock([1.0, 1.2, 2.0, 2.4]), + ) + + assert receipt["status"] == "pass" + assert [check["name"] for check in receipt["checks"]] == [ + "latest_price", + "bounded_history", + ] + assert all(check["status"] == "pass" for check in receipt["checks"]) + assert receipt["checks"][1]["details"]["records_checked"] == 2 + + +def test_failure_receipt_redacts_exception_message_and_key() -> None: + secret = "secret-value-that-must-not-escape" + + class FailingClient(FakeClient): + def __init__(self, **_: Any) -> None: + raise RuntimeError(f"upstream echoed {secret}") + + receipt = monitor.run_synthetic_checks(secret, client_factory=FailingClient) + rendered = json.dumps(receipt) + + assert receipt["status"] == "fail" + assert receipt["checks"][0]["error_type"] == "RuntimeError" + assert secret not in rendered + assert "upstream echoed" not in rendered + + +def test_empty_history_fails_without_response_content() -> None: + class EmptyHistoryClient(FakeClient): + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.historical = SimpleNamespace(get=lambda **_kwargs: SimpleNamespace(data=[])) + + receipt = monitor.run_synthetic_checks( + "not-a-real-key", + client_factory=EmptyHistoryClient, + monotonic=_clock([1.0, 1.1, 2.0, 2.2]), + ) + + assert receipt["status"] == "fail" + history = receipt["checks"][1] + assert history["name"] == "bounded_history" + assert history["error_type"] == "ValueError" + assert "details" not in history + + +def test_time_budget_is_a_failure() -> None: + receipt = monitor.run_synthetic_checks( + "not-a-real-key", + client_factory=FakeClient, + monotonic=_clock([1.0, 32.0, 40.0, 40.1]), + ) + + assert receipt["status"] == "fail" + assert receipt["checks"][0]["error_type"] == "TimeBudgetExceeded"