v2 Phase 1: ingestion + SQLite foundation

- Persistence via Node's built-in node:sqlite (zero native deps) on a Docker volume: response cache, source health, fixture↔provider id map, per-fixture enrichment, ingest log. Runtime bumped to node:24-slim + --experimental-sqlite. - Resilient fetcher: DB cache + per-source rate-limit + jittered backoff + circuit-breaker (blocked sources trip + skip; UI reads DB, never breaks). - ESPN hidden API as the PRIMARY rich source (works from the VPS where SofaScore 403s): scoreboard (live scores w/ clock) + summary (venue, H2H, recent form, lineups, team stats). football-data / SofaScore are fallbacks. - Scheduler: maps all 72 group fixtures to ESPN event ids, polls live on a dynamic cadence, enriches imminent fixtures into match_ext. /api/sources/health. - Verified: DB populates (real H2H + form for opening matches), 22 tests pass, Docker image runs node:sqlite on the volume and persists across restart. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-11 15:21:01 +02:00
parent 9c7605391c
commit b6f62679c9
13 changed files with 697 additions and 88 deletions
@@ -0,0 +1,187 @@
+import { DatabaseSync } from 'node:sqlite';
+import { mkdirSync } from 'node:fs';
+import path from 'node:path';
+
+// Persistence via Node's built-in SQLite (no native deps). One file on a Docker
+// volume holds the response cache, source health, fixture↔provider id mapping,
+// per-fixture enrichment, historical player goals, and ingest logs.
+
+const DATA_DIR = path.resolve(process.env.DATA_DIR ?? path.join(process.cwd(), 'data'));
+
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS kv_cache (
+  key        TEXT PRIMARY KEY,
+  value      TEXT NOT NULL,
+  fetched_at INTEGER NOT NULL,
+  ttl        INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS source_health (
+  source     TEXT PRIMARY KEY,
+  last_ok    INTEGER,
+  last_err   INTEGER,
+  consec_fail INTEGER NOT NULL DEFAULT 0,
+  open_until INTEGER NOT NULL DEFAULT 0,
+  note       TEXT
+);
+
+CREATE TABLE IF NOT EXISTS source_map (
+  fixture_num INTEGER NOT NULL,
+  source      TEXT NOT NULL,
+  event_id    TEXT NOT NULL,
+  PRIMARY KEY (fixture_num, source)
+);
+
+CREATE TABLE IF NOT EXISTS match_ext (
+  fixture_num INTEGER PRIMARY KEY,
+  json        TEXT NOT NULL,
+  updated_at  INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS injuries (
+  team       TEXT NOT NULL,
+  player     TEXT NOT NULL,
+  status     TEXT,
+  reason     TEXT,
+  return_date TEXT,
+  updated_at INTEGER NOT NULL,
+  PRIMARY KEY (team, player)
+);
+
+CREATE TABLE IF NOT EXISTS squad (
+  team        TEXT NOT NULL,
+  player      TEXT NOT NULL,
+  position    TEXT,
+  number      INTEGER,
+  market_value INTEGER,
+  age         INTEGER,
+  updated_at  INTEGER NOT NULL,
+  PRIMARY KEY (team, player)
+);
+
+CREATE TABLE IF NOT EXISTS goalscorers (
+  date    TEXT NOT NULL,
+  home    TEXT NOT NULL,
+  away    TEXT NOT NULL,
+  team    TEXT NOT NULL,
+  scorer  TEXT,
+  minute  INTEGER,
+  own_goal INTEGER NOT NULL DEFAULT 0,
+  penalty INTEGER NOT NULL DEFAULT 0
+);
+CREATE INDEX IF NOT EXISTS idx_goalscorers_scorer ON goalscorers(scorer);
+CREATE INDEX IF NOT EXISTS idx_goalscorers_team ON goalscorers(team);
+
+CREATE TABLE IF NOT EXISTS ingest_log (
+  id     INTEGER PRIMARY KEY AUTOINCREMENT,
+  source TEXT NOT NULL,
+  job    TEXT NOT NULL,
+  ok     INTEGER NOT NULL,
+  ms     INTEGER,
+  note   TEXT,
+  at     INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_ingest_log_at ON ingest_log(at);
+`;
+
+let _db: DatabaseSync | null = null;
+
+export function db(): DatabaseSync {
+  if (_db) return _db;
+  mkdirSync(DATA_DIR, { recursive: true });
+  const file = path.join(DATA_DIR, 'cup26.db');
+  const d = new DatabaseSync(file);
+  d.exec('PRAGMA journal_mode = WAL;');
+  d.exec('PRAGMA busy_timeout = 4000;');
+  d.exec(SCHEMA);
+  _db = d;
+  console.log(`[db] sqlite ready → ${file}`);
+  return d;
+}
+
+// ---- response cache ----
+export function cacheGet(key: string): { value: string; fresh: boolean } | null {
+  const row = db().prepare('SELECT value, fetched_at, ttl FROM kv_cache WHERE key = ?').get(key) as
+    | { value: string; fetched_at: number; ttl: number }
+    | undefined;
+  if (!row) return null;
+  return { value: row.value, fresh: Date.now() - row.fetched_at < row.ttl };
+}
+
+export function cacheSet(key: string, value: string, ttl: number): void {
+  db()
+    .prepare('INSERT OR REPLACE INTO kv_cache (key, value, fetched_at, ttl) VALUES (?, ?, ?, ?)')
+    .run(key, value, Date.now(), ttl);
+}
+
+// ---- source health / circuit breaker ----
+export interface SourceHealthRow {
+  source: string;
+  last_ok: number | null;
+  last_err: number | null;
+  consec_fail: number;
+  open_until: number;
+  note: string | null;
+}
+
+export function healthGet(source: string): SourceHealthRow | null {
+  return (db().prepare('SELECT * FROM source_health WHERE source = ?').get(source) as SourceHealthRow | undefined) ?? null;
+}
+
+export function healthAll(): SourceHealthRow[] {
+  return db().prepare('SELECT * FROM source_health ORDER BY source').all() as unknown as SourceHealthRow[];
+}
+
+export function healthRecord(source: string, ok: boolean, note?: string, openMs?: number): void {
+  const now = Date.now();
+  const cur = healthGet(source);
+  const consec = ok ? 0 : (cur?.consec_fail ?? 0) + 1;
+  const openUntil = ok ? 0 : openMs ? now + openMs : (cur?.open_until ?? 0);
+  db()
+    .prepare(
+      `INSERT INTO source_health (source, last_ok, last_err, consec_fail, open_until, note)
+       VALUES (@source, @last_ok, @last_err, @consec, @open_until, @note)
+       ON CONFLICT(source) DO UPDATE SET
+         last_ok=COALESCE(@last_ok, source_health.last_ok),
+         last_err=COALESCE(@last_err, source_health.last_err),
+         consec_fail=@consec, open_until=@open_until, note=@note`,
+    )
+    .run({
+      source,
+      last_ok: ok ? now : null,
+      last_err: ok ? null : now,
+      consec: consec,
+      open_until: openUntil,
+      note: note ?? null,
+    });
+}
+
+export function logIngest(source: string, job: string, ok: boolean, ms: number, note?: string): void {
+  db().prepare('INSERT INTO ingest_log (source, job, ok, ms, note, at) VALUES (?, ?, ?, ?, ?, ?)')
+    .run(source, job, ok ? 1 : 0, Math.round(ms), note ?? null, Date.now());
+}
+
+// ---- fixture ↔ provider event-id mapping ----
+export function setSourceMap(fixtureNum: number, source: string, eventId: string): void {
+  db().prepare('INSERT OR REPLACE INTO source_map (fixture_num, source, event_id) VALUES (?, ?, ?)')
+    .run(fixtureNum, source, eventId);
+}
+
+export function getSourceMap(fixtureNum: number, source: string): string | null {
+  const row = db().prepare('SELECT event_id FROM source_map WHERE fixture_num = ? AND source = ?').get(fixtureNum, source) as
+    | { event_id: string } | undefined;
+  return row?.event_id ?? null;
+}
+
+// ---- per-fixture enrichment blob ----
+export function setMatchExt(fixtureNum: number, json: unknown): void {
+  db().prepare('INSERT OR REPLACE INTO match_ext (fixture_num, json, updated_at) VALUES (?, ?, ?)')
+    .run(fixtureNum, JSON.stringify(json), Date.now());
+}
+
+export function getMatchExt<T = unknown>(fixtureNum: number): { data: T; updatedAt: number } | null {
+  const row = db().prepare('SELECT json, updated_at FROM match_ext WHERE fixture_num = ?').get(fixtureNum) as
+    | { json: string; updated_at: number } | undefined;
+  if (!row) return null;
+  return { data: JSON.parse(row.json) as T, updatedAt: row.updated_at };
+}