From b6f62679c9832b2e947d63afd4d6650a396f52b4 Mon Sep 17 00:00:00 2001 From: Nils Briggen Date: Thu, 11 Jun 2026 15:21:01 +0200 Subject: [PATCH] v2 Phase 1: ingestion + SQLite foundation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Persistence via Node's built-in node:sqlite (zero native deps) on a Docker volume: response cache, source health, fixture↔provider id map, per-fixture enrichment, ingest log. Runtime bumped to node:24-slim + --experimental-sqlite. - Resilient fetcher: DB cache + per-source rate-limit + jittered backoff + circuit-breaker (blocked sources trip + skip; UI reads DB, never breaks). - ESPN hidden API as the PRIMARY rich source (works from the VPS where SofaScore 403s): scoreboard (live scores w/ clock) + summary (venue, H2H, recent form, lineups, team stats). football-data / SofaScore are fallbacks. - Scheduler: maps all 72 group fixtures to ESPN event ids, polls live on a dynamic cadence, enriches imminent fixtures into match_ext. /api/sources/health. - Verified: DB populates (real H2H + form for opening matches), 22 tests pass, Docker image runs node:sqlite on the volume and persists across restart. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 4 + Dockerfile | 11 +- deploy/cup26.compose.yml | 9 +- server/src/db/db.ts | 187 +++++++++++++++++++++++++++++ server/src/index.ts | 11 +- server/src/ingest/espn.test.ts | 92 ++++++++++++++ server/src/ingest/espn.ts | 34 ++++++ server/src/ingest/espnNormalize.ts | 132 ++++++++++++++++++++ server/src/ingest/fetcher.ts | 88 ++++++++++++++ server/src/ingest/scheduler.ts | 128 ++++++++++++++++++++ server/src/liveLoop.ts | 77 ------------ server/src/tournament.ts | 10 ++ src/lib/types.ts | 2 +- 13 files changed, 697 insertions(+), 88 deletions(-) create mode 100644 server/src/db/db.ts create mode 100644 server/src/ingest/espn.test.ts create mode 100644 server/src/ingest/espn.ts create mode 100644 server/src/ingest/espnNormalize.ts create mode 100644 server/src/ingest/fetcher.ts create mode 100644 server/src/ingest/scheduler.ts delete mode 100644 server/src/liveLoop.ts diff --git a/.gitignore b/.gitignore index bef64fa..595f664 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ server/node_modules .DS_Store .env .env.local +# local SQLite (dev); prod DB lives on the Docker volume +/data/cup26.db +/data/cup26.db-wal +/data/cup26.db-shm # Generated build artifacts (reproduced by `npm run data:build`) public/data/fixtures.json public/data/ratings.json diff --git a/Dockerfile b/Dockerfile index 956de46..a4b4fce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,15 +8,16 @@ RUN bun run data:build # icons + fixtures.json + ratings.json → public/ RUN bun run build # tsc -b && vite build → /app/dist RUN bun run build:server # esbuild → /app/server/dist/index.js -# --- runtime stage: small node image, non-root, serves dist + /ws + /api --- -FROM node:20-slim AS runtime -ENV NODE_ENV=production STATIC_DIR=/app/dist PORT=8787 +# --- runtime stage: node 24 (built-in node:sqlite), non-root, serves dist + /ws + /api --- +FROM node:24-slim AS runtime +ENV NODE_ENV=production STATIC_DIR=/app/dist PORT=8787 DATA_DIR=/data WORKDIR /app/server COPY server/package.json ./ RUN npm install --omit=dev --no-audit --no-fund COPY --from=build /app/server/dist ./dist COPY --from=build /app/dist /app/dist -RUN chown -R node:node /app +# SQLite lives on a named volume; create it node-owned so a fresh mount is writable. +RUN mkdir -p /data && chown -R node:node /app /data USER node EXPOSE 8787 -CMD ["node", "dist/index.js"] +CMD ["node", "--experimental-sqlite", "dist/index.js"] diff --git a/deploy/cup26.compose.yml b/deploy/cup26.compose.yml index 67d400f..5c2e36d 100644 --- a/deploy/cup26.compose.yml +++ b/deploy/cup26.compose.yml @@ -14,10 +14,14 @@ services: - STATIC_DIR=/app/dist - NODE_ENV=production - ALLOWED_ORIGINS=https://cup26.briggen.dev + - DATA_DIR=/data # football-data.org free API token (https://www.football-data.org/client/register) - FOOTBALL_DATA_TOKEN=${FOOTBALL_DATA_TOKEN:-} - # opt-in: layer the unofficial real-time source on top of the free API + # opt-in unofficial sources (usually blocked from a datacenter IP; ESPN is primary) - ENABLE_SOFASCORE=${ENABLE_SOFASCORE:-false} + - ENABLE_FOTMOB=${ENABLE_FOTMOB:-false} + volumes: + - cup26-data:/data security_opt: - no-new-privileges:true mem_limit: 384m @@ -33,6 +37,9 @@ services: networks: - proxy +volumes: + cup26-data: + networks: proxy: external: true diff --git a/server/src/db/db.ts b/server/src/db/db.ts new file mode 100644 index 0000000..2f9d3c6 --- /dev/null +++ b/server/src/db/db.ts @@ -0,0 +1,187 @@ +import { DatabaseSync } from 'node:sqlite'; +import { mkdirSync } from 'node:fs'; +import path from 'node:path'; + +// Persistence via Node's built-in SQLite (no native deps). One file on a Docker +// volume holds the response cache, source health, fixture↔provider id mapping, +// per-fixture enrichment, historical player goals, and ingest logs. + +const DATA_DIR = path.resolve(process.env.DATA_DIR ?? path.join(process.cwd(), 'data')); + +const SCHEMA = ` +CREATE TABLE IF NOT EXISTS kv_cache ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + fetched_at INTEGER NOT NULL, + ttl INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS source_health ( + source TEXT PRIMARY KEY, + last_ok INTEGER, + last_err INTEGER, + consec_fail INTEGER NOT NULL DEFAULT 0, + open_until INTEGER NOT NULL DEFAULT 0, + note TEXT +); + +CREATE TABLE IF NOT EXISTS source_map ( + fixture_num INTEGER NOT NULL, + source TEXT NOT NULL, + event_id TEXT NOT NULL, + PRIMARY KEY (fixture_num, source) +); + +CREATE TABLE IF NOT EXISTS match_ext ( + fixture_num INTEGER PRIMARY KEY, + json TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS injuries ( + team TEXT NOT NULL, + player TEXT NOT NULL, + status TEXT, + reason TEXT, + return_date TEXT, + updated_at INTEGER NOT NULL, + PRIMARY KEY (team, player) +); + +CREATE TABLE IF NOT EXISTS squad ( + team TEXT NOT NULL, + player TEXT NOT NULL, + position TEXT, + number INTEGER, + market_value INTEGER, + age INTEGER, + updated_at INTEGER NOT NULL, + PRIMARY KEY (team, player) +); + +CREATE TABLE IF NOT EXISTS goalscorers ( + date TEXT NOT NULL, + home TEXT NOT NULL, + away TEXT NOT NULL, + team TEXT NOT NULL, + scorer TEXT, + minute INTEGER, + own_goal INTEGER NOT NULL DEFAULT 0, + penalty INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX IF NOT EXISTS idx_goalscorers_scorer ON goalscorers(scorer); +CREATE INDEX IF NOT EXISTS idx_goalscorers_team ON goalscorers(team); + +CREATE TABLE IF NOT EXISTS ingest_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL, + job TEXT NOT NULL, + ok INTEGER NOT NULL, + ms INTEGER, + note TEXT, + at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_ingest_log_at ON ingest_log(at); +`; + +let _db: DatabaseSync | null = null; + +export function db(): DatabaseSync { + if (_db) return _db; + mkdirSync(DATA_DIR, { recursive: true }); + const file = path.join(DATA_DIR, 'cup26.db'); + const d = new DatabaseSync(file); + d.exec('PRAGMA journal_mode = WAL;'); + d.exec('PRAGMA busy_timeout = 4000;'); + d.exec(SCHEMA); + _db = d; + console.log(`[db] sqlite ready → ${file}`); + return d; +} + +// ---- response cache ---- +export function cacheGet(key: string): { value: string; fresh: boolean } | null { + const row = db().prepare('SELECT value, fetched_at, ttl FROM kv_cache WHERE key = ?').get(key) as + | { value: string; fetched_at: number; ttl: number } + | undefined; + if (!row) return null; + return { value: row.value, fresh: Date.now() - row.fetched_at < row.ttl }; +} + +export function cacheSet(key: string, value: string, ttl: number): void { + db() + .prepare('INSERT OR REPLACE INTO kv_cache (key, value, fetched_at, ttl) VALUES (?, ?, ?, ?)') + .run(key, value, Date.now(), ttl); +} + +// ---- source health / circuit breaker ---- +export interface SourceHealthRow { + source: string; + last_ok: number | null; + last_err: number | null; + consec_fail: number; + open_until: number; + note: string | null; +} + +export function healthGet(source: string): SourceHealthRow | null { + return (db().prepare('SELECT * FROM source_health WHERE source = ?').get(source) as SourceHealthRow | undefined) ?? null; +} + +export function healthAll(): SourceHealthRow[] { + return db().prepare('SELECT * FROM source_health ORDER BY source').all() as unknown as SourceHealthRow[]; +} + +export function healthRecord(source: string, ok: boolean, note?: string, openMs?: number): void { + const now = Date.now(); + const cur = healthGet(source); + const consec = ok ? 0 : (cur?.consec_fail ?? 0) + 1; + const openUntil = ok ? 0 : openMs ? now + openMs : (cur?.open_until ?? 0); + db() + .prepare( + `INSERT INTO source_health (source, last_ok, last_err, consec_fail, open_until, note) + VALUES (@source, @last_ok, @last_err, @consec, @open_until, @note) + ON CONFLICT(source) DO UPDATE SET + last_ok=COALESCE(@last_ok, source_health.last_ok), + last_err=COALESCE(@last_err, source_health.last_err), + consec_fail=@consec, open_until=@open_until, note=@note`, + ) + .run({ + source, + last_ok: ok ? now : null, + last_err: ok ? null : now, + consec: consec, + open_until: openUntil, + note: note ?? null, + }); +} + +export function logIngest(source: string, job: string, ok: boolean, ms: number, note?: string): void { + db().prepare('INSERT INTO ingest_log (source, job, ok, ms, note, at) VALUES (?, ?, ?, ?, ?, ?)') + .run(source, job, ok ? 1 : 0, Math.round(ms), note ?? null, Date.now()); +} + +// ---- fixture ↔ provider event-id mapping ---- +export function setSourceMap(fixtureNum: number, source: string, eventId: string): void { + db().prepare('INSERT OR REPLACE INTO source_map (fixture_num, source, event_id) VALUES (?, ?, ?)') + .run(fixtureNum, source, eventId); +} + +export function getSourceMap(fixtureNum: number, source: string): string | null { + const row = db().prepare('SELECT event_id FROM source_map WHERE fixture_num = ? AND source = ?').get(fixtureNum, source) as + | { event_id: string } | undefined; + return row?.event_id ?? null; +} + +// ---- per-fixture enrichment blob ---- +export function setMatchExt(fixtureNum: number, json: unknown): void { + db().prepare('INSERT OR REPLACE INTO match_ext (fixture_num, json, updated_at) VALUES (?, ?, ?)') + .run(fixtureNum, JSON.stringify(json), Date.now()); +} + +export function getMatchExt(fixtureNum: number): { data: T; updatedAt: number } | null { + const row = db().prepare('SELECT json, updated_at FROM match_ext WHERE fixture_num = ?').get(fixtureNum) as + | { json: string; updated_at: number } | undefined; + if (!row) return null; + return { data: JSON.parse(row.json) as T, updatedAt: row.updated_at }; +} diff --git a/server/src/index.ts b/server/src/index.ts index 617bc13..277211c 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -6,8 +6,9 @@ import fastifyWebsocket from '@fastify/websocket'; import fastifyStatic from '@fastify/static'; import type { WebSocket } from 'ws'; import { TournamentState } from './tournament'; -import { startLiveLoop } from './liveLoop'; +import { startScheduler } from './ingest/scheduler'; import { ModelEngine } from './model'; +import { db, healthAll } from './db/db'; import type { MatchStatus, ServerMessage } from '../../src/lib/types'; const PORT = Number(process.env.PORT ?? 8787); @@ -27,6 +28,7 @@ export function buildServer() { catch { const err = new Error('Invalid JSON') as Error & { statusCode?: number }; err.statusCode = 400; done(err, undefined); } }); + db(); // open + migrate SQLite before anything reads it const state = new TournamentState(STATIC_DIR); const model = new ModelEngine(STATIC_DIR); model.recompute(state.allFixtures()); @@ -52,6 +54,7 @@ export function buildServer() { // ---- REST: snapshot + predictions (initial load + a no-WS fallback) ---- app.get('/api/snapshot', async () => state.snapshot()); app.get('/api/predictions', async () => model.current()); + app.get('/api/sources/health', async () => ({ sources: healthAll() })); app.get('/healthz', async () => ({ ok: true })); // ---- dev-only: inject a score to exercise the live → standings → WS loop ---- @@ -102,9 +105,9 @@ export function buildServer() { }); }); - // ---- live polling loop (no-op unless a source is configured) ---- - const stopLive = startLiveLoop(state, () => broadcast()); - app.addHook('onClose', async () => stopLive()); + // ---- ingestion scheduler (ESPN primary; football-data / SofaScore fallback) ---- + const stopIngest = startScheduler(state, () => broadcast()); + app.addHook('onClose', async () => stopIngest()); // ---- static SPA (prod) with history fallback ---- if (existsSync(STATIC_DIR)) { diff --git a/server/src/ingest/espn.test.ts b/server/src/ingest/espn.test.ts new file mode 100644 index 0000000..8b09e46 --- /dev/null +++ b/server/src/ingest/espn.test.ts @@ -0,0 +1,92 @@ +import { describe, it, expect } from 'vitest'; +import { normalizeScoreboard, normalizeSummary, type RawSummary } from './espnNormalize'; + +describe('normalizeScoreboard', () => { + it('normalizes events, canonicalizes names, maps status + scores', () => { + const out = normalizeScoreboard({ + events: [ + { + id: '760414', + date: '2026-06-12T02:00Z', + name: 'Czechia at South Korea', + status: { type: { state: 'in' }, displayClock: "67'", period: 2 }, + competitions: [ + { + competitors: [ + { homeAway: 'home', team: { id: '1', displayName: 'South Korea' }, score: '1' }, + { homeAway: 'away', team: { id: '2', displayName: 'Czechia' }, score: '2' }, + ], + }, + ], + }, + { + id: '760415', + date: '2026-06-11T19:00Z', + name: 'South Africa at Mexico', + status: { type: { state: 'pre' } }, + competitions: [ + { + competitors: [ + { homeAway: 'home', team: { id: '203', displayName: 'Mexico' } }, + { homeAway: 'away', team: { id: '467', displayName: 'South Africa' } }, + ], + }, + ], + }, + ], + }); + expect(out).toHaveLength(2); + const live = out[0]!; + expect(live.eventId).toBe('760414'); + expect(live.home).toBe('South Korea'); + expect(live.away).toBe('Czech Republic'); // Czechia → canonical + expect(live.status).toBe('live'); + expect(live.homeScore).toBe(1); + expect(live.awayScore).toBe(2); + expect(live.minute).toBe(67); + + const sched = out[1]!; + expect(sched.status).toBe('scheduled'); + expect(sched.homeScore).toBeNull(); + expect(sched.minute).toBeNull(); + }); +}); + +describe('normalizeSummary', () => { + it('extracts venue, teams, H2H, form, lineups and stats', () => { + const raw: RawSummary = { + gameInfo: { venue: { fullName: 'Estadio Banorte' } }, + header: { + competitions: [ + { competitors: [ + { id: '203', team: { displayName: 'Mexico' }, homeAway: 'home' }, + { id: '467', team: { displayName: 'South Africa' }, homeAway: 'away' }, + ] }, + ], + }, + headToHeadGames: [ + { events: [ + { gameDate: '2010-06-11T14:00Z', homeTeamId: '203', awayTeamId: '467', homeTeamScore: '1', awayTeamScore: '1' }, + ] }, + ], + boxscore: { + form: [ + { team: { id: '203' }, events: [{ gameResult: 'W' }, { gameResult: 'd' }] }, + ], + teams: [ + { team: { id: '203' }, statistics: [{ name: 'possessionPct', displayValue: '58%' }] }, + ], + }, + rosters: [ + { team: { id: '203' }, roster: [{ athlete: { displayName: 'A. Player' }, position: { abbreviation: 'F' }, starter: true, jersey: '9' }] }, + ], + }; + const s = normalizeSummary(raw); + expect(s.venue).toBe('Estadio Banorte'); + expect(s.teams.map((t) => t.name)).toEqual(['Mexico', 'South Africa']); + expect(s.h2h[0]).toMatchObject({ homeScore: 1, awayScore: 1, homeTeamId: '203' }); + expect(s.form['203']).toEqual(['W', 'D']); // uppercased + expect(s.lineups['203']![0]).toMatchObject({ name: 'A. Player', position: 'F', starter: true, number: 9 }); + expect(s.stats['203']!.possessionPct).toBe('58%'); + }); +}); diff --git a/server/src/ingest/espn.ts b/server/src/ingest/espn.ts new file mode 100644 index 0000000..7110e73 --- /dev/null +++ b/server/src/ingest/espn.ts @@ -0,0 +1,34 @@ +import { cachedFetch } from './fetcher'; +import { + normalizeScoreboard, + normalizeSummary, + type EspnEvent, + type EspnMatch, + type EspnSummary, + type RawSummary, +} from './espnNormalize'; + +// ESPN's hidden site API — datacenter-friendly (works from the VPS where +// SofaScore 403s), no auth, covers the World Cup (`fifa.world`). The scoreboard +// drives live scores; the summary endpoint is a rich pre-match/live blob. +const BASE = 'https://site.api.espn.com/apis/site/v2/sports/soccer/fifa.world'; +const SOURCE = 'espn'; + +export type { EspnMatch, EspnSummary } from './espnNormalize'; + +export async function fetchEspnScoreboard(dateYYYYMMDD?: string): Promise { + const url = `${BASE}/scoreboard${dateYYYYMMDD ? `?dates=${dateYYYYMMDD}` : ''}`; + const body = await cachedFetch<{ events?: EspnEvent[] }>(SOURCE, `espn:sb:${dateYYYYMMDD ?? 'now'}`, url, { + ttl: 20_000, + minInterval: 3_000, + }); + return normalizeScoreboard(body); +} + +export async function fetchEspnSummary(eventId: string): Promise { + const raw = await cachedFetch(SOURCE, `espn:summary:${eventId}`, `${BASE}/summary?event=${eventId}`, { + ttl: 5 * 60_000, + minInterval: 2_000, + }); + return normalizeSummary(raw); +} diff --git a/server/src/ingest/espnNormalize.ts b/server/src/ingest/espnNormalize.ts new file mode 100644 index 0000000..770b40e --- /dev/null +++ b/server/src/ingest/espnNormalize.ts @@ -0,0 +1,132 @@ +import { canonicalTeam } from '../../../src/lib/teams'; +import type { LiveMatch } from '../tournament'; +import type { MatchStatus } from '../../../src/lib/types'; + +// Pure ESPN normalizers — no network or DB, so they're unit-testable in +// isolation (and don't drag node:sqlite into the Vite test graph). + +interface EspnCompetitor { + homeAway: 'home' | 'away'; + team: { id: string; displayName: string; name?: string; abbreviation?: string }; + score?: string; +} +interface EspnStatus { type: { state: 'pre' | 'in' | 'post'; completed?: boolean; name?: string }; displayClock?: string; period?: number } +export interface EspnEvent { + id: string; + date: string; + name: string; + competitions: { competitors: EspnCompetitor[]; venue?: { fullName?: string } }[]; + status: EspnStatus; +} + +export interface EspnMatch extends LiveMatch { + eventId: string; +} + +function mapStatus(s: EspnStatus): MatchStatus { + if (s.type.state === 'in') return 'live'; + if (s.type.state === 'post') return 'finished'; + return 'scheduled'; +} + +function parseMinute(s: EspnStatus): number | null { + if (s.type.state !== 'in') return null; + const m = s.displayClock?.match(/(\d+)/); + return m ? Number(m[1]) : (s.period ? (s.period - 1) * 45 : null); +} + +/** Scoreboard body → matches with ESPN event ids. */ +export function normalizeScoreboard(body: { events?: EspnEvent[] }): EspnMatch[] { + const out: EspnMatch[] = []; + for (const e of body.events ?? []) { + const comp = e.competitions[0]; + if (!comp) continue; + const home = comp.competitors.find((c) => c.homeAway === 'home'); + const away = comp.competitors.find((c) => c.homeAway === 'away'); + if (!home || !away) continue; + out.push({ + eventId: e.id, + home: canonicalTeam(home.team.displayName), + away: canonicalTeam(away.team.displayName), + group: null, + stage: 'group', + kickoff: e.date, + homeScore: home.score != null && home.score !== '' ? Number(home.score) : null, + awayScore: away.score != null && away.score !== '' ? Number(away.score) : null, + status: mapStatus(e.status), + minute: parseMinute(e.status), + }); + } + return out; +} + +// ---- summary (rich enrichment) ---- +export interface EspnH2HGame { date: string; homeTeamId: string; awayTeamId: string; homeScore: number; awayScore: number } +export interface EspnTeamRef { id: string; name: string; homeAway: 'home' | 'away' } +export interface EspnLineupPlayer { name: string; position: string | null; starter: boolean; number: number | null } +export interface EspnSummary { + fetchedAt: number; + venue: string | null; + teams: EspnTeamRef[]; + h2h: EspnH2HGame[]; + /** Recent results per team id, most recent first: 'W' | 'D' | 'L'. */ + form: Record; + lineups: Record; + /** Team match stats (possession, shots, …) once the game is live/finished. */ + stats: Record>; +} + +export interface RawSummary { + gameInfo?: { venue?: { fullName?: string } }; + header?: { competitions?: { competitors?: { id: string; team?: { displayName?: string }; homeAway: 'home' | 'away' }[] }[] }; + headToHeadGames?: { events?: { gameDate: string; homeTeamId: string; awayTeamId: string; homeTeamScore: string; awayTeamScore: string }[] }[]; + boxscore?: { + form?: { team?: { id: string }; events?: { gameResult?: string }[] }[]; + teams?: { team?: { id: string }; statistics?: { name: string; displayValue: string }[] }[]; + }; + rosters?: { team?: { id: string }; roster?: { athlete?: { displayName?: string }; position?: { abbreviation?: string }; starter?: boolean; jersey?: string }[] }[]; +} + +export function normalizeSummary(raw: RawSummary): EspnSummary { + const teams: EspnTeamRef[] = (raw.header?.competitions?.[0]?.competitors ?? []).map((c) => ({ + id: c.id, + name: canonicalTeam(c.team?.displayName ?? ''), + homeAway: c.homeAway, + })); + + const h2h: EspnH2HGame[] = (raw.headToHeadGames?.[0]?.events ?? []).map((g) => ({ + date: g.gameDate, + homeTeamId: g.homeTeamId, + awayTeamId: g.awayTeamId, + homeScore: Number(g.homeTeamScore), + awayScore: Number(g.awayTeamScore), + })); + + const form: Record = {}; + for (const f of raw.boxscore?.form ?? []) { + const id = f.team?.id; + if (!id) continue; + form[id] = (f.events ?? []).map((e) => (e.gameResult ?? '').toUpperCase()).filter(Boolean); + } + + const lineups: Record = {}; + for (const r of raw.rosters ?? []) { + const id = r.team?.id; + if (!id) continue; + lineups[id] = (r.roster ?? []).map((p) => ({ + name: p.athlete?.displayName ?? '', + position: p.position?.abbreviation ?? null, + starter: !!p.starter, + number: p.jersey ? Number(p.jersey) : null, + })); + } + + const stats: Record> = {}; + for (const t of raw.boxscore?.teams ?? []) { + const id = t.team?.id; + if (!id) continue; + stats[id] = Object.fromEntries((t.statistics ?? []).map((s) => [s.name, s.displayValue])); + } + + return { fetchedAt: Date.now(), venue: raw.gameInfo?.venue?.fullName ?? null, teams, h2h, form, lineups, stats }; +} diff --git a/server/src/ingest/fetcher.ts b/server/src/ingest/fetcher.ts new file mode 100644 index 0000000..2b25d8c --- /dev/null +++ b/server/src/ingest/fetcher.ts @@ -0,0 +1,88 @@ +import { cacheGet, cacheSet, healthGet, healthRecord } from '../db/db'; + +// One resilient fetch path for every source: DB-cached, rate-limited, and +// circuit-broken. A blocked source (e.g. SofaScore 403 from the VPS) trips its +// breaker and is skipped; callers transparently get stale cache or a thrown +// error they already handle — the UI, which reads the DB, never blocks. + +const CIRCUIT_THRESHOLD = 4; // consecutive failures before opening +const CIRCUIT_COOLDOWN = 10 * 60_000; // base cooldown, escalates with failures + +const lastCall = new Map(); + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +export function browserHeaders(extra: Record = {}): Record { + return { + 'User-Agent': + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', + Accept: 'application/json, text/plain, */*', + 'Accept-Language': 'en-US,en;q=0.9', + ...extra, + }; +} + +export interface FetchOpts { + /** Cache freshness window (ms). */ + ttl: number; + /** Minimum interval between calls to this source (ms). */ + minInterval?: number; + headers?: Record; + timeoutMs?: number; + /** Parse as text instead of JSON. */ + text?: boolean; +} + +export function circuitOpen(source: string): boolean { + const h = healthGet(source); + return !!h && h.open_until > Date.now(); +} + +async function rateLimit(source: string, minInterval: number): Promise { + if (!minInterval) return; + const last = lastCall.get(source) ?? 0; + const wait = minInterval - (Date.now() - last); + if (wait > 0) await sleep(wait + Math.random() * 250); // jitter + lastCall.set(source, Date.now()); +} + +/** + * Fetch JSON (or text) for `source` at `url`, keyed in the cache by `key`. + * Returns fresh cache when valid, otherwise fetches; on failure returns stale + * cache if present, else throws. + */ +export async function cachedFetch( + source: string, + key: string, + url: string, + opts: FetchOpts, +): Promise { + const cached = cacheGet(key); + if (cached?.fresh) return (opts.text ? cached.value : JSON.parse(cached.value)) as T; + + if (circuitOpen(source)) { + if (cached) return (opts.text ? cached.value : JSON.parse(cached.value)) as T; + throw new Error(`${source}: circuit open`); + } + + await rateLimit(source, opts.minInterval ?? 0); + + try { + const res = await fetch(url, { + headers: browserHeaders(opts.headers), + signal: AbortSignal.timeout(opts.timeoutMs ?? 12_000), + }); + if (!res.ok) throw new Error(`${res.status}`); + const body = await res.text(); + cacheSet(key, body, opts.ttl); + healthRecord(source, true); + return (opts.text ? body : JSON.parse(body)) as T; + } catch (e) { + const note = e instanceof Error ? e.message : String(e); + const consec = (healthGet(source)?.consec_fail ?? 0) + 1; + const openMs = consec >= CIRCUIT_THRESHOLD ? CIRCUIT_COOLDOWN * Math.min(6, consec - CIRCUIT_THRESHOLD + 1) : undefined; + healthRecord(source, false, note, openMs); + if (cached) return (opts.text ? cached.value : JSON.parse(cached.value)) as T; // serve stale + throw new Error(`${source} fetch failed: ${note}`); + } +} diff --git a/server/src/ingest/scheduler.ts b/server/src/ingest/scheduler.ts new file mode 100644 index 0000000..2c993f6 --- /dev/null +++ b/server/src/ingest/scheduler.ts @@ -0,0 +1,128 @@ +import { TournamentState, type LiveMatch, type Source } from '../tournament'; +import { fetchEspnScoreboard, fetchEspnSummary } from './espn'; +import { fetchFootballData } from '../footballData'; +import { fetchSofascore } from '../sofascore'; +import { setSourceMap, getSourceMap, setMatchExt, logIngest } from '../db/db'; +import type { Fixture } from '../../../src/lib/types'; + +// The ingestion orchestrator. Replaces the old live loop: ESPN is the primary +// live + enrichment source (works from the VPS); football-data.org and SofaScore +// are fallbacks. Everything persists to SQLite; the UI reads the DB, so a blocked +// source only means staler data. + +const LIVE_WINDOW_MS = 3 * 60 * 60 * 1000; +const POLL_LIVE_MS = 20_000; +const POLL_IDLE_MS = 10 * 60_000; +const ENRICH_MS = 20 * 60_000; + +function dateUTC(iso: string): string { + const d = new Date(iso); + return `${d.getUTCFullYear()}${String(d.getUTCMonth() + 1).padStart(2, '0')}${String(d.getUTCDate()).padStart(2, '0')}`; +} + +export function startScheduler(state: TournamentState, onChange: () => void): () => void { + const token = process.env.FOOTBALL_DATA_TOKEN?.trim(); + const sofa = process.env.ENABLE_SOFASCORE === 'true'; + let stopped = false; + let liveTimer: ReturnType | undefined; + let enrichTimer: ReturnType | undefined; + + console.log(`[ingest] scheduler — espn:on football-data:${token ? 'on' : 'off'} sofascore:${sofa ? 'on' : 'off'}`); + + // ---- one-time: map every fixture to its ESPN event id (across all dates) ---- + const mapAllFixtures = async (): Promise => { + const dates = [...new Set(state.allFixtures().map((f) => dateUTC(f.kickoff)))]; + let mapped = 0; + for (const date of dates) { + if (stopped) return; + const t0 = Date.now(); + try { + const evs = await fetchEspnScoreboard(date); + for (const m of evs) { + const num = state.matchFixtureNum(m.home, m.away, 'group', m.kickoff); + if (num) { setSourceMap(num, 'espn', m.eventId); mapped++; } + } + logIngest('espn', 'map', true, Date.now() - t0, `${date}: ${evs.length} events`); + } catch (e) { + logIngest('espn', 'map', false, Date.now() - t0, e instanceof Error ? e.message : String(e)); + } + } + console.log(`[ingest] mapped ${mapped} fixtures to ESPN event ids`); + }; + + // ---- live scoreboard ---- + const liveTick = async (): Promise => { + const t0 = Date.now(); + let matches: LiveMatch[] = []; + let source: Source = 'seed'; + try { + const espn = await fetchEspnScoreboard(); + matches = espn; + source = 'espn'; + for (const m of espn) { + const num = state.matchFixtureNum(m.home, m.away, 'group', m.kickoff); + if (num) setSourceMap(num, 'espn', m.eventId); + } + logIngest('espn', 'scoreboard', true, Date.now() - t0, `${espn.length} events`); + } catch (e) { + logIngest('espn', 'scoreboard', false, Date.now() - t0, e instanceof Error ? e.message : String(e)); + } + if (!matches.length && token) { + try { matches = await fetchFootballData(token); source = 'football-data'; } catch { /* logged via health */ } + } + if (!matches.length && sofa) { + try { matches = await fetchSofascore(); source = 'sofascore'; } catch { /* expected block */ } + } + if (matches.length) { + const changed = state.mergeLive(matches, source); + if (changed.length) { console.log(`[ingest] ${changed.length} fixture(s) updated via ${source}`); onChange(); } + } + }; + + // ---- enrichment: ESPN summary for imminent/live fixtures ---- + const enrichTick = async (): Promise => { + const now = Date.now(); + const relevant = state.allFixtures().filter((f) => { + const k = new Date(f.kickoff).getTime(); + return f.status === 'live' || (k > now - LIVE_WINDOW_MS && k < now + 3 * 24 * 60 * 60 * 1000); + }); + for (const f of relevant) { + if (stopped) return; + const eid = getSourceMap(f.num, 'espn'); + if (!eid) continue; + const t0 = Date.now(); + try { + const summary = await fetchEspnSummary(eid); + setMatchExt(f.num, summary); + logIngest('espn', 'summary', true, Date.now() - t0, `M${f.num}`); + } catch (e) { + logIngest('espn', 'summary', false, Date.now() - t0, e instanceof Error ? e.message : String(e)); + } + } + }; + + const scheduleLive = (): void => { + if (stopped) return; + const delay = state.hasLiveActivity(LIVE_WINDOW_MS) ? POLL_LIVE_MS : POLL_IDLE_MS; + liveTimer = setTimeout(async () => { await liveTick().catch(() => {}); scheduleLive(); }, delay); + }; + const scheduleEnrich = (): void => { + if (stopped) return; + enrichTimer = setTimeout(async () => { await enrichTick().catch(() => {}); scheduleEnrich(); }, ENRICH_MS); + }; + + // boot: map ids, then run the loops + void (async () => { + await mapAllFixtures().catch(() => {}); + await liveTick().catch(() => {}); + await enrichTick().catch(() => {}); + scheduleLive(); + scheduleEnrich(); + })(); + + return () => { + stopped = true; + if (liveTimer) clearTimeout(liveTimer); + if (enrichTimer) clearTimeout(enrichTimer); + }; +} diff --git a/server/src/liveLoop.ts b/server/src/liveLoop.ts deleted file mode 100644 index 36b253a..0000000 --- a/server/src/liveLoop.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { TournamentState, type LiveMatch, type Source } from './tournament'; -import { fetchFootballData } from './footballData'; -import { fetchSofascore } from './sofascore'; -import type { Fixture } from '../../src/lib/types'; - -const LIVE_WINDOW_MS = 3 * 60 * 60 * 1000; // ±3h around a kickoff = "match window" -const POLL_LIVE_MS = 60_000; // football-data free tier is 10 req/min — 1/min is safe -const POLL_IDLE_MS = 10 * 60_000; - -/** - * Poll the live sources and merge results onto the tournament state. SofaScore - * (if enabled) is tried first for freshness; football-data.org is the fallback - * and the reliable backbone. Returns a stop function. - */ -export function startLiveLoop( - state: TournamentState, - onChange: (changed: Fixture[]) => void, -): () => void { - const token = process.env.FOOTBALL_DATA_TOKEN?.trim(); - const sofa = process.env.ENABLE_SOFASCORE === 'true'; - - if (!token && !sofa) { - console.log('[live] seed-only (set FOOTBALL_DATA_TOKEN and/or ENABLE_SOFASCORE=true to enable)'); - return () => {}; - } - console.log(`[live] enabled — football-data:${token ? 'on' : 'off'} sofascore:${sofa ? 'on' : 'off'}`); - - let stopped = false; - let timer: ReturnType | undefined; - - const tick = async (): Promise => { - let matches: LiveMatch[] = []; - let source: Source = 'seed'; - - if (sofa) { - try { - matches = await fetchSofascore(); - if (matches.length) source = 'sofascore'; - } catch (e) { - console.warn('[live] sofascore failed:', e instanceof Error ? e.message : e); - } - } - if (!matches.length && token) { - try { - matches = await fetchFootballData(token); - source = 'football-data'; - } catch (e) { - console.warn('[live] football-data failed:', e instanceof Error ? e.message : e); - } - } - - if (matches.length) { - const changed = state.mergeLive(matches, source); - if (changed.length) { - console.log(`[live] ${changed.length} fixture(s) updated via ${source}`); - onChange(changed); - } - } - }; - - const schedule = (): void => { - if (stopped) return; - const delay = state.hasLiveActivity(LIVE_WINDOW_MS) ? POLL_LIVE_MS : POLL_IDLE_MS; - timer = setTimeout(async () => { - await tick().catch((e) => console.warn('[live] tick error:', e)); - schedule(); - }, delay); - }; - - // Kick off immediately, then settle into the dynamic schedule. - void tick().catch((e) => console.warn('[live] initial tick error:', e)).finally(schedule); - - return () => { - stopped = true; - if (timer) clearTimeout(timer); - }; -} diff --git a/server/src/tournament.ts b/server/src/tournament.ts index a6935c8..39c5b70 100644 --- a/server/src/tournament.ts +++ b/server/src/tournament.ts @@ -68,6 +68,16 @@ export class TournamentState { return this.fixtures; } + /** Which fixture number does a (home, away) pairing map to? Used by the + * ingestion layer to key provider event ids against our fixtures. */ + matchFixtureNum(home: string, away: string, stage: Fixture['stage'] = 'group', kickoff: string | null = null): number | null { + const f = this.findFixture({ + home, away, group: null, stage, kickoff, + homeScore: null, awayScore: null, status: 'scheduled', minute: null, + }); + return f?.num ?? null; + } + /** True if any match is live or kicks off within `windowMs` of now — used to * poll the APIs often during match windows and rarely when nothing is on. */ hasLiveActivity(windowMs: number): boolean { diff --git a/src/lib/types.ts b/src/lib/types.ts index 100ffd6..0f85698 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -63,7 +63,7 @@ export interface Snapshot { /** When the live layer last refreshed (ISO), or null if seed-only. */ updatedAt: string | null; /** Which live source produced the current scores. */ - source: 'seed' | 'football-data' | 'sofascore'; + source: 'seed' | 'espn' | 'football-data' | 'sofascore'; fixtures: Fixture[]; /** Group letter → ranked standings. */ tables: Record;