v2 Phase 1: ingestion + SQLite foundation

- Persistence via Node's built-in node:sqlite (zero native deps) on a Docker
  volume: response cache, source health, fixture↔provider id map, per-fixture
  enrichment, ingest log. Runtime bumped to node:24-slim + --experimental-sqlite.
- Resilient fetcher: DB cache + per-source rate-limit + jittered backoff +
  circuit-breaker (blocked sources trip + skip; UI reads DB, never breaks).
- ESPN hidden API as the PRIMARY rich source (works from the VPS where SofaScore
  403s): scoreboard (live scores w/ clock) + summary (venue, H2H, recent form,
  lineups, team stats). football-data / SofaScore are fallbacks.
- Scheduler: maps all 72 group fixtures to ESPN event ids, polls live on a
  dynamic cadence, enriches imminent fixtures into match_ext. /api/sources/health.
- Verified: DB populates (real H2H + form for opening matches), 22 tests pass,
  Docker image runs node:sqlite on the volume and persists across restart.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-11 15:21:01 +02:00
parent 9c7605391c
commit b6f62679c9
13 changed files with 697 additions and 88 deletions
+92
View File
@@ -0,0 +1,92 @@
import { describe, it, expect } from 'vitest';
import { normalizeScoreboard, normalizeSummary, type RawSummary } from './espnNormalize';
describe('normalizeScoreboard', () => {
it('normalizes events, canonicalizes names, maps status + scores', () => {
const out = normalizeScoreboard({
events: [
{
id: '760414',
date: '2026-06-12T02:00Z',
name: 'Czechia at South Korea',
status: { type: { state: 'in' }, displayClock: "67'", period: 2 },
competitions: [
{
competitors: [
{ homeAway: 'home', team: { id: '1', displayName: 'South Korea' }, score: '1' },
{ homeAway: 'away', team: { id: '2', displayName: 'Czechia' }, score: '2' },
],
},
],
},
{
id: '760415',
date: '2026-06-11T19:00Z',
name: 'South Africa at Mexico',
status: { type: { state: 'pre' } },
competitions: [
{
competitors: [
{ homeAway: 'home', team: { id: '203', displayName: 'Mexico' } },
{ homeAway: 'away', team: { id: '467', displayName: 'South Africa' } },
],
},
],
},
],
});
expect(out).toHaveLength(2);
const live = out[0]!;
expect(live.eventId).toBe('760414');
expect(live.home).toBe('South Korea');
expect(live.away).toBe('Czech Republic'); // Czechia → canonical
expect(live.status).toBe('live');
expect(live.homeScore).toBe(1);
expect(live.awayScore).toBe(2);
expect(live.minute).toBe(67);
const sched = out[1]!;
expect(sched.status).toBe('scheduled');
expect(sched.homeScore).toBeNull();
expect(sched.minute).toBeNull();
});
});
describe('normalizeSummary', () => {
it('extracts venue, teams, H2H, form, lineups and stats', () => {
const raw: RawSummary = {
gameInfo: { venue: { fullName: 'Estadio Banorte' } },
header: {
competitions: [
{ competitors: [
{ id: '203', team: { displayName: 'Mexico' }, homeAway: 'home' },
{ id: '467', team: { displayName: 'South Africa' }, homeAway: 'away' },
] },
],
},
headToHeadGames: [
{ events: [
{ gameDate: '2010-06-11T14:00Z', homeTeamId: '203', awayTeamId: '467', homeTeamScore: '1', awayTeamScore: '1' },
] },
],
boxscore: {
form: [
{ team: { id: '203' }, events: [{ gameResult: 'W' }, { gameResult: 'd' }] },
],
teams: [
{ team: { id: '203' }, statistics: [{ name: 'possessionPct', displayValue: '58%' }] },
],
},
rosters: [
{ team: { id: '203' }, roster: [{ athlete: { displayName: 'A. Player' }, position: { abbreviation: 'F' }, starter: true, jersey: '9' }] },
],
};
const s = normalizeSummary(raw);
expect(s.venue).toBe('Estadio Banorte');
expect(s.teams.map((t) => t.name)).toEqual(['Mexico', 'South Africa']);
expect(s.h2h[0]).toMatchObject({ homeScore: 1, awayScore: 1, homeTeamId: '203' });
expect(s.form['203']).toEqual(['W', 'D']); // uppercased
expect(s.lineups['203']![0]).toMatchObject({ name: 'A. Player', position: 'F', starter: true, number: 9 });
expect(s.stats['203']!.possessionPct).toBe('58%');
});
});
+34
View File
@@ -0,0 +1,34 @@
import { cachedFetch } from './fetcher';
import {
normalizeScoreboard,
normalizeSummary,
type EspnEvent,
type EspnMatch,
type EspnSummary,
type RawSummary,
} from './espnNormalize';
// ESPN's hidden site API — datacenter-friendly (works from the VPS where
// SofaScore 403s), no auth, covers the World Cup (`fifa.world`). The scoreboard
// drives live scores; the summary endpoint is a rich pre-match/live blob.
const BASE = 'https://site.api.espn.com/apis/site/v2/sports/soccer/fifa.world';
const SOURCE = 'espn';
export type { EspnMatch, EspnSummary } from './espnNormalize';
export async function fetchEspnScoreboard(dateYYYYMMDD?: string): Promise<EspnMatch[]> {
const url = `${BASE}/scoreboard${dateYYYYMMDD ? `?dates=${dateYYYYMMDD}` : ''}`;
const body = await cachedFetch<{ events?: EspnEvent[] }>(SOURCE, `espn:sb:${dateYYYYMMDD ?? 'now'}`, url, {
ttl: 20_000,
minInterval: 3_000,
});
return normalizeScoreboard(body);
}
export async function fetchEspnSummary(eventId: string): Promise<EspnSummary> {
const raw = await cachedFetch<RawSummary>(SOURCE, `espn:summary:${eventId}`, `${BASE}/summary?event=${eventId}`, {
ttl: 5 * 60_000,
minInterval: 2_000,
});
return normalizeSummary(raw);
}
+132
View File
@@ -0,0 +1,132 @@
import { canonicalTeam } from '../../../src/lib/teams';
import type { LiveMatch } from '../tournament';
import type { MatchStatus } from '../../../src/lib/types';
// Pure ESPN normalizers — no network or DB, so they're unit-testable in
// isolation (and don't drag node:sqlite into the Vite test graph).
interface EspnCompetitor {
homeAway: 'home' | 'away';
team: { id: string; displayName: string; name?: string; abbreviation?: string };
score?: string;
}
interface EspnStatus { type: { state: 'pre' | 'in' | 'post'; completed?: boolean; name?: string }; displayClock?: string; period?: number }
export interface EspnEvent {
id: string;
date: string;
name: string;
competitions: { competitors: EspnCompetitor[]; venue?: { fullName?: string } }[];
status: EspnStatus;
}
export interface EspnMatch extends LiveMatch {
eventId: string;
}
function mapStatus(s: EspnStatus): MatchStatus {
if (s.type.state === 'in') return 'live';
if (s.type.state === 'post') return 'finished';
return 'scheduled';
}
function parseMinute(s: EspnStatus): number | null {
if (s.type.state !== 'in') return null;
const m = s.displayClock?.match(/(\d+)/);
return m ? Number(m[1]) : (s.period ? (s.period - 1) * 45 : null);
}
/** Scoreboard body → matches with ESPN event ids. */
export function normalizeScoreboard(body: { events?: EspnEvent[] }): EspnMatch[] {
const out: EspnMatch[] = [];
for (const e of body.events ?? []) {
const comp = e.competitions[0];
if (!comp) continue;
const home = comp.competitors.find((c) => c.homeAway === 'home');
const away = comp.competitors.find((c) => c.homeAway === 'away');
if (!home || !away) continue;
out.push({
eventId: e.id,
home: canonicalTeam(home.team.displayName),
away: canonicalTeam(away.team.displayName),
group: null,
stage: 'group',
kickoff: e.date,
homeScore: home.score != null && home.score !== '' ? Number(home.score) : null,
awayScore: away.score != null && away.score !== '' ? Number(away.score) : null,
status: mapStatus(e.status),
minute: parseMinute(e.status),
});
}
return out;
}
// ---- summary (rich enrichment) ----
export interface EspnH2HGame { date: string; homeTeamId: string; awayTeamId: string; homeScore: number; awayScore: number }
export interface EspnTeamRef { id: string; name: string; homeAway: 'home' | 'away' }
export interface EspnLineupPlayer { name: string; position: string | null; starter: boolean; number: number | null }
export interface EspnSummary {
fetchedAt: number;
venue: string | null;
teams: EspnTeamRef[];
h2h: EspnH2HGame[];
/** Recent results per team id, most recent first: 'W' | 'D' | 'L'. */
form: Record<string, string[]>;
lineups: Record<string, EspnLineupPlayer[]>;
/** Team match stats (possession, shots, …) once the game is live/finished. */
stats: Record<string, Record<string, string>>;
}
export interface RawSummary {
gameInfo?: { venue?: { fullName?: string } };
header?: { competitions?: { competitors?: { id: string; team?: { displayName?: string }; homeAway: 'home' | 'away' }[] }[] };
headToHeadGames?: { events?: { gameDate: string; homeTeamId: string; awayTeamId: string; homeTeamScore: string; awayTeamScore: string }[] }[];
boxscore?: {
form?: { team?: { id: string }; events?: { gameResult?: string }[] }[];
teams?: { team?: { id: string }; statistics?: { name: string; displayValue: string }[] }[];
};
rosters?: { team?: { id: string }; roster?: { athlete?: { displayName?: string }; position?: { abbreviation?: string }; starter?: boolean; jersey?: string }[] }[];
}
export function normalizeSummary(raw: RawSummary): EspnSummary {
const teams: EspnTeamRef[] = (raw.header?.competitions?.[0]?.competitors ?? []).map((c) => ({
id: c.id,
name: canonicalTeam(c.team?.displayName ?? ''),
homeAway: c.homeAway,
}));
const h2h: EspnH2HGame[] = (raw.headToHeadGames?.[0]?.events ?? []).map((g) => ({
date: g.gameDate,
homeTeamId: g.homeTeamId,
awayTeamId: g.awayTeamId,
homeScore: Number(g.homeTeamScore),
awayScore: Number(g.awayTeamScore),
}));
const form: Record<string, string[]> = {};
for (const f of raw.boxscore?.form ?? []) {
const id = f.team?.id;
if (!id) continue;
form[id] = (f.events ?? []).map((e) => (e.gameResult ?? '').toUpperCase()).filter(Boolean);
}
const lineups: Record<string, EspnLineupPlayer[]> = {};
for (const r of raw.rosters ?? []) {
const id = r.team?.id;
if (!id) continue;
lineups[id] = (r.roster ?? []).map((p) => ({
name: p.athlete?.displayName ?? '',
position: p.position?.abbreviation ?? null,
starter: !!p.starter,
number: p.jersey ? Number(p.jersey) : null,
}));
}
const stats: Record<string, Record<string, string>> = {};
for (const t of raw.boxscore?.teams ?? []) {
const id = t.team?.id;
if (!id) continue;
stats[id] = Object.fromEntries((t.statistics ?? []).map((s) => [s.name, s.displayValue]));
}
return { fetchedAt: Date.now(), venue: raw.gameInfo?.venue?.fullName ?? null, teams, h2h, form, lineups, stats };
}
+88
View File
@@ -0,0 +1,88 @@
import { cacheGet, cacheSet, healthGet, healthRecord } from '../db/db';
// One resilient fetch path for every source: DB-cached, rate-limited, and
// circuit-broken. A blocked source (e.g. SofaScore 403 from the VPS) trips its
// breaker and is skipped; callers transparently get stale cache or a thrown
// error they already handle — the UI, which reads the DB, never blocks.
const CIRCUIT_THRESHOLD = 4; // consecutive failures before opening
const CIRCUIT_COOLDOWN = 10 * 60_000; // base cooldown, escalates with failures
const lastCall = new Map<string, number>();
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
export function browserHeaders(extra: Record<string, string> = {}): Record<string, string> {
return {
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
Accept: 'application/json, text/plain, */*',
'Accept-Language': 'en-US,en;q=0.9',
...extra,
};
}
export interface FetchOpts {
/** Cache freshness window (ms). */
ttl: number;
/** Minimum interval between calls to this source (ms). */
minInterval?: number;
headers?: Record<string, string>;
timeoutMs?: number;
/** Parse as text instead of JSON. */
text?: boolean;
}
export function circuitOpen(source: string): boolean {
const h = healthGet(source);
return !!h && h.open_until > Date.now();
}
async function rateLimit(source: string, minInterval: number): Promise<void> {
if (!minInterval) return;
const last = lastCall.get(source) ?? 0;
const wait = minInterval - (Date.now() - last);
if (wait > 0) await sleep(wait + Math.random() * 250); // jitter
lastCall.set(source, Date.now());
}
/**
* Fetch JSON (or text) for `source` at `url`, keyed in the cache by `key`.
* Returns fresh cache when valid, otherwise fetches; on failure returns stale
* cache if present, else throws.
*/
export async function cachedFetch<T = unknown>(
source: string,
key: string,
url: string,
opts: FetchOpts,
): Promise<T> {
const cached = cacheGet(key);
if (cached?.fresh) return (opts.text ? cached.value : JSON.parse(cached.value)) as T;
if (circuitOpen(source)) {
if (cached) return (opts.text ? cached.value : JSON.parse(cached.value)) as T;
throw new Error(`${source}: circuit open`);
}
await rateLimit(source, opts.minInterval ?? 0);
try {
const res = await fetch(url, {
headers: browserHeaders(opts.headers),
signal: AbortSignal.timeout(opts.timeoutMs ?? 12_000),
});
if (!res.ok) throw new Error(`${res.status}`);
const body = await res.text();
cacheSet(key, body, opts.ttl);
healthRecord(source, true);
return (opts.text ? body : JSON.parse(body)) as T;
} catch (e) {
const note = e instanceof Error ? e.message : String(e);
const consec = (healthGet(source)?.consec_fail ?? 0) + 1;
const openMs = consec >= CIRCUIT_THRESHOLD ? CIRCUIT_COOLDOWN * Math.min(6, consec - CIRCUIT_THRESHOLD + 1) : undefined;
healthRecord(source, false, note, openMs);
if (cached) return (opts.text ? cached.value : JSON.parse(cached.value)) as T; // serve stale
throw new Error(`${source} fetch failed: ${note}`);
}
}
+128
View File
@@ -0,0 +1,128 @@
import { TournamentState, type LiveMatch, type Source } from '../tournament';
import { fetchEspnScoreboard, fetchEspnSummary } from './espn';
import { fetchFootballData } from '../footballData';
import { fetchSofascore } from '../sofascore';
import { setSourceMap, getSourceMap, setMatchExt, logIngest } from '../db/db';
import type { Fixture } from '../../../src/lib/types';
// The ingestion orchestrator. Replaces the old live loop: ESPN is the primary
// live + enrichment source (works from the VPS); football-data.org and SofaScore
// are fallbacks. Everything persists to SQLite; the UI reads the DB, so a blocked
// source only means staler data.
const LIVE_WINDOW_MS = 3 * 60 * 60 * 1000;
const POLL_LIVE_MS = 20_000;
const POLL_IDLE_MS = 10 * 60_000;
const ENRICH_MS = 20 * 60_000;
function dateUTC(iso: string): string {
const d = new Date(iso);
return `${d.getUTCFullYear()}${String(d.getUTCMonth() + 1).padStart(2, '0')}${String(d.getUTCDate()).padStart(2, '0')}`;
}
export function startScheduler(state: TournamentState, onChange: () => void): () => void {
const token = process.env.FOOTBALL_DATA_TOKEN?.trim();
const sofa = process.env.ENABLE_SOFASCORE === 'true';
let stopped = false;
let liveTimer: ReturnType<typeof setTimeout> | undefined;
let enrichTimer: ReturnType<typeof setTimeout> | undefined;
console.log(`[ingest] scheduler — espn:on football-data:${token ? 'on' : 'off'} sofascore:${sofa ? 'on' : 'off'}`);
// ---- one-time: map every fixture to its ESPN event id (across all dates) ----
const mapAllFixtures = async (): Promise<void> => {
const dates = [...new Set(state.allFixtures().map((f) => dateUTC(f.kickoff)))];
let mapped = 0;
for (const date of dates) {
if (stopped) return;
const t0 = Date.now();
try {
const evs = await fetchEspnScoreboard(date);
for (const m of evs) {
const num = state.matchFixtureNum(m.home, m.away, 'group', m.kickoff);
if (num) { setSourceMap(num, 'espn', m.eventId); mapped++; }
}
logIngest('espn', 'map', true, Date.now() - t0, `${date}: ${evs.length} events`);
} catch (e) {
logIngest('espn', 'map', false, Date.now() - t0, e instanceof Error ? e.message : String(e));
}
}
console.log(`[ingest] mapped ${mapped} fixtures to ESPN event ids`);
};
// ---- live scoreboard ----
const liveTick = async (): Promise<void> => {
const t0 = Date.now();
let matches: LiveMatch[] = [];
let source: Source = 'seed';
try {
const espn = await fetchEspnScoreboard();
matches = espn;
source = 'espn';
for (const m of espn) {
const num = state.matchFixtureNum(m.home, m.away, 'group', m.kickoff);
if (num) setSourceMap(num, 'espn', m.eventId);
}
logIngest('espn', 'scoreboard', true, Date.now() - t0, `${espn.length} events`);
} catch (e) {
logIngest('espn', 'scoreboard', false, Date.now() - t0, e instanceof Error ? e.message : String(e));
}
if (!matches.length && token) {
try { matches = await fetchFootballData(token); source = 'football-data'; } catch { /* logged via health */ }
}
if (!matches.length && sofa) {
try { matches = await fetchSofascore(); source = 'sofascore'; } catch { /* expected block */ }
}
if (matches.length) {
const changed = state.mergeLive(matches, source);
if (changed.length) { console.log(`[ingest] ${changed.length} fixture(s) updated via ${source}`); onChange(); }
}
};
// ---- enrichment: ESPN summary for imminent/live fixtures ----
const enrichTick = async (): Promise<void> => {
const now = Date.now();
const relevant = state.allFixtures().filter((f) => {
const k = new Date(f.kickoff).getTime();
return f.status === 'live' || (k > now - LIVE_WINDOW_MS && k < now + 3 * 24 * 60 * 60 * 1000);
});
for (const f of relevant) {
if (stopped) return;
const eid = getSourceMap(f.num, 'espn');
if (!eid) continue;
const t0 = Date.now();
try {
const summary = await fetchEspnSummary(eid);
setMatchExt(f.num, summary);
logIngest('espn', 'summary', true, Date.now() - t0, `M${f.num}`);
} catch (e) {
logIngest('espn', 'summary', false, Date.now() - t0, e instanceof Error ? e.message : String(e));
}
}
};
const scheduleLive = (): void => {
if (stopped) return;
const delay = state.hasLiveActivity(LIVE_WINDOW_MS) ? POLL_LIVE_MS : POLL_IDLE_MS;
liveTimer = setTimeout(async () => { await liveTick().catch(() => {}); scheduleLive(); }, delay);
};
const scheduleEnrich = (): void => {
if (stopped) return;
enrichTimer = setTimeout(async () => { await enrichTick().catch(() => {}); scheduleEnrich(); }, ENRICH_MS);
};
// boot: map ids, then run the loops
void (async () => {
await mapAllFixtures().catch(() => {});
await liveTick().catch(() => {});
await enrichTick().catch(() => {});
scheduleLive();
scheduleEnrich();
})();
return () => {
stopped = true;
if (liveTimer) clearTimeout(liveTimer);
if (enrichTimer) clearTimeout(enrichTimer);
};
}