v2 Phase 1: ingestion + SQLite foundation
- Persistence via Node's built-in node:sqlite (zero native deps) on a Docker volume: response cache, source health, fixture↔provider id map, per-fixture enrichment, ingest log. Runtime bumped to node:24-slim + --experimental-sqlite. - Resilient fetcher: DB cache + per-source rate-limit + jittered backoff + circuit-breaker (blocked sources trip + skip; UI reads DB, never breaks). - ESPN hidden API as the PRIMARY rich source (works from the VPS where SofaScore 403s): scoreboard (live scores w/ clock) + summary (venue, H2H, recent form, lineups, team stats). football-data / SofaScore are fallbacks. - Scheduler: maps all 72 group fixtures to ESPN event ids, polls live on a dynamic cadence, enriches imminent fixtures into match_ext. /api/sources/health. - Verified: DB populates (real H2H + form for opening matches), 22 tests pass, Docker image runs node:sqlite on the volume and persists across restart. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -7,6 +7,10 @@ server/node_modules
|
||||
.DS_Store
|
||||
.env
|
||||
.env.local
|
||||
# local SQLite (dev); prod DB lives on the Docker volume
|
||||
/data/cup26.db
|
||||
/data/cup26.db-wal
|
||||
/data/cup26.db-shm
|
||||
# Generated build artifacts (reproduced by `npm run data:build`)
|
||||
public/data/fixtures.json
|
||||
public/data/ratings.json
|
||||
|
||||
+6
-5
@@ -8,15 +8,16 @@ RUN bun run data:build # icons + fixtures.json + ratings.json → public/
|
||||
RUN bun run build # tsc -b && vite build → /app/dist
|
||||
RUN bun run build:server # esbuild → /app/server/dist/index.js
|
||||
|
||||
# --- runtime stage: small node image, non-root, serves dist + /ws + /api ---
|
||||
FROM node:20-slim AS runtime
|
||||
ENV NODE_ENV=production STATIC_DIR=/app/dist PORT=8787
|
||||
# --- runtime stage: node 24 (built-in node:sqlite), non-root, serves dist + /ws + /api ---
|
||||
FROM node:24-slim AS runtime
|
||||
ENV NODE_ENV=production STATIC_DIR=/app/dist PORT=8787 DATA_DIR=/data
|
||||
WORKDIR /app/server
|
||||
COPY server/package.json ./
|
||||
RUN npm install --omit=dev --no-audit --no-fund
|
||||
COPY --from=build /app/server/dist ./dist
|
||||
COPY --from=build /app/dist /app/dist
|
||||
RUN chown -R node:node /app
|
||||
# SQLite lives on a named volume; create it node-owned so a fresh mount is writable.
|
||||
RUN mkdir -p /data && chown -R node:node /app /data
|
||||
USER node
|
||||
EXPOSE 8787
|
||||
CMD ["node", "dist/index.js"]
|
||||
CMD ["node", "--experimental-sqlite", "dist/index.js"]
|
||||
|
||||
@@ -14,10 +14,14 @@ services:
|
||||
- STATIC_DIR=/app/dist
|
||||
- NODE_ENV=production
|
||||
- ALLOWED_ORIGINS=https://cup26.briggen.dev
|
||||
- DATA_DIR=/data
|
||||
# football-data.org free API token (https://www.football-data.org/client/register)
|
||||
- FOOTBALL_DATA_TOKEN=${FOOTBALL_DATA_TOKEN:-}
|
||||
# opt-in: layer the unofficial real-time source on top of the free API
|
||||
# opt-in unofficial sources (usually blocked from a datacenter IP; ESPN is primary)
|
||||
- ENABLE_SOFASCORE=${ENABLE_SOFASCORE:-false}
|
||||
- ENABLE_FOTMOB=${ENABLE_FOTMOB:-false}
|
||||
volumes:
|
||||
- cup26-data:/data
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
mem_limit: 384m
|
||||
@@ -33,6 +37,9 @@ services:
|
||||
networks:
|
||||
- proxy
|
||||
|
||||
volumes:
|
||||
cup26-data:
|
||||
|
||||
networks:
|
||||
proxy:
|
||||
external: true
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { mkdirSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
// Persistence via Node's built-in SQLite (no native deps). One file on a Docker
|
||||
// volume holds the response cache, source health, fixture↔provider id mapping,
|
||||
// per-fixture enrichment, historical player goals, and ingest logs.
|
||||
|
||||
const DATA_DIR = path.resolve(process.env.DATA_DIR ?? path.join(process.cwd(), 'data'));
|
||||
|
||||
const SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS kv_cache (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
fetched_at INTEGER NOT NULL,
|
||||
ttl INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_health (
|
||||
source TEXT PRIMARY KEY,
|
||||
last_ok INTEGER,
|
||||
last_err INTEGER,
|
||||
consec_fail INTEGER NOT NULL DEFAULT 0,
|
||||
open_until INTEGER NOT NULL DEFAULT 0,
|
||||
note TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_map (
|
||||
fixture_num INTEGER NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
event_id TEXT NOT NULL,
|
||||
PRIMARY KEY (fixture_num, source)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS match_ext (
|
||||
fixture_num INTEGER PRIMARY KEY,
|
||||
json TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS injuries (
|
||||
team TEXT NOT NULL,
|
||||
player TEXT NOT NULL,
|
||||
status TEXT,
|
||||
reason TEXT,
|
||||
return_date TEXT,
|
||||
updated_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (team, player)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS squad (
|
||||
team TEXT NOT NULL,
|
||||
player TEXT NOT NULL,
|
||||
position TEXT,
|
||||
number INTEGER,
|
||||
market_value INTEGER,
|
||||
age INTEGER,
|
||||
updated_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (team, player)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS goalscorers (
|
||||
date TEXT NOT NULL,
|
||||
home TEXT NOT NULL,
|
||||
away TEXT NOT NULL,
|
||||
team TEXT NOT NULL,
|
||||
scorer TEXT,
|
||||
minute INTEGER,
|
||||
own_goal INTEGER NOT NULL DEFAULT 0,
|
||||
penalty INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_goalscorers_scorer ON goalscorers(scorer);
|
||||
CREATE INDEX IF NOT EXISTS idx_goalscorers_team ON goalscorers(team);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ingest_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source TEXT NOT NULL,
|
||||
job TEXT NOT NULL,
|
||||
ok INTEGER NOT NULL,
|
||||
ms INTEGER,
|
||||
note TEXT,
|
||||
at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ingest_log_at ON ingest_log(at);
|
||||
`;
|
||||
|
||||
let _db: DatabaseSync | null = null;
|
||||
|
||||
export function db(): DatabaseSync {
|
||||
if (_db) return _db;
|
||||
mkdirSync(DATA_DIR, { recursive: true });
|
||||
const file = path.join(DATA_DIR, 'cup26.db');
|
||||
const d = new DatabaseSync(file);
|
||||
d.exec('PRAGMA journal_mode = WAL;');
|
||||
d.exec('PRAGMA busy_timeout = 4000;');
|
||||
d.exec(SCHEMA);
|
||||
_db = d;
|
||||
console.log(`[db] sqlite ready → ${file}`);
|
||||
return d;
|
||||
}
|
||||
|
||||
// ---- response cache ----
|
||||
export function cacheGet(key: string): { value: string; fresh: boolean } | null {
|
||||
const row = db().prepare('SELECT value, fetched_at, ttl FROM kv_cache WHERE key = ?').get(key) as
|
||||
| { value: string; fetched_at: number; ttl: number }
|
||||
| undefined;
|
||||
if (!row) return null;
|
||||
return { value: row.value, fresh: Date.now() - row.fetched_at < row.ttl };
|
||||
}
|
||||
|
||||
export function cacheSet(key: string, value: string, ttl: number): void {
|
||||
db()
|
||||
.prepare('INSERT OR REPLACE INTO kv_cache (key, value, fetched_at, ttl) VALUES (?, ?, ?, ?)')
|
||||
.run(key, value, Date.now(), ttl);
|
||||
}
|
||||
|
||||
// ---- source health / circuit breaker ----
|
||||
export interface SourceHealthRow {
|
||||
source: string;
|
||||
last_ok: number | null;
|
||||
last_err: number | null;
|
||||
consec_fail: number;
|
||||
open_until: number;
|
||||
note: string | null;
|
||||
}
|
||||
|
||||
export function healthGet(source: string): SourceHealthRow | null {
|
||||
return (db().prepare('SELECT * FROM source_health WHERE source = ?').get(source) as SourceHealthRow | undefined) ?? null;
|
||||
}
|
||||
|
||||
export function healthAll(): SourceHealthRow[] {
|
||||
return db().prepare('SELECT * FROM source_health ORDER BY source').all() as unknown as SourceHealthRow[];
|
||||
}
|
||||
|
||||
export function healthRecord(source: string, ok: boolean, note?: string, openMs?: number): void {
|
||||
const now = Date.now();
|
||||
const cur = healthGet(source);
|
||||
const consec = ok ? 0 : (cur?.consec_fail ?? 0) + 1;
|
||||
const openUntil = ok ? 0 : openMs ? now + openMs : (cur?.open_until ?? 0);
|
||||
db()
|
||||
.prepare(
|
||||
`INSERT INTO source_health (source, last_ok, last_err, consec_fail, open_until, note)
|
||||
VALUES (@source, @last_ok, @last_err, @consec, @open_until, @note)
|
||||
ON CONFLICT(source) DO UPDATE SET
|
||||
last_ok=COALESCE(@last_ok, source_health.last_ok),
|
||||
last_err=COALESCE(@last_err, source_health.last_err),
|
||||
consec_fail=@consec, open_until=@open_until, note=@note`,
|
||||
)
|
||||
.run({
|
||||
source,
|
||||
last_ok: ok ? now : null,
|
||||
last_err: ok ? null : now,
|
||||
consec: consec,
|
||||
open_until: openUntil,
|
||||
note: note ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
export function logIngest(source: string, job: string, ok: boolean, ms: number, note?: string): void {
|
||||
db().prepare('INSERT INTO ingest_log (source, job, ok, ms, note, at) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(source, job, ok ? 1 : 0, Math.round(ms), note ?? null, Date.now());
|
||||
}
|
||||
|
||||
// ---- fixture ↔ provider event-id mapping ----
|
||||
export function setSourceMap(fixtureNum: number, source: string, eventId: string): void {
|
||||
db().prepare('INSERT OR REPLACE INTO source_map (fixture_num, source, event_id) VALUES (?, ?, ?)')
|
||||
.run(fixtureNum, source, eventId);
|
||||
}
|
||||
|
||||
export function getSourceMap(fixtureNum: number, source: string): string | null {
|
||||
const row = db().prepare('SELECT event_id FROM source_map WHERE fixture_num = ? AND source = ?').get(fixtureNum, source) as
|
||||
| { event_id: string } | undefined;
|
||||
return row?.event_id ?? null;
|
||||
}
|
||||
|
||||
// ---- per-fixture enrichment blob ----
|
||||
export function setMatchExt(fixtureNum: number, json: unknown): void {
|
||||
db().prepare('INSERT OR REPLACE INTO match_ext (fixture_num, json, updated_at) VALUES (?, ?, ?)')
|
||||
.run(fixtureNum, JSON.stringify(json), Date.now());
|
||||
}
|
||||
|
||||
export function getMatchExt<T = unknown>(fixtureNum: number): { data: T; updatedAt: number } | null {
|
||||
const row = db().prepare('SELECT json, updated_at FROM match_ext WHERE fixture_num = ?').get(fixtureNum) as
|
||||
| { json: string; updated_at: number } | undefined;
|
||||
if (!row) return null;
|
||||
return { data: JSON.parse(row.json) as T, updatedAt: row.updated_at };
|
||||
}
|
||||
+7
-4
@@ -6,8 +6,9 @@ import fastifyWebsocket from '@fastify/websocket';
|
||||
import fastifyStatic from '@fastify/static';
|
||||
import type { WebSocket } from 'ws';
|
||||
import { TournamentState } from './tournament';
|
||||
import { startLiveLoop } from './liveLoop';
|
||||
import { startScheduler } from './ingest/scheduler';
|
||||
import { ModelEngine } from './model';
|
||||
import { db, healthAll } from './db/db';
|
||||
import type { MatchStatus, ServerMessage } from '../../src/lib/types';
|
||||
|
||||
const PORT = Number(process.env.PORT ?? 8787);
|
||||
@@ -27,6 +28,7 @@ export function buildServer() {
|
||||
catch { const err = new Error('Invalid JSON') as Error & { statusCode?: number }; err.statusCode = 400; done(err, undefined); }
|
||||
});
|
||||
|
||||
db(); // open + migrate SQLite before anything reads it
|
||||
const state = new TournamentState(STATIC_DIR);
|
||||
const model = new ModelEngine(STATIC_DIR);
|
||||
model.recompute(state.allFixtures());
|
||||
@@ -52,6 +54,7 @@ export function buildServer() {
|
||||
// ---- REST: snapshot + predictions (initial load + a no-WS fallback) ----
|
||||
app.get('/api/snapshot', async () => state.snapshot());
|
||||
app.get('/api/predictions', async () => model.current());
|
||||
app.get('/api/sources/health', async () => ({ sources: healthAll() }));
|
||||
app.get('/healthz', async () => ({ ok: true }));
|
||||
|
||||
// ---- dev-only: inject a score to exercise the live → standings → WS loop ----
|
||||
@@ -102,9 +105,9 @@ export function buildServer() {
|
||||
});
|
||||
});
|
||||
|
||||
// ---- live polling loop (no-op unless a source is configured) ----
|
||||
const stopLive = startLiveLoop(state, () => broadcast());
|
||||
app.addHook('onClose', async () => stopLive());
|
||||
// ---- ingestion scheduler (ESPN primary; football-data / SofaScore fallback) ----
|
||||
const stopIngest = startScheduler(state, () => broadcast());
|
||||
app.addHook('onClose', async () => stopIngest());
|
||||
|
||||
// ---- static SPA (prod) with history fallback ----
|
||||
if (existsSync(STATIC_DIR)) {
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { normalizeScoreboard, normalizeSummary, type RawSummary } from './espnNormalize';
|
||||
|
||||
describe('normalizeScoreboard', () => {
|
||||
it('normalizes events, canonicalizes names, maps status + scores', () => {
|
||||
const out = normalizeScoreboard({
|
||||
events: [
|
||||
{
|
||||
id: '760414',
|
||||
date: '2026-06-12T02:00Z',
|
||||
name: 'Czechia at South Korea',
|
||||
status: { type: { state: 'in' }, displayClock: "67'", period: 2 },
|
||||
competitions: [
|
||||
{
|
||||
competitors: [
|
||||
{ homeAway: 'home', team: { id: '1', displayName: 'South Korea' }, score: '1' },
|
||||
{ homeAway: 'away', team: { id: '2', displayName: 'Czechia' }, score: '2' },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: '760415',
|
||||
date: '2026-06-11T19:00Z',
|
||||
name: 'South Africa at Mexico',
|
||||
status: { type: { state: 'pre' } },
|
||||
competitions: [
|
||||
{
|
||||
competitors: [
|
||||
{ homeAway: 'home', team: { id: '203', displayName: 'Mexico' } },
|
||||
{ homeAway: 'away', team: { id: '467', displayName: 'South Africa' } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(out).toHaveLength(2);
|
||||
const live = out[0]!;
|
||||
expect(live.eventId).toBe('760414');
|
||||
expect(live.home).toBe('South Korea');
|
||||
expect(live.away).toBe('Czech Republic'); // Czechia → canonical
|
||||
expect(live.status).toBe('live');
|
||||
expect(live.homeScore).toBe(1);
|
||||
expect(live.awayScore).toBe(2);
|
||||
expect(live.minute).toBe(67);
|
||||
|
||||
const sched = out[1]!;
|
||||
expect(sched.status).toBe('scheduled');
|
||||
expect(sched.homeScore).toBeNull();
|
||||
expect(sched.minute).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeSummary', () => {
|
||||
it('extracts venue, teams, H2H, form, lineups and stats', () => {
|
||||
const raw: RawSummary = {
|
||||
gameInfo: { venue: { fullName: 'Estadio Banorte' } },
|
||||
header: {
|
||||
competitions: [
|
||||
{ competitors: [
|
||||
{ id: '203', team: { displayName: 'Mexico' }, homeAway: 'home' },
|
||||
{ id: '467', team: { displayName: 'South Africa' }, homeAway: 'away' },
|
||||
] },
|
||||
],
|
||||
},
|
||||
headToHeadGames: [
|
||||
{ events: [
|
||||
{ gameDate: '2010-06-11T14:00Z', homeTeamId: '203', awayTeamId: '467', homeTeamScore: '1', awayTeamScore: '1' },
|
||||
] },
|
||||
],
|
||||
boxscore: {
|
||||
form: [
|
||||
{ team: { id: '203' }, events: [{ gameResult: 'W' }, { gameResult: 'd' }] },
|
||||
],
|
||||
teams: [
|
||||
{ team: { id: '203' }, statistics: [{ name: 'possessionPct', displayValue: '58%' }] },
|
||||
],
|
||||
},
|
||||
rosters: [
|
||||
{ team: { id: '203' }, roster: [{ athlete: { displayName: 'A. Player' }, position: { abbreviation: 'F' }, starter: true, jersey: '9' }] },
|
||||
],
|
||||
};
|
||||
const s = normalizeSummary(raw);
|
||||
expect(s.venue).toBe('Estadio Banorte');
|
||||
expect(s.teams.map((t) => t.name)).toEqual(['Mexico', 'South Africa']);
|
||||
expect(s.h2h[0]).toMatchObject({ homeScore: 1, awayScore: 1, homeTeamId: '203' });
|
||||
expect(s.form['203']).toEqual(['W', 'D']); // uppercased
|
||||
expect(s.lineups['203']![0]).toMatchObject({ name: 'A. Player', position: 'F', starter: true, number: 9 });
|
||||
expect(s.stats['203']!.possessionPct).toBe('58%');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,34 @@
|
||||
import { cachedFetch } from './fetcher';
|
||||
import {
|
||||
normalizeScoreboard,
|
||||
normalizeSummary,
|
||||
type EspnEvent,
|
||||
type EspnMatch,
|
||||
type EspnSummary,
|
||||
type RawSummary,
|
||||
} from './espnNormalize';
|
||||
|
||||
// ESPN's hidden site API — datacenter-friendly (works from the VPS where
|
||||
// SofaScore 403s), no auth, covers the World Cup (`fifa.world`). The scoreboard
|
||||
// drives live scores; the summary endpoint is a rich pre-match/live blob.
|
||||
const BASE = 'https://site.api.espn.com/apis/site/v2/sports/soccer/fifa.world';
|
||||
const SOURCE = 'espn';
|
||||
|
||||
export type { EspnMatch, EspnSummary } from './espnNormalize';
|
||||
|
||||
export async function fetchEspnScoreboard(dateYYYYMMDD?: string): Promise<EspnMatch[]> {
|
||||
const url = `${BASE}/scoreboard${dateYYYYMMDD ? `?dates=${dateYYYYMMDD}` : ''}`;
|
||||
const body = await cachedFetch<{ events?: EspnEvent[] }>(SOURCE, `espn:sb:${dateYYYYMMDD ?? 'now'}`, url, {
|
||||
ttl: 20_000,
|
||||
minInterval: 3_000,
|
||||
});
|
||||
return normalizeScoreboard(body);
|
||||
}
|
||||
|
||||
export async function fetchEspnSummary(eventId: string): Promise<EspnSummary> {
|
||||
const raw = await cachedFetch<RawSummary>(SOURCE, `espn:summary:${eventId}`, `${BASE}/summary?event=${eventId}`, {
|
||||
ttl: 5 * 60_000,
|
||||
minInterval: 2_000,
|
||||
});
|
||||
return normalizeSummary(raw);
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import { canonicalTeam } from '../../../src/lib/teams';
|
||||
import type { LiveMatch } from '../tournament';
|
||||
import type { MatchStatus } from '../../../src/lib/types';
|
||||
|
||||
// Pure ESPN normalizers — no network or DB, so they're unit-testable in
|
||||
// isolation (and don't drag node:sqlite into the Vite test graph).
|
||||
|
||||
interface EspnCompetitor {
|
||||
homeAway: 'home' | 'away';
|
||||
team: { id: string; displayName: string; name?: string; abbreviation?: string };
|
||||
score?: string;
|
||||
}
|
||||
interface EspnStatus { type: { state: 'pre' | 'in' | 'post'; completed?: boolean; name?: string }; displayClock?: string; period?: number }
|
||||
export interface EspnEvent {
|
||||
id: string;
|
||||
date: string;
|
||||
name: string;
|
||||
competitions: { competitors: EspnCompetitor[]; venue?: { fullName?: string } }[];
|
||||
status: EspnStatus;
|
||||
}
|
||||
|
||||
export interface EspnMatch extends LiveMatch {
|
||||
eventId: string;
|
||||
}
|
||||
|
||||
function mapStatus(s: EspnStatus): MatchStatus {
|
||||
if (s.type.state === 'in') return 'live';
|
||||
if (s.type.state === 'post') return 'finished';
|
||||
return 'scheduled';
|
||||
}
|
||||
|
||||
function parseMinute(s: EspnStatus): number | null {
|
||||
if (s.type.state !== 'in') return null;
|
||||
const m = s.displayClock?.match(/(\d+)/);
|
||||
return m ? Number(m[1]) : (s.period ? (s.period - 1) * 45 : null);
|
||||
}
|
||||
|
||||
/** Scoreboard body → matches with ESPN event ids. */
|
||||
export function normalizeScoreboard(body: { events?: EspnEvent[] }): EspnMatch[] {
|
||||
const out: EspnMatch[] = [];
|
||||
for (const e of body.events ?? []) {
|
||||
const comp = e.competitions[0];
|
||||
if (!comp) continue;
|
||||
const home = comp.competitors.find((c) => c.homeAway === 'home');
|
||||
const away = comp.competitors.find((c) => c.homeAway === 'away');
|
||||
if (!home || !away) continue;
|
||||
out.push({
|
||||
eventId: e.id,
|
||||
home: canonicalTeam(home.team.displayName),
|
||||
away: canonicalTeam(away.team.displayName),
|
||||
group: null,
|
||||
stage: 'group',
|
||||
kickoff: e.date,
|
||||
homeScore: home.score != null && home.score !== '' ? Number(home.score) : null,
|
||||
awayScore: away.score != null && away.score !== '' ? Number(away.score) : null,
|
||||
status: mapStatus(e.status),
|
||||
minute: parseMinute(e.status),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---- summary (rich enrichment) ----
|
||||
export interface EspnH2HGame { date: string; homeTeamId: string; awayTeamId: string; homeScore: number; awayScore: number }
|
||||
export interface EspnTeamRef { id: string; name: string; homeAway: 'home' | 'away' }
|
||||
export interface EspnLineupPlayer { name: string; position: string | null; starter: boolean; number: number | null }
|
||||
export interface EspnSummary {
|
||||
fetchedAt: number;
|
||||
venue: string | null;
|
||||
teams: EspnTeamRef[];
|
||||
h2h: EspnH2HGame[];
|
||||
/** Recent results per team id, most recent first: 'W' | 'D' | 'L'. */
|
||||
form: Record<string, string[]>;
|
||||
lineups: Record<string, EspnLineupPlayer[]>;
|
||||
/** Team match stats (possession, shots, …) once the game is live/finished. */
|
||||
stats: Record<string, Record<string, string>>;
|
||||
}
|
||||
|
||||
export interface RawSummary {
|
||||
gameInfo?: { venue?: { fullName?: string } };
|
||||
header?: { competitions?: { competitors?: { id: string; team?: { displayName?: string }; homeAway: 'home' | 'away' }[] }[] };
|
||||
headToHeadGames?: { events?: { gameDate: string; homeTeamId: string; awayTeamId: string; homeTeamScore: string; awayTeamScore: string }[] }[];
|
||||
boxscore?: {
|
||||
form?: { team?: { id: string }; events?: { gameResult?: string }[] }[];
|
||||
teams?: { team?: { id: string }; statistics?: { name: string; displayValue: string }[] }[];
|
||||
};
|
||||
rosters?: { team?: { id: string }; roster?: { athlete?: { displayName?: string }; position?: { abbreviation?: string }; starter?: boolean; jersey?: string }[] }[];
|
||||
}
|
||||
|
||||
export function normalizeSummary(raw: RawSummary): EspnSummary {
|
||||
const teams: EspnTeamRef[] = (raw.header?.competitions?.[0]?.competitors ?? []).map((c) => ({
|
||||
id: c.id,
|
||||
name: canonicalTeam(c.team?.displayName ?? ''),
|
||||
homeAway: c.homeAway,
|
||||
}));
|
||||
|
||||
const h2h: EspnH2HGame[] = (raw.headToHeadGames?.[0]?.events ?? []).map((g) => ({
|
||||
date: g.gameDate,
|
||||
homeTeamId: g.homeTeamId,
|
||||
awayTeamId: g.awayTeamId,
|
||||
homeScore: Number(g.homeTeamScore),
|
||||
awayScore: Number(g.awayTeamScore),
|
||||
}));
|
||||
|
||||
const form: Record<string, string[]> = {};
|
||||
for (const f of raw.boxscore?.form ?? []) {
|
||||
const id = f.team?.id;
|
||||
if (!id) continue;
|
||||
form[id] = (f.events ?? []).map((e) => (e.gameResult ?? '').toUpperCase()).filter(Boolean);
|
||||
}
|
||||
|
||||
const lineups: Record<string, EspnLineupPlayer[]> = {};
|
||||
for (const r of raw.rosters ?? []) {
|
||||
const id = r.team?.id;
|
||||
if (!id) continue;
|
||||
lineups[id] = (r.roster ?? []).map((p) => ({
|
||||
name: p.athlete?.displayName ?? '',
|
||||
position: p.position?.abbreviation ?? null,
|
||||
starter: !!p.starter,
|
||||
number: p.jersey ? Number(p.jersey) : null,
|
||||
}));
|
||||
}
|
||||
|
||||
const stats: Record<string, Record<string, string>> = {};
|
||||
for (const t of raw.boxscore?.teams ?? []) {
|
||||
const id = t.team?.id;
|
||||
if (!id) continue;
|
||||
stats[id] = Object.fromEntries((t.statistics ?? []).map((s) => [s.name, s.displayValue]));
|
||||
}
|
||||
|
||||
return { fetchedAt: Date.now(), venue: raw.gameInfo?.venue?.fullName ?? null, teams, h2h, form, lineups, stats };
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import { cacheGet, cacheSet, healthGet, healthRecord } from '../db/db';
|
||||
|
||||
// One resilient fetch path for every source: DB-cached, rate-limited, and
|
||||
// circuit-broken. A blocked source (e.g. SofaScore 403 from the VPS) trips its
|
||||
// breaker and is skipped; callers transparently get stale cache or a thrown
|
||||
// error they already handle — the UI, which reads the DB, never blocks.
|
||||
|
||||
const CIRCUIT_THRESHOLD = 4; // consecutive failures before opening
|
||||
const CIRCUIT_COOLDOWN = 10 * 60_000; // base cooldown, escalates with failures
|
||||
|
||||
const lastCall = new Map<string, number>();
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
export function browserHeaders(extra: Record<string, string> = {}): Record<string, string> {
|
||||
return {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
Accept: 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
export interface FetchOpts {
|
||||
/** Cache freshness window (ms). */
|
||||
ttl: number;
|
||||
/** Minimum interval between calls to this source (ms). */
|
||||
minInterval?: number;
|
||||
headers?: Record<string, string>;
|
||||
timeoutMs?: number;
|
||||
/** Parse as text instead of JSON. */
|
||||
text?: boolean;
|
||||
}
|
||||
|
||||
export function circuitOpen(source: string): boolean {
|
||||
const h = healthGet(source);
|
||||
return !!h && h.open_until > Date.now();
|
||||
}
|
||||
|
||||
async function rateLimit(source: string, minInterval: number): Promise<void> {
|
||||
if (!minInterval) return;
|
||||
const last = lastCall.get(source) ?? 0;
|
||||
const wait = minInterval - (Date.now() - last);
|
||||
if (wait > 0) await sleep(wait + Math.random() * 250); // jitter
|
||||
lastCall.set(source, Date.now());
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch JSON (or text) for `source` at `url`, keyed in the cache by `key`.
|
||||
* Returns fresh cache when valid, otherwise fetches; on failure returns stale
|
||||
* cache if present, else throws.
|
||||
*/
|
||||
export async function cachedFetch<T = unknown>(
|
||||
source: string,
|
||||
key: string,
|
||||
url: string,
|
||||
opts: FetchOpts,
|
||||
): Promise<T> {
|
||||
const cached = cacheGet(key);
|
||||
if (cached?.fresh) return (opts.text ? cached.value : JSON.parse(cached.value)) as T;
|
||||
|
||||
if (circuitOpen(source)) {
|
||||
if (cached) return (opts.text ? cached.value : JSON.parse(cached.value)) as T;
|
||||
throw new Error(`${source}: circuit open`);
|
||||
}
|
||||
|
||||
await rateLimit(source, opts.minInterval ?? 0);
|
||||
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
headers: browserHeaders(opts.headers),
|
||||
signal: AbortSignal.timeout(opts.timeoutMs ?? 12_000),
|
||||
});
|
||||
if (!res.ok) throw new Error(`${res.status}`);
|
||||
const body = await res.text();
|
||||
cacheSet(key, body, opts.ttl);
|
||||
healthRecord(source, true);
|
||||
return (opts.text ? body : JSON.parse(body)) as T;
|
||||
} catch (e) {
|
||||
const note = e instanceof Error ? e.message : String(e);
|
||||
const consec = (healthGet(source)?.consec_fail ?? 0) + 1;
|
||||
const openMs = consec >= CIRCUIT_THRESHOLD ? CIRCUIT_COOLDOWN * Math.min(6, consec - CIRCUIT_THRESHOLD + 1) : undefined;
|
||||
healthRecord(source, false, note, openMs);
|
||||
if (cached) return (opts.text ? cached.value : JSON.parse(cached.value)) as T; // serve stale
|
||||
throw new Error(`${source} fetch failed: ${note}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import { TournamentState, type LiveMatch, type Source } from '../tournament';
|
||||
import { fetchEspnScoreboard, fetchEspnSummary } from './espn';
|
||||
import { fetchFootballData } from '../footballData';
|
||||
import { fetchSofascore } from '../sofascore';
|
||||
import { setSourceMap, getSourceMap, setMatchExt, logIngest } from '../db/db';
|
||||
import type { Fixture } from '../../../src/lib/types';
|
||||
|
||||
// The ingestion orchestrator. Replaces the old live loop: ESPN is the primary
|
||||
// live + enrichment source (works from the VPS); football-data.org and SofaScore
|
||||
// are fallbacks. Everything persists to SQLite; the UI reads the DB, so a blocked
|
||||
// source only means staler data.
|
||||
|
||||
const LIVE_WINDOW_MS = 3 * 60 * 60 * 1000;
|
||||
const POLL_LIVE_MS = 20_000;
|
||||
const POLL_IDLE_MS = 10 * 60_000;
|
||||
const ENRICH_MS = 20 * 60_000;
|
||||
|
||||
function dateUTC(iso: string): string {
|
||||
const d = new Date(iso);
|
||||
return `${d.getUTCFullYear()}${String(d.getUTCMonth() + 1).padStart(2, '0')}${String(d.getUTCDate()).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
export function startScheduler(state: TournamentState, onChange: () => void): () => void {
|
||||
const token = process.env.FOOTBALL_DATA_TOKEN?.trim();
|
||||
const sofa = process.env.ENABLE_SOFASCORE === 'true';
|
||||
let stopped = false;
|
||||
let liveTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let enrichTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
|
||||
console.log(`[ingest] scheduler — espn:on football-data:${token ? 'on' : 'off'} sofascore:${sofa ? 'on' : 'off'}`);
|
||||
|
||||
// ---- one-time: map every fixture to its ESPN event id (across all dates) ----
|
||||
const mapAllFixtures = async (): Promise<void> => {
|
||||
const dates = [...new Set(state.allFixtures().map((f) => dateUTC(f.kickoff)))];
|
||||
let mapped = 0;
|
||||
for (const date of dates) {
|
||||
if (stopped) return;
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
const evs = await fetchEspnScoreboard(date);
|
||||
for (const m of evs) {
|
||||
const num = state.matchFixtureNum(m.home, m.away, 'group', m.kickoff);
|
||||
if (num) { setSourceMap(num, 'espn', m.eventId); mapped++; }
|
||||
}
|
||||
logIngest('espn', 'map', true, Date.now() - t0, `${date}: ${evs.length} events`);
|
||||
} catch (e) {
|
||||
logIngest('espn', 'map', false, Date.now() - t0, e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
}
|
||||
console.log(`[ingest] mapped ${mapped} fixtures to ESPN event ids`);
|
||||
};
|
||||
|
||||
// ---- live scoreboard ----
|
||||
const liveTick = async (): Promise<void> => {
|
||||
const t0 = Date.now();
|
||||
let matches: LiveMatch[] = [];
|
||||
let source: Source = 'seed';
|
||||
try {
|
||||
const espn = await fetchEspnScoreboard();
|
||||
matches = espn;
|
||||
source = 'espn';
|
||||
for (const m of espn) {
|
||||
const num = state.matchFixtureNum(m.home, m.away, 'group', m.kickoff);
|
||||
if (num) setSourceMap(num, 'espn', m.eventId);
|
||||
}
|
||||
logIngest('espn', 'scoreboard', true, Date.now() - t0, `${espn.length} events`);
|
||||
} catch (e) {
|
||||
logIngest('espn', 'scoreboard', false, Date.now() - t0, e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
if (!matches.length && token) {
|
||||
try { matches = await fetchFootballData(token); source = 'football-data'; } catch { /* logged via health */ }
|
||||
}
|
||||
if (!matches.length && sofa) {
|
||||
try { matches = await fetchSofascore(); source = 'sofascore'; } catch { /* expected block */ }
|
||||
}
|
||||
if (matches.length) {
|
||||
const changed = state.mergeLive(matches, source);
|
||||
if (changed.length) { console.log(`[ingest] ${changed.length} fixture(s) updated via ${source}`); onChange(); }
|
||||
}
|
||||
};
|
||||
|
||||
// ---- enrichment: ESPN summary for imminent/live fixtures ----
|
||||
const enrichTick = async (): Promise<void> => {
|
||||
const now = Date.now();
|
||||
const relevant = state.allFixtures().filter((f) => {
|
||||
const k = new Date(f.kickoff).getTime();
|
||||
return f.status === 'live' || (k > now - LIVE_WINDOW_MS && k < now + 3 * 24 * 60 * 60 * 1000);
|
||||
});
|
||||
for (const f of relevant) {
|
||||
if (stopped) return;
|
||||
const eid = getSourceMap(f.num, 'espn');
|
||||
if (!eid) continue;
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
const summary = await fetchEspnSummary(eid);
|
||||
setMatchExt(f.num, summary);
|
||||
logIngest('espn', 'summary', true, Date.now() - t0, `M${f.num}`);
|
||||
} catch (e) {
|
||||
logIngest('espn', 'summary', false, Date.now() - t0, e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const scheduleLive = (): void => {
|
||||
if (stopped) return;
|
||||
const delay = state.hasLiveActivity(LIVE_WINDOW_MS) ? POLL_LIVE_MS : POLL_IDLE_MS;
|
||||
liveTimer = setTimeout(async () => { await liveTick().catch(() => {}); scheduleLive(); }, delay);
|
||||
};
|
||||
const scheduleEnrich = (): void => {
|
||||
if (stopped) return;
|
||||
enrichTimer = setTimeout(async () => { await enrichTick().catch(() => {}); scheduleEnrich(); }, ENRICH_MS);
|
||||
};
|
||||
|
||||
// boot: map ids, then run the loops
|
||||
void (async () => {
|
||||
await mapAllFixtures().catch(() => {});
|
||||
await liveTick().catch(() => {});
|
||||
await enrichTick().catch(() => {});
|
||||
scheduleLive();
|
||||
scheduleEnrich();
|
||||
})();
|
||||
|
||||
return () => {
|
||||
stopped = true;
|
||||
if (liveTimer) clearTimeout(liveTimer);
|
||||
if (enrichTimer) clearTimeout(enrichTimer);
|
||||
};
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
import { TournamentState, type LiveMatch, type Source } from './tournament';
|
||||
import { fetchFootballData } from './footballData';
|
||||
import { fetchSofascore } from './sofascore';
|
||||
import type { Fixture } from '../../src/lib/types';
|
||||
|
||||
const LIVE_WINDOW_MS = 3 * 60 * 60 * 1000; // ±3h around a kickoff = "match window"
|
||||
const POLL_LIVE_MS = 60_000; // football-data free tier is 10 req/min — 1/min is safe
|
||||
const POLL_IDLE_MS = 10 * 60_000;
|
||||
|
||||
/**
|
||||
* Poll the live sources and merge results onto the tournament state. SofaScore
|
||||
* (if enabled) is tried first for freshness; football-data.org is the fallback
|
||||
* and the reliable backbone. Returns a stop function.
|
||||
*/
|
||||
export function startLiveLoop(
|
||||
state: TournamentState,
|
||||
onChange: (changed: Fixture[]) => void,
|
||||
): () => void {
|
||||
const token = process.env.FOOTBALL_DATA_TOKEN?.trim();
|
||||
const sofa = process.env.ENABLE_SOFASCORE === 'true';
|
||||
|
||||
if (!token && !sofa) {
|
||||
console.log('[live] seed-only (set FOOTBALL_DATA_TOKEN and/or ENABLE_SOFASCORE=true to enable)');
|
||||
return () => {};
|
||||
}
|
||||
console.log(`[live] enabled — football-data:${token ? 'on' : 'off'} sofascore:${sofa ? 'on' : 'off'}`);
|
||||
|
||||
let stopped = false;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
|
||||
const tick = async (): Promise<void> => {
|
||||
let matches: LiveMatch[] = [];
|
||||
let source: Source = 'seed';
|
||||
|
||||
if (sofa) {
|
||||
try {
|
||||
matches = await fetchSofascore();
|
||||
if (matches.length) source = 'sofascore';
|
||||
} catch (e) {
|
||||
console.warn('[live] sofascore failed:', e instanceof Error ? e.message : e);
|
||||
}
|
||||
}
|
||||
if (!matches.length && token) {
|
||||
try {
|
||||
matches = await fetchFootballData(token);
|
||||
source = 'football-data';
|
||||
} catch (e) {
|
||||
console.warn('[live] football-data failed:', e instanceof Error ? e.message : e);
|
||||
}
|
||||
}
|
||||
|
||||
if (matches.length) {
|
||||
const changed = state.mergeLive(matches, source);
|
||||
if (changed.length) {
|
||||
console.log(`[live] ${changed.length} fixture(s) updated via ${source}`);
|
||||
onChange(changed);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const schedule = (): void => {
|
||||
if (stopped) return;
|
||||
const delay = state.hasLiveActivity(LIVE_WINDOW_MS) ? POLL_LIVE_MS : POLL_IDLE_MS;
|
||||
timer = setTimeout(async () => {
|
||||
await tick().catch((e) => console.warn('[live] tick error:', e));
|
||||
schedule();
|
||||
}, delay);
|
||||
};
|
||||
|
||||
// Kick off immediately, then settle into the dynamic schedule.
|
||||
void tick().catch((e) => console.warn('[live] initial tick error:', e)).finally(schedule);
|
||||
|
||||
return () => {
|
||||
stopped = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
}
|
||||
@@ -68,6 +68,16 @@ export class TournamentState {
|
||||
return this.fixtures;
|
||||
}
|
||||
|
||||
/** Which fixture number does a (home, away) pairing map to? Used by the
|
||||
* ingestion layer to key provider event ids against our fixtures. */
|
||||
matchFixtureNum(home: string, away: string, stage: Fixture['stage'] = 'group', kickoff: string | null = null): number | null {
|
||||
const f = this.findFixture({
|
||||
home, away, group: null, stage, kickoff,
|
||||
homeScore: null, awayScore: null, status: 'scheduled', minute: null,
|
||||
});
|
||||
return f?.num ?? null;
|
||||
}
|
||||
|
||||
/** True if any match is live or kicks off within `windowMs` of now — used to
|
||||
* poll the APIs often during match windows and rarely when nothing is on. */
|
||||
hasLiveActivity(windowMs: number): boolean {
|
||||
|
||||
+1
-1
@@ -63,7 +63,7 @@ export interface Snapshot {
|
||||
/** When the live layer last refreshed (ISO), or null if seed-only. */
|
||||
updatedAt: string | null;
|
||||
/** Which live source produced the current scores. */
|
||||
source: 'seed' | 'football-data' | 'sofascore';
|
||||
source: 'seed' | 'espn' | 'football-data' | 'sofascore';
|
||||
fixtures: Fixture[];
|
||||
/** Group letter → ranked standings. */
|
||||
tables: Record<string, StandingRow[]>;
|
||||
|
||||
Reference in New Issue
Block a user