From d887664fce41e31b4d4594673ec81887d0a194be Mon Sep 17 00:00:00 2001 From: Nils Briggen Date: Thu, 11 Jun 2026 17:03:15 +0200 Subject: [PATCH] =?UTF-8?q?v3=20Phase=20A=20(urgent):=20bookmaker=20odds?= =?UTF-8?q?=20capture=20=E2=80=94=20the=20Model-vs-Market=20benchmark?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - docs/V3-GIGA-PLAN.md: the v3 plan (data lake to 7+GB, GIGA ensemble, scoreboard, UI clarity) with the honest reframe: perfect accuracy = perfect calibration; 'beat the betting sites' = a live public head-to-head, scored with proper rules. - src/lib/odds.ts: American-ml → decimal, de-vig (normalize away the margin), overround; 3 vitest cases incl. tonight's real opening lines. - ESPN core odds adapter (server/src/ingest/espnOdds.ts): captures DraftKings 1X2 moneylines + O/U + spread per fixture; follows $ref pointer items. - odds_history table (insert-if-changed → clean line-movement history); scheduler odds sweep every 3h over a 14-day horizon, first sweep at boot; /api/odds + /api/odds/:num. Odds are a benchmark ONLY — never a model input. - Boot speed-up: fixture↔event mapping skips dates already fully mapped. - Verified: 54 fixtures captured with real moneylines locally. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 2 ++ docs/V3-GIGA-PLAN.md | 55 +++++++++++++++++++++++++++++++ server/src/db/db.ts | 56 +++++++++++++++++++++++++++++++ server/src/index.ts | 9 ++++- server/src/ingest/espnOdds.ts | 60 ++++++++++++++++++++++++++++++++++ server/src/ingest/scheduler.ts | 58 +++++++++++++++++++++++++++++--- src/lib/odds.test.ts | 28 ++++++++++++++++ src/lib/odds.ts | 32 ++++++++++++++++++ 8 files changed, 295 insertions(+), 5 deletions(-) create mode 100644 docs/V3-GIGA-PLAN.md create mode 100644 server/src/ingest/espnOdds.ts create mode 100644 src/lib/odds.test.ts create mode 100644 src/lib/odds.ts diff --git a/.gitignore b/.gitignore index 03dade4..66ae466 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ server/node_modules /data/cup26.db /data/cup26.db-wal /data/cup26.db-shm +# raw data lake (7+ GB StatsBomb corpus etc.) — processed aggregates ship, raw stays local +/data/lake/ # Generated build artifacts (reproduced by `npm run data:build`) public/data/fixtures.json public/data/ratings.json diff --git a/docs/V3-GIGA-PLAN.md b/docs/V3-GIGA-PLAN.md new file mode 100644 index 0000000..81a51c9 --- /dev/null +++ b/docs/V3-GIGA-PLAN.md @@ -0,0 +1,55 @@ +# Cup26 v3 — "GIGA": the data lake, the ensemble, the scoreboard + +## The three asks, translated honestly + +| Ask | Reality | What we build | +|---|---|---| +| **Perfect accuracy** | Impossible — match outcomes are irreducibly random. A true 60% favourite loses 40% of the time. | Perfect **calibration** (ECE → ~0.01, proven on the reliability diagram) + maximal sharpness from an ensemble. "Perfect" = our probabilities mean exactly what they say. | +| **More accurate than betting sites** | The closing line is the strongest public forecaster; nobody beats it systematically. | A **live public Model-vs-Market scoreboard**: capture DraftKings odds (free via ESPN core API) for every WC match, de-vig them, score market vs our model with RPS/Brier after each result. Target: within noise of the market; stretch: beat it on slices (draws, knockouts). Nobody else shows this publicly — win or lose, we show the gap. The model itself stays **pure** (odds are a benchmark, never an input — per the earlier decision). | +| **Literal gigabytes of data** | Verified: StatsBomb open data alone is **7.06 GB** of event JSON. | A real data lake: 7 GB StatsBomb corpus + ESPN historical sweep + odds-history capture + Transfermarkt squads/values/injuries. Raw lake lives on the dev machine; compact aggregates (tens of MB) ship to prod; a `/data` page shows live counters (bytes, events, matches, players, freshness) — "show all the data." | + +## Pillar A — The Data Lake (Phase A) + +1. **StatsBomb full corpus** (`github.com/statsbomb/open-data`, ~7 GB): clone locally, process EVERY competition (WC 2018/2022, Euros, La Liga, EPL, WSL, Copa, …) into: + - `player_profiles` — per-90 xG, xA, shots, key passes, pressures, duels, set-piece involvement for every player in the corpus; matched to 2026 squads by name. + - `team_fingerprints` — directness, press height, set-piece share, xG-by-zone (the "gaps & strengths" engine, now from real event data). + - `score_state_rates` — empirical goal rates by score state/minute (powers in-play v2). + - `shootout/penalty priors` — conversion rates (with martj42 `shootouts.csv`). +2. **ESPN historical sweep**: scoreboards + summaries for WC 2010–2022 (verified working) → fills H2H/form archive with venues, stats. +3. **ESPN odds capture** (`sports.core.api.espn.com/.../odds`, verified live): VPS scheduler snapshots odds for every 2026 fixture a few times daily + near kickoff → `odds_history` (line movement!). Benchmark only. +4. **Transfermarkt adapter** (planned in v2, now built): squads, market values, injuries/suspensions → `squad`, `injuries` tables (daily cadence). +5. **`/api/data/stats` + `/data` page**: total bytes processed, events, matches, players, per-source freshness/health. The gigabytes, visible. + +## Pillar B — Model v3: the GIGA ensemble (Phase B) + +Three independent ratings, ensembled: +1. **Elo–Dixon-Coles v2** (current model + covariates): tuned time-decay, rest days, venue geo/altitude, stage effects. +2. **Squad-value model**: Transfermarkt market values + age curves + injury deductions → goal expectations (literature: rivals Elo; low correlation with it → great ensemble partner). +3. **Player-aggregate model**: squad xG production/defence from StatsBomb player profiles (coverage-weighted; honest about sparse squads). + +Then: **log-opinion-pool ensemble** (weights tuned on 2018–2022 via the existing backtest harness) → **isotonic calibration** on held-out years → only ship what wins the bake-off. Plus: shootout model in the Monte Carlo (knockouts), in-play v2 (score-state intensity, red cards). + +Measurement: extended walk-forward backtest scores every variant (RPS/Brier/log-loss/ECE); the Methodology page shows the bake-off table. Live: the scoreboard (below) is the ultimate test. + +## Pillar C — UI: clear, clean, labelled (Phase C) + +1. **Glossary + tooltips**: every metric (xG, Elo, RPS, ECE…) gets an inline "?" → plain-language explainer; one shared `` component. +2. **Provenance chips everywhere**: "ESPN · 2 min ago", "150-year archive", "StatsBomb corpus", "model v3". +3. **`/scoreboard`** — Model vs Market: per-match cards (our probs vs de-vigged DraftKings), running RPS/Brier totals, "who was closer" badges. The headline feature. +4. **`/data`** — the lake, visible: counters, coverage, freshness, browsable source tables. +5. **Squad views** on team pages: players with club, age, market value, injury status, corpus stats where available. +6. **Clarity pass**: consistent units/labels, collapsible advanced sections, density/mobile polish. + +## Phasing + +- **Phase A — Lake** (StatsBomb ingest + Transfermarkt + ESPN history/odds capture + /data) ← start here; odds capture must begin ASAP (every day of line history is unrecoverable). +- **Phase B — Ensemble** (covariates, squad-value + player models, ensemble + calibration, bake-off, shootout, in-play v2). +- **Phase C — UI** (scoreboard, data explorer, glossary/tooltips, squads, clarity pass). +- **Phase D — Deploy & live validation** through the group stage; scoreboard accumulates. + +## Honest success criteria + +- Calibration: ECE ≤ 0.015 out-of-sample (already 0.01; keep it while adding sharpness). +- Sharpness: beat v2's RPS 0.171 out-of-sample (every point matters; the market sits only a little better). +- Market: publish the head-to-head; target = statistical tie; stretch = ahead on any meaningful slice after ≥48 matches. +- Data: > 7 GB processed, with the proof on `/data`. diff --git a/server/src/db/db.ts b/server/src/db/db.ts index 2f9d3c6..e576972 100644 --- a/server/src/db/db.ts +++ b/server/src/db/db.ts @@ -72,6 +72,19 @@ CREATE TABLE IF NOT EXISTS goalscorers ( CREATE INDEX IF NOT EXISTS idx_goalscorers_scorer ON goalscorers(scorer); CREATE INDEX IF NOT EXISTS idx_goalscorers_team ON goalscorers(team); +CREATE TABLE IF NOT EXISTS odds_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + fixture_num INTEGER NOT NULL, + provider TEXT NOT NULL, + captured_at INTEGER NOT NULL, + home_ml REAL, + draw_ml REAL, + away_ml REAL, + over_under REAL, + spread REAL +); +CREATE INDEX IF NOT EXISTS idx_odds_fixture ON odds_history(fixture_num, captured_at); + CREATE TABLE IF NOT EXISTS ingest_log ( id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL, @@ -173,6 +186,49 @@ export function getSourceMap(fixtureNum: number, source: string): string | null return row?.event_id ?? null; } +// ---- bookmaker odds history (benchmark only, never a model input) ---- +export interface OddsRow { + fixture_num: number; + provider: string; + captured_at: number; + home_ml: number | null; + draw_ml: number | null; + away_ml: number | null; + over_under: number | null; + spread: number | null; +} + +/** Insert a snapshot only if the lines moved since the last capture. */ +export function recordOdds(o: Omit): boolean { + const last = db() + .prepare('SELECT home_ml, draw_ml, away_ml, over_under, spread FROM odds_history WHERE fixture_num = ? AND provider = ? ORDER BY captured_at DESC LIMIT 1') + .get(o.fixture_num, o.provider) as Omit | undefined; + if ( + last && + last.home_ml === o.home_ml && last.draw_ml === o.draw_ml && last.away_ml === o.away_ml && + last.over_under === o.over_under && last.spread === o.spread + ) return false; + db() + .prepare('INSERT INTO odds_history (fixture_num, provider, captured_at, home_ml, draw_ml, away_ml, over_under, spread) VALUES (?, ?, ?, ?, ?, ?, ?, ?)') + .run(o.fixture_num, o.provider, Date.now(), o.home_ml, o.draw_ml, o.away_ml, o.over_under, o.spread); + return true; +} + +export function oddsFor(fixtureNum: number): OddsRow[] { + return db() + .prepare('SELECT fixture_num, provider, captured_at, home_ml, draw_ml, away_ml, over_under, spread FROM odds_history WHERE fixture_num = ? ORDER BY captured_at') + .all(fixtureNum) as unknown as OddsRow[]; +} + +export function latestOddsAll(): OddsRow[] { + return db() + .prepare(`SELECT o.* FROM odds_history o + JOIN (SELECT fixture_num, provider, MAX(captured_at) m FROM odds_history GROUP BY fixture_num, provider) t + ON o.fixture_num = t.fixture_num AND o.provider = t.provider AND o.captured_at = t.m + ORDER BY o.fixture_num`) + .all() as unknown as OddsRow[]; +} + // ---- per-fixture enrichment blob ---- export function setMatchExt(fixtureNum: number, json: unknown): void { db().prepare('INSERT OR REPLACE INTO match_ext (fixture_num, json, updated_at) VALUES (?, ?, ?)') diff --git a/server/src/index.ts b/server/src/index.ts index 09f2468..e8f17d8 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -9,7 +9,7 @@ import { TournamentState } from './tournament'; import { startScheduler } from './ingest/scheduler'; import { ModelEngine } from './model'; import { buildPreview, buildTeamProfile } from './preview'; -import { db, healthAll } from './db/db'; +import { db, healthAll, oddsFor, latestOddsAll } from './db/db'; import { canonicalTeam } from '../../src/lib/teams'; import type { MatchStatus, ServerMessage } from '../../src/lib/types'; @@ -57,6 +57,13 @@ export function buildServer() { app.get('/api/snapshot', async () => state.snapshot()); app.get('/api/predictions', async () => model.current()); app.get('/api/sources/health', async () => ({ sources: healthAll() })); + // Bookmaker odds (benchmark for the Model-vs-Market scoreboard; not a model input) + app.get('/api/odds', async () => ({ odds: latestOddsAll() })); + app.get('/api/odds/:num', async (req, reply) => { + const num = Number((req.params as { num: string }).num); + if (!Number.isFinite(num)) return reply.code(400).send({ error: 'bad num' }); + return { history: oddsFor(num) }; + }); app.get('/api/preview/:num', async (req, reply) => { const num = Number((req.params as { num: string }).num); const preview = Number.isFinite(num) ? buildPreview(num, state, model) : null; diff --git a/server/src/ingest/espnOdds.ts b/server/src/ingest/espnOdds.ts new file mode 100644 index 0000000..722c536 --- /dev/null +++ b/server/src/ingest/espnOdds.ts @@ -0,0 +1,60 @@ +import { cachedFetch } from './fetcher'; + +// ESPN's core API exposes real bookmaker odds (DraftKings et al.) per event — +// free and reachable from the VPS. Captured into odds_history as the public +// BENCHMARK for the Model-vs-Market scoreboard. Never fed into the model. +const CORE = 'https://sports.core.api.espn.com/v2/sports/soccer/leagues/fifa.world'; +const SOURCE = 'espn-odds'; + +interface RawOddsItem { + /** The core API sometimes returns pointer items instead of inline objects. */ + $ref?: string; + provider?: { name?: string }; + overUnder?: number; + spread?: number; + homeTeamOdds?: { moneyLine?: number }; + awayTeamOdds?: { moneyLine?: number }; + drawOdds?: { moneyLine?: number }; +} + +export interface CapturedOdds { + provider: string; + homeMl: number | null; + drawMl: number | null; + awayMl: number | null; + overUnder: number | null; + spread: number | null; +} + +export async function fetchEspnOdds(eventId: string): Promise { + const body = await cachedFetch<{ items?: RawOddsItem[] }>( + SOURCE, + `espn:odds:${eventId}`, + `${CORE}/events/${eventId}/competitions/${eventId}/odds`, + { ttl: 30 * 60_000, minInterval: 1_500 }, + ); + const out: CapturedOdds[] = []; + for (let it of body.items ?? []) { + // Pointer item → dereference (force https; ESPN refs come back as http). + if (it.$ref && !it.provider) { + const url = it.$ref.replace(/^http:/, 'https:'); + try { + it = await cachedFetch(SOURCE, `espn:oddsref:${eventId}:${url.slice(-40)}`, url, { + ttl: 30 * 60_000, + minInterval: 1_500, + }); + } catch { + continue; + } + } + out.push({ + provider: it.provider?.name ?? 'unknown', + homeMl: it.homeTeamOdds?.moneyLine ?? null, + drawMl: it.drawOdds?.moneyLine ?? null, + awayMl: it.awayTeamOdds?.moneyLine ?? null, + overUnder: it.overUnder ?? null, + spread: it.spread ?? null, + }); + } + return out; +} diff --git a/server/src/ingest/scheduler.ts b/server/src/ingest/scheduler.ts index 1723a54..88d9041 100644 --- a/server/src/ingest/scheduler.ts +++ b/server/src/ingest/scheduler.ts @@ -1,8 +1,9 @@ import { TournamentState, type LiveMatch, type Source } from '../tournament'; import { fetchEspnScoreboard, fetchEspnSummary } from './espn'; +import { fetchEspnOdds } from './espnOdds'; import { fetchFootballData } from '../footballData'; import { fetchSofascore } from '../sofascore'; -import { setSourceMap, getSourceMap, setMatchExt, logIngest } from '../db/db'; +import { setSourceMap, getSourceMap, setMatchExt, recordOdds, logIngest } from '../db/db'; import type { Fixture } from '../../../src/lib/types'; // The ingestion orchestrator. Replaces the old live loop: ESPN is the primary @@ -14,6 +15,8 @@ const LIVE_WINDOW_MS = 3 * 60 * 60 * 1000; const POLL_LIVE_MS = 20_000; const POLL_IDLE_MS = 10 * 60_000; const ENRICH_MS = 20 * 60_000; +const ODDS_MS = 3 * 60 * 60 * 1000; // odds sweep cadence (line-movement history) +const ODDS_HORIZON_MS = 14 * 24 * 60 * 60 * 1000; function dateUTC(iso: string): string { const d = new Date(iso); @@ -29,9 +32,17 @@ export function startScheduler(state: TournamentState, onChange: () => void): () console.log(`[ingest] scheduler — espn:on football-data:${token ? 'on' : 'off'} sofascore:${sofa ? 'on' : 'off'}`); - // ---- one-time: map every fixture to its ESPN event id (across all dates) ---- + // ---- one-time: map every fixture to its ESPN event id (across all dates). + // Mappings persist in source_map, so on restarts we only fetch dates that + // still contain unmapped fixtures — boots get fast, odds capture starts early. const mapAllFixtures = async (): Promise => { - const dates = [...new Set(state.allFixtures().map((f) => dateUTC(f.kickoff)))]; + const byDate = new Map(); + for (const f of state.allFixtures()) { + const d = dateUTC(f.kickoff); + if (!byDate.has(d)) byDate.set(d, []); + byDate.get(d)!.push(f); + } + const dates = [...byDate.keys()].filter((d) => byDate.get(d)!.some((f) => !getSourceMap(f.num, 'espn'))); let mapped = 0; for (const date of dates) { if (stopped) return; @@ -109,6 +120,41 @@ export function startScheduler(state: TournamentState, onChange: () => void): () } }; + // ---- odds capture: every upcoming fixture inside the horizon, plus imminent + // ones near kickoff. Insert-if-changed keeps a clean line-movement history. + const oddsTick = async (): Promise => { + const now = Date.now(); + const relevant = state.allFixtures().filter((f) => { + if (f.status === 'finished') return false; + const k = new Date(f.kickoff).getTime(); + return k > now - 3 * 60 * 60 * 1000 && k < now + ODDS_HORIZON_MS; + }); + let captured = 0; + const t0 = Date.now(); + for (const f of relevant) { + if (stopped) return; + const eid = getSourceMap(f.num, 'espn'); + if (!eid) continue; + try { + for (const o of await fetchEspnOdds(eid)) { + if (recordOdds({ + fixture_num: f.num, provider: o.provider, + home_ml: o.homeMl, draw_ml: o.drawMl, away_ml: o.awayMl, + over_under: o.overUnder, spread: o.spread, + })) captured++; + } + } catch { /* breaker/health track it */ } + } + logIngest('espn-odds', 'sweep', true, Date.now() - t0, `${relevant.length} fixtures, ${captured} new lines`); + if (captured) console.log(`[ingest] odds: ${captured} new line(s) captured`); + }; + + let oddsTimer: ReturnType | undefined; + const scheduleOdds = (): void => { + if (stopped) return; + oddsTimer = setTimeout(async () => { await oddsTick().catch(() => {}); scheduleOdds(); }, ODDS_MS); + }; + const scheduleLive = (): void => { if (stopped) return; const delay = state.hasLiveActivity(LIVE_WINDOW_MS) ? POLL_LIVE_MS : POLL_IDLE_MS; @@ -119,18 +165,22 @@ export function startScheduler(state: TournamentState, onChange: () => void): () enrichTimer = setTimeout(async () => { await enrichTick().catch(() => {}); scheduleEnrich(); }, ENRICH_MS); }; - // boot: map ids, then run the loops + // boot: map ids, then run the loops (odds first after mapping — line history + // is unrecoverable, so capture as early as possible) void (async () => { await mapAllFixtures().catch(() => {}); + await oddsTick().catch(() => {}); await liveTick().catch(() => {}); await enrichTick().catch(() => {}); scheduleLive(); scheduleEnrich(); + scheduleOdds(); })(); return () => { stopped = true; if (liveTimer) clearTimeout(liveTimer); if (enrichTimer) clearTimeout(enrichTimer); + if (oddsTimer) clearTimeout(oddsTimer); }; } diff --git a/src/lib/odds.test.ts b/src/lib/odds.test.ts new file mode 100644 index 0000000..585f5b7 --- /dev/null +++ b/src/lib/odds.test.ts @@ -0,0 +1,28 @@ +import { describe, it, expect } from 'vitest'; +import { americanToDecimal, deVig, overround } from './odds'; + +describe('americanToDecimal', () => { + it('converts favourites and underdogs', () => { + expect(americanToDecimal(-235)).toBeCloseTo(1.4255, 3); + expect(americanToDecimal(340)).toBeCloseTo(4.4, 3); + expect(americanToDecimal(100)).toBeCloseTo(2.0, 6); + expect(americanToDecimal(-100)).toBeCloseTo(2.0, 6); + }); +}); + +describe('deVig', () => { + it('produces probabilities that sum to 1 with the margin removed', () => { + // tonight's real opening-match lines: Mexico -235 / draw +340 / SA +750 + const p = deVig(-235, 340, 750)!; + expect(p.home + p.draw + p.away).toBeCloseTo(1, 9); + expect(p.home).toBeGreaterThan(0.6); // heavy favourite + expect(p.away).toBeLessThan(0.15); + // raw implied sum exceeds 1 (the vig) + expect(overround(-235, 340, 750)).toBeGreaterThan(1); + }); + + it('returns null on missing lines', () => { + expect(deVig(null, 340, 750)).toBeNull(); + expect(deVig(-235, 0, 750)).toBeNull(); + }); +}); diff --git a/src/lib/odds.ts b/src/lib/odds.ts new file mode 100644 index 0000000..1f3c125 --- /dev/null +++ b/src/lib/odds.ts @@ -0,0 +1,32 @@ +// Bookmaker-odds math, shared by the server (capture) and the client +// (Model-vs-Market scoreboard). Odds are a BENCHMARK only — never a model input. + +import type { MatchProbs } from './types'; + +/** American moneyline → decimal odds (e.g. -235 → 1.4255, +340 → 4.40). */ +export function americanToDecimal(ml: number): number { + if (!Number.isFinite(ml) || ml === 0) return NaN; + return ml < 0 ? 1 + 100 / Math.abs(ml) : 1 + ml / 100; +} + +/** + * De-vig a 1X2 set of American moneylines into fair probabilities: convert to + * implied probabilities (1/decimal) and normalize away the bookmaker margin. + * Returns null if any line is missing/invalid. + */ +export function deVig(homeMl: number | null, drawMl: number | null, awayMl: number | null): MatchProbs | null { + if (homeMl == null || drawMl == null || awayMl == null) return null; + const dh = americanToDecimal(homeMl); + const dd = americanToDecimal(drawMl); + const da = americanToDecimal(awayMl); + if (!Number.isFinite(dh) || !Number.isFinite(dd) || !Number.isFinite(da)) return null; + const ih = 1 / dh, id = 1 / dd, ia = 1 / da; + const total = ih + id + ia; + if (total <= 0) return null; + return { home: ih / total, draw: id / total, away: ia / total }; +} + +/** Bookmaker overround (margin) for display: e.g. 1.052 → "5.2%". */ +export function overround(homeMl: number, drawMl: number, awayMl: number): number { + return 1 / americanToDecimal(homeMl) + 1 / americanToDecimal(drawMl) + 1 / americanToDecimal(awayMl); +}