diff --git a/.gitignore b/.gitignore index ef1e8c0..03dade4 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ public/data/fixtures.json public/data/ratings.json public/data/h2h.json public/data/scorers.json +public/data/backtest.json public/pwa-192x192.png public/pwa-512x512.png public/favicon.svg diff --git a/package.json b/package.json index bf2b698..fa805c8 100644 --- a/package.json +++ b/package.json @@ -17,9 +17,10 @@ "data:fixtures": "bun scripts/buildFixtures.ts", "data:ratings": "bun scripts/buildRatings.ts", "data:preview": "bun scripts/buildPreviewData.ts", + "data:backtest": "bun scripts/buildBacktest.ts", "data:viz": "bun scripts/buildStatsbombViz.ts", "data:icons": "bun scripts/make-icons.mjs", - "data:build": "bun run data:icons && bun run data:fixtures && bun run data:ratings && bun run data:preview" + "data:build": "bun run data:icons && bun run data:fixtures && bun run data:ratings && bun run data:preview && bun run data:backtest" }, "dependencies": { "@tanstack/react-router": "^1.95.0", diff --git a/scripts/buildBacktest.ts b/scripts/buildBacktest.ts new file mode 100644 index 0000000..d585db0 --- /dev/null +++ b/scripts/buildBacktest.ts @@ -0,0 +1,199 @@ +// Walk-forward backtest of the Elo + Dixon-Coles model on historical +// internationals, with an honest train/test split (params fit on pre-2018, +// tested out-of-sample on 2018→now). Outputs proper scoring metrics (Brier, +// log-loss, RPS, accuracy), baseline comparisons, and a calibration/reliability +// analysis → public/data/backtest.json, shown on the Methodology page. +import { readFileSync, writeFileSync, mkdirSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { canonicalTeam } from '../src/lib/teams'; +import { HOME_ADV_ELO, importanceWeight, eloDelta } from '../src/lib/model/elo'; +import { lambdasFromElo, scoreMatrix, outcomeProbs, poissonPmf, type ModelParams } from '../src/lib/model/poisson'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); +const CSV = join(ROOT, 'data', 'raw', 'international-results.csv'); +const OUT_DIR = join(ROOT, 'public', 'data'); + +const START_ELO = 1500; +const PARAM_FROM = '2006-01-01'; +const TRAIN_END = '2018-01-01'; // params fit on [PARAM_FROM, TRAIN_END) +const TEST_TO = '2026-06-01'; // exclude the 2026 World Cup itself + +interface Row { date: string; home: string; away: string; hs: number; as: number; tournament: string; neutral: boolean } +type Probs = { h: number; d: number; a: number }; +type Outcome = 'h' | 'd' | 'a'; + +function parse(): Row[] { + const lines = readFileSync(CSV, 'utf8').split('\n'); + const rows: Row[] = []; + for (let i = 1; i < lines.length; i++) { + const c = lines[i]?.split(','); + if (!c || c.length < 9) continue; + const hs = Number(c[3]); const as = Number(c[4]); + if (!Number.isFinite(hs) || !Number.isFinite(as)) continue; + rows.push({ date: c[0]!, home: c[1]!, away: c[2]!, hs, as, tournament: c[5]!, neutral: c[8]!.trim().toUpperCase() === 'TRUE' }); + } + rows.sort((a, b) => a.date.localeCompare(b.date)); + return rows; +} + +function dcTau(h: number, a: number, lh: number, la: number, rho: number): number { + if (h === 0 && a === 0) return 1 - lh * la * rho; + if (h === 0 && a === 1) return 1 + lh * rho; + if (h === 1 && a === 0) return 1 + la * rho; + if (h === 1 && a === 1) return 1 - rho; + return 1; +} + +const outcomeOf = (hs: number, as: number): Outcome => (hs > as ? 'h' : hs < as ? 'a' : 'd'); + +// ---- scoring ---- +function brier(p: Probs, o: Outcome): number { + return (p.h - (o === 'h' ? 1 : 0)) ** 2 + (p.d - (o === 'd' ? 1 : 0)) ** 2 + (p.a - (o === 'a' ? 1 : 0)) ** 2; +} +function logloss(p: Probs, o: Outcome): number { + return -Math.log(Math.max(1e-12, p[o])); +} +function rps(p: Probs, o: Outcome): number { + // ordered H, D, A + const y = { h: o === 'h' ? 1 : 0, d: o === 'd' ? 1 : 0, a: o === 'a' ? 1 : 0 }; + const c1p = p.h, c1y = y.h; + const c2p = p.h + p.d, c2y = y.h + y.d; + return ((c1p - c1y) ** 2 + (c2p - c2y) ** 2) / 2; +} +const argmax = (p: Probs): Outcome => (p.h >= p.d && p.h >= p.a ? 'h' : p.d >= p.a ? 'd' : 'a'); + +function scoreSet(preds: { p: Probs; o: Outcome }[]) { + let b = 0, l = 0, r = 0, correct = 0; + for (const { p, o } of preds) { b += brier(p, o); l += logloss(p, o); r += rps(p, o); if (argmax(p) === o) correct++; } + const n = preds.length; + return { brier: b / n, logloss: l / n, rps: r / n, accuracy: correct / n }; +} + +function main(): void { + const rows = parse(); + const elo = new Map(); + const get = (t: string) => elo.get(t) ?? START_ELO; + + // ---- fit params on the training window ---- + let sumTotal = 0, nCal = 0, sxy = 0, sxx = 0; + const calib: { d: number; h: number; a: number }[] = []; + // we need a first pass to fit params using pre-match Elo; do it inline while walking + // (Elo is updated every match; calibration accumulates only within the train window) + + const testPreds: { p: Probs; o: Outcome }[] = []; + const baseUniform: { p: Probs; o: Outcome }[] = []; + const baseRate: { p: Probs; o: Outcome }[] = []; + const baseElo: { p: Probs; o: Outcome }[] = []; + const reliabilityRaw: { p: number; y: number }[] = []; + let wcCorrect = 0, wcN = 0; + + // outcome base rates over the test window (computed in a quick pre-scan) + let th = 0, td = 0, ta = 0, tn = 0; + for (const r of rows) { + if (r.date < TRAIN_END || r.date >= TEST_TO) continue; + const o = outcomeOf(r.hs, r.as); if (o === 'h') th++; else if (o === 'd') td++; else ta++; tn++; + } + const rate: Probs = { h: th / tn, d: td / tn, a: ta / tn }; + + // params get finalized at TRAIN_END; predictions before that use a rolling fit. + let params: ModelParams = { goalsPerElo: 0.0057, avgGoals: 2.7, rho: -0.05, homeAdvElo: HOME_ADV_ELO }; + let paramsFixed = false; + + const finalizeParams = (): ModelParams => { + const avgGoals = sumTotal / Math.max(1, nCal); + const goalsPerElo = sxy / Math.max(1e-9, sxx); + let bestRho = -0.05, bestLL = -Infinity; + for (let rho = -0.2; rho <= 0.1 + 1e-9; rho += 0.02) { + let ll = 0; + for (const m of calib) { + const sup = goalsPerElo * m.d; + const lh = Math.min(8, Math.max(0.15, avgGoals / 2 + sup / 2)); + const la = Math.min(8, Math.max(0.15, avgGoals / 2 - sup / 2)); + ll += Math.log(Math.max(1e-12, poissonPmf(lh, m.h) * poissonPmf(la, m.a) * dcTau(m.h, m.a, lh, la, rho))); + } + if (ll > bestLL) { bestLL = ll; bestRho = rho; } + } + return { goalsPerElo, avgGoals, rho: Math.round(bestRho * 100) / 100, homeAdvElo: HOME_ADV_ELO }; + }; + + const predict = (eh: number, ea: number, homeAdv: number): Probs => { + const { lambdaHome, lambdaAway } = lambdasFromElo(eh, ea, params, homeAdv); + const p = outcomeProbs(scoreMatrix(lambdaHome, lambdaAway, params.rho)); + return { h: p.home, d: p.draw, a: p.away }; + }; + + for (const r of rows) { + if (!paramsFixed && r.date >= TRAIN_END) { params = finalizeParams(); paramsFixed = true; } + + const home = canonicalTeam(r.home), away = canonicalTeam(r.away); + const eh = get(home), ea = get(away); + const homeAdv = r.neutral ? 0 : HOME_ADV_ELO; + const o = outcomeOf(r.hs, r.as); + + // accumulate calibration only in the param-fitting window + if (r.date >= PARAM_FROM && r.date < TRAIN_END) { + const effDiff = eh - ea + homeAdv; + sumTotal += r.hs + r.as; nCal++; sxy += effDiff * (r.hs - r.as); sxx += effDiff * effDiff; + calib.push({ d: effDiff, h: r.hs, a: r.as }); + } + + // score out-of-sample + if (r.date >= TRAIN_END && r.date < TEST_TO) { + const p = predict(eh, ea, homeAdv); + testPreds.push({ p, o }); + baseUniform.push({ p: { h: 1 / 3, d: 1 / 3, a: 1 / 3 }, o }); + baseRate.push({ p: rate, o }); + // elo-only: expected score → W/A split, fixed draw rate + const e = 1 / (1 + 10 ** ((ea - eh - homeAdv) / 400)); + baseElo.push({ p: { h: e * (1 - rate.d), d: rate.d, a: (1 - e) * (1 - rate.d) }, o }); + reliabilityRaw.push({ p: p.h, y: o === 'h' ? 1 : 0 }, { p: p.d, y: o === 'd' ? 1 : 0 }, { p: p.a, y: o === 'a' ? 1 : 0 }); + if (r.tournament.includes('FIFA World Cup') && !r.tournament.includes('qualification')) { + wcN++; if (argmax(p) === o) wcCorrect++; + } + } + + // update Elo (always) + const k = importanceWeight(r.tournament); + const d = eloDelta(r.hs, r.as, eh, ea, k, homeAdv); + elo.set(home, eh + d); elo.set(away, ea - d); + } + + // reliability bins (10) + ECE + const bins = Array.from({ length: 10 }, () => ({ sp: 0, sy: 0, n: 0 })); + for (const { p, y } of reliabilityRaw) { + const idx = Math.min(9, Math.floor(p * 10)); + bins[idx]!.sp += p; bins[idx]!.sy += y; bins[idx]!.n++; + } + const reliability = bins.map((b) => ({ predicted: b.n ? b.sp / b.n : 0, observed: b.n ? b.sy / b.n : 0, count: b.n })); + const totalN = reliabilityRaw.length; + const ece = bins.reduce((s, b) => s + (b.n ? (b.n / totalN) * Math.abs(b.sp / b.n - b.sy / b.n) : 0), 0); + + const out = { + generatedAt: new Date().toISOString(), + paramFrom: PARAM_FROM, + trainEnd: TRAIN_END, + testTo: TEST_TO, + tested: testPreds.length, + params, + model: scoreSet(testPreds), + baselines: { + uniform: scoreSet(baseUniform), + baseRate: scoreSet(baseRate), + eloOnly: scoreSet(baseElo), + }, + reliability, + ece: Math.round(ece * 1000) / 1000, + worldCup: { tested: wcN, accuracy: wcN ? wcCorrect / wcN : 0 }, + }; + + mkdirSync(OUT_DIR, { recursive: true }); + writeFileSync(join(OUT_DIR, 'backtest.json'), JSON.stringify(out)); + console.log(`backtest → public/data/backtest.json (${out.tested} test matches, ${TRAIN_END}→${TEST_TO})`); + console.log(` model: brier ${out.model.brier.toFixed(3)} logloss ${out.model.logloss.toFixed(3)} rps ${out.model.rps.toFixed(3)} acc ${(out.model.accuracy * 100).toFixed(1)}%`); + console.log(` uniform: brier ${out.baselines.uniform.brier.toFixed(3)} logloss ${out.baselines.uniform.logloss.toFixed(3)} rps ${out.baselines.uniform.rps.toFixed(3)}`); + console.log(` baseRate:brier ${out.baselines.baseRate.brier.toFixed(3)} rps ${out.baselines.baseRate.rps.toFixed(3)} | eloOnly rps ${out.baselines.eloOnly.rps.toFixed(3)}`); + console.log(` ECE ${out.ece} | WC accuracy ${(out.worldCup.accuracy * 100).toFixed(1)}% (${out.worldCup.tested})`); +} + +main(); diff --git a/src/app/RootLayout.tsx b/src/app/RootLayout.tsx index fc8e354..80d5232 100644 --- a/src/app/RootLayout.tsx +++ b/src/app/RootLayout.tsx @@ -1,6 +1,6 @@ import { useEffect } from 'react'; import { Link, Outlet } from '@tanstack/react-router'; -import { GitMerge, Moon, Radio, Sparkles, Sun, Table, TrendingUp, Trophy } from 'lucide-react'; +import { Gauge, GitMerge, Moon, Radio, Sparkles, Sun, Table, TrendingUp, Trophy } from 'lucide-react'; import type { LucideIcon } from 'lucide-react'; import { useUiStore } from '@/stores/uiStore'; import { useTournamentStore } from '@/stores/tournamentStore'; @@ -11,6 +11,7 @@ const NAV: { to: string; label: string; exact: boolean; icon: LucideIcon }[] = [ { to: '/groups', label: 'Groups', exact: false, icon: Table }, { to: '/bracket', label: 'Bracket', exact: false, icon: GitMerge }, { to: '/predict', label: 'Predict', exact: false, icon: TrendingUp }, + { to: '/methodology', label: 'Model', exact: false, icon: Gauge }, { to: '/story', label: 'Story', exact: false, icon: Sparkles }, ]; diff --git a/src/components/ReliabilityDiagram.tsx b/src/components/ReliabilityDiagram.tsx new file mode 100644 index 0000000..44550f6 --- /dev/null +++ b/src/components/ReliabilityDiagram.tsx @@ -0,0 +1,49 @@ +import { scaleLinear } from 'd3-scale'; +import type { BacktestReport } from '@/lib/types'; + +const W = 360; +const H = 360; +const M = 34; + +/** Calibration plot: predicted probability (x) vs observed frequency (y). Points + * on the diagonal = perfectly calibrated. Dot size ∝ sample count. */ +export function ReliabilityDiagram({ reliability, ece }: { reliability: BacktestReport['reliability']; ece: number }) { + const sx = scaleLinear().domain([0, 1]).range([M, W - 8]); + const sy = scaleLinear().domain([0, 1]).range([H - M, 8]); + const maxCount = Math.max(...reliability.map((b) => b.count), 1); + const pts = reliability.filter((b) => b.count > 0); + + return ( +
+ + {/* axes */} + + + {/* perfect-calibration diagonal */} + + {[0.25, 0.5, 0.75, 1].map((t) => ( + + {t} + {t} + + ))} + {/* model points + connecting line */} + `${sx(b.predicted)},${sy(b.observed)}`).join(' ')} + fill="none" stroke="var(--app-accent)" strokeWidth={1.5} opacity={0.5} + /> + {pts.map((b, i) => ( + + {`predicted ${(b.predicted * 100).toFixed(0)}% → happened ${(b.observed * 100).toFixed(0)}% (${b.count})`} + + ))} + predicted probability + actually happened + +

+ Points hug the diagonal — when the model says 30%, it happens about 30% of the time. + Calibration error (ECE) is just {ece.toFixed(2)} (lower is better; under 0.05 is excellent). +

+
+ ); +} diff --git a/src/features/methodology/MethodologyPage.tsx b/src/features/methodology/MethodologyPage.tsx new file mode 100644 index 0000000..4b353d2 --- /dev/null +++ b/src/features/methodology/MethodologyPage.tsx @@ -0,0 +1,119 @@ +import { useEffect, useState } from 'react'; +import { Activity, Dices, Gauge, TrendingUp } from 'lucide-react'; +import { PageHeader } from '@/components/ui/PageHeader'; +import { Card, CardBody, CardHeader } from '@/components/ui/Card'; +import { ReliabilityDiagram } from '@/components/ReliabilityDiagram'; +import type { BacktestReport } from '@/lib/types'; + +const STEPS = [ + { icon: TrendingUp, title: 'Elo ratings', body: '150 years of international results (49,000 matches) build a strength rating for every nation, updated after each game and weighted by match importance.' }, + { icon: Activity, title: 'Expected goals', body: "Each team's goal expectation comes from the rating gap, calibrated on how Elo differences have actually translated into goals — plus host home advantage." }, + { icon: Gauge, title: 'Dixon-Coles', body: 'A bivariate-Poisson scoreline model (with the Dixon-Coles low-score correction) turns those goal expectations into the full distribution of scorelines and win/draw/loss.' }, + { icon: Dices, title: 'Monte Carlo', body: 'The whole 48-team tournament is simulated 20,000 times — sampling every remaining match — to produce championship and advancement odds that update after each result.' }, +]; + +function RpsBar({ label, rps, best, highlight }: { label: string; rps: number; best: number; highlight?: boolean }) { + // shorter bar = better (lower RPS). Scale relative to the worst (uniform ~0.24). + const width = Math.min(100, (rps / 0.26) * 100); + return ( +
+ {label} +
+
+
+ {rps.toFixed(3)} + {rps === best && best} +
+ ); +} + +function Stat({ value, label }: { value: string; label: string }) { + return ( +
+
{value}
+
{label}
+
+ ); +} + +export function MethodologyPage() { + const [bt, setBt] = useState(null); + + useEffect(() => { + let alive = true; + fetch('/data/backtest.json').then((r) => r.json()).then((d: BacktestReport) => alive && setBt(d)).catch(() => {}); + return () => { alive = false; }; + }, []); + + const bestRps = bt ? Math.min(bt.model.rps, bt.baselines.uniform.rps, bt.baselines.baseRate.rps, bt.baselines.eloOnly.rps) : 0; + + return ( +
+ + +
+ {STEPS.map((s, i) => ( + + +
+ + STEP {i + 1} +
+

{s.title}

+

{s.body}

+ + + ))} +
+ + + How good is it? (out-of-sample backtest) + + {bt ? ( + <> +

+ Model parameters were fit on internationals before {bt.trainEnd.slice(0, 4)}, then tested walk-forward on{' '} + {bt.tested.toLocaleString()} matches it had never seen ({bt.trainEnd.slice(0, 4)}–{bt.testTo.slice(0, 4)}) — predicting each game using only prior data. +

+
+ + + +
+
+
Ranked Probability Score — lower is better, vs baselines
+
+ + + + +
+
+ + ) : ( +
+ )} + + + + + Is it calibrated? + + {bt ? :
} + + + + + Honest limits + +
    +
  • • These are model probabilities, not betting advice. We don't use bookmaker odds, so we make no claim to beat the market — sharp betting markets remain the most accurate forecaster there is.
    • +
  • • International expected-goals and event data are sparse, so per-match tactical breakdowns are richer for some teams than others, and improve as live data accumulates during the tournament.
    • +
  • • Football is high-variance: a 60% favourite still loses plenty. Calibration means our 60% really is ~60% — not that the favourite always wins.
    • +
  • • Everything is transparent and reproducible from public data (results, ESPN, StatsBomb) — that openness is the point.
    • +
+ + +
+ ); +} diff --git a/src/lib/types.ts b/src/lib/types.ts index 9d743c1..0ede4d3 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -160,6 +160,22 @@ export interface TeamProfile { qualifyOdds: number | null; } +// ---- model backtest (Methodology page) ---- + +export interface BacktestScores { brier: number; logloss: number; rps: number; accuracy: number } +export interface BacktestReport { + generatedAt: string; + trainEnd: string; + testTo: string; + tested: number; + params: { goalsPerElo: number; avgGoals: number; rho: number; homeAdvElo: number }; + model: BacktestScores; + baselines: { uniform: BacktestScores; baseRate: BacktestScores; eloOnly: BacktestScores }; + reliability: { predicted: number; observed: number; count: number }[]; + ece: number; + worldCup: { tested: number; accuracy: number }; +} + // ---- data-story viz (StatsBomb open data, a fixed historical match) ---- export interface VizShot { diff --git a/src/router.tsx b/src/router.tsx index 8cde730..76aacae 100644 --- a/src/router.tsx +++ b/src/router.tsx @@ -8,6 +8,7 @@ import { PredictionsPage } from './features/predictions/PredictionsPage'; import { StoryPage } from './features/story/StoryPage'; import { MatchPreviewPage } from './features/match/MatchPreviewPage'; import { TeamProfilePage } from './features/team/TeamProfilePage'; +import { MethodologyPage } from './features/methodology/MethodologyPage'; const rootRoute = createRootRoute({ component: RootLayout, @@ -21,8 +22,9 @@ const predictRoute = createRoute({ getParentRoute: () => rootRoute, path: '/pred const storyRoute = createRoute({ getParentRoute: () => rootRoute, path: '/story', component: StoryPage }); const matchRoute = createRoute({ getParentRoute: () => rootRoute, path: '/match/$num', component: MatchPreviewPage }); const teamRoute = createRoute({ getParentRoute: () => rootRoute, path: '/team/$name', component: TeamProfilePage }); +const methodologyRoute = createRoute({ getParentRoute: () => rootRoute, path: '/methodology', component: MethodologyPage }); -const routeTree = rootRoute.addChildren([indexRoute, groupsRoute, bracketRoute, predictRoute, storyRoute, matchRoute, teamRoute]); +const routeTree = rootRoute.addChildren([indexRoute, groupsRoute, bracketRoute, predictRoute, storyRoute, matchRoute, teamRoute, methodologyRoute]); export const router = createRouter({ routeTree, defaultPreload: 'intent' });