diff --git a/server/src/db/db.ts b/server/src/db/db.ts index e576972..7748ac0 100644 --- a/server/src/db/db.ts +++ b/server/src/db/db.ts @@ -85,6 +85,14 @@ CREATE TABLE IF NOT EXISTS odds_history ( ); CREATE INDEX IF NOT EXISTS idx_odds_fixture ON odds_history(fixture_num, captured_at); +CREATE TABLE IF NOT EXISTS prediction_snapshots ( + fixture_num INTEGER PRIMARY KEY, + taken_at INTEGER NOT NULL, + late INTEGER NOT NULL DEFAULT 0, + model_json TEXT NOT NULL, + market_json TEXT +); + CREATE TABLE IF NOT EXISTS ingest_log ( id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL, @@ -229,6 +237,31 @@ export function latestOddsAll(): OddsRow[] { .all() as unknown as OddsRow[]; } +// ---- frozen pre-kickoff forecasts (Model-vs-Market scoreboard) ---- +export interface SnapshotRow { + fixture_num: number; + taken_at: number; + late: number; + model_json: string; + market_json: string | null; +} + +export function hasSnapshot(fixtureNum: number): boolean { + return !!db().prepare('SELECT 1 FROM prediction_snapshots WHERE fixture_num = ?').get(fixtureNum); +} + +export function saveSnapshot(fixtureNum: number, late: boolean, model: unknown, market: unknown | null): void { + db() + .prepare('INSERT OR IGNORE INTO prediction_snapshots (fixture_num, taken_at, late, model_json, market_json) VALUES (?, ?, ?, ?, ?)') + .run(fixtureNum, Date.now(), late ? 1 : 0, JSON.stringify(model), market ? JSON.stringify(market) : null); +} + +export function allSnapshots(): SnapshotRow[] { + return db() + .prepare('SELECT fixture_num, taken_at, late, model_json, market_json FROM prediction_snapshots ORDER BY fixture_num') + .all() as unknown as SnapshotRow[]; +} + // ---- per-fixture enrichment blob ---- export function setMatchExt(fixtureNum: number, json: unknown): void { db().prepare('INSERT OR REPLACE INTO match_ext (fixture_num, json, updated_at) VALUES (?, ?, ?)') diff --git a/server/src/index.ts b/server/src/index.ts index e8f17d8..f9b0b3d 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -9,6 +9,7 @@ import { TournamentState } from './tournament'; import { startScheduler } from './ingest/scheduler'; import { ModelEngine } from './model'; import { buildPreview, buildTeamProfile } from './preview'; +import { buildScoreboard, snapshotCheck } from './scoreboard'; import { db, healthAll, oddsFor, latestOddsAll } from './db/db'; import { canonicalTeam } from '../../src/lib/teams'; import type { MatchStatus, ServerMessage } from '../../src/lib/types'; @@ -59,6 +60,7 @@ export function buildServer() { app.get('/api/sources/health', async () => ({ sources: healthAll() })); // Bookmaker odds (benchmark for the Model-vs-Market scoreboard; not a model input) app.get('/api/odds', async () => ({ odds: latestOddsAll() })); + app.get('/api/scoreboard', async () => buildScoreboard(state)); app.get('/api/odds/:num', async (req, reply) => { const num = Number((req.params as { num: string }).num); if (!Number.isFinite(num)) return reply.code(400).send({ error: 'bad num' }); @@ -93,6 +95,7 @@ export function buildServer() { 'football-data', ); if (!f) return reply.code(404).send({ error: 'no such fixture' }); + snapshotCheck(state, model); broadcast(); return { ok: true, fixture: f }; }); @@ -125,7 +128,10 @@ export function buildServer() { }); // ---- ingestion scheduler (ESPN primary; football-data / SofaScore fallback) ---- - const stopIngest = startScheduler(state, () => broadcast()); + const stopIngest = startScheduler(state, () => broadcast(), () => { + const taken = snapshotCheck(state, model); + if (taken) console.log(`[scoreboard] froze ${taken} pre-kickoff forecast(s)`); + }); app.addHook('onClose', async () => stopIngest()); // ---- static SPA (prod) with history fallback ---- diff --git a/server/src/ingest/scheduler.ts b/server/src/ingest/scheduler.ts index 88d9041..6807b2f 100644 --- a/server/src/ingest/scheduler.ts +++ b/server/src/ingest/scheduler.ts @@ -23,7 +23,11 @@ function dateUTC(iso: string): string { return `${d.getUTCFullYear()}${String(d.getUTCMonth() + 1).padStart(2, '0')}${String(d.getUTCDate()).padStart(2, '0')}`; } -export function startScheduler(state: TournamentState, onChange: () => void): () => void { +export function startScheduler( + state: TournamentState, + onChange: () => void, + onTick?: () => void, +): () => void { const token = process.env.FOOTBALL_DATA_TOKEN?.trim(); const sofa = process.env.ENABLE_SOFASCORE === 'true'; let stopped = false; @@ -88,6 +92,7 @@ export function startScheduler(state: TournamentState, onChange: () => void): () const changed = state.mergeLive(matches, source); if (changed.length) { console.log(`[ingest] ${changed.length} fixture(s) updated via ${source}`); onChange(); } } + onTick?.(); // e.g. freeze pre-kickoff forecast snapshots // Keep live matches' stats + event timeline fresh at the live cadence. for (const f of state.allFixtures()) { diff --git a/server/src/scoreboard.ts b/server/src/scoreboard.ts new file mode 100644 index 0000000..a21e114 --- /dev/null +++ b/server/src/scoreboard.ts @@ -0,0 +1,113 @@ +import { allSnapshots, hasSnapshot, saveSnapshot, oddsFor } from './db/db'; +import { deVig } from '../../src/lib/odds'; +import { brierScore, outcomeOfScore, rpsScore } from '../../src/lib/model/scoring'; +import type { TournamentState } from './tournament'; +import type { ModelEngine } from './model'; +import type { MarketSnapshot, MatchProbs, ScoreboardData, ScoreboardRow } from '../../src/lib/types'; + +// The Model-vs-Market scoreboard. Fairness rules: +// - the model's forecast is FROZEN at most SNAPSHOT_BEFORE_MS before kickoff +// (whatever the model believed then — no hindsight); +// - the market benchmark is the latest captured pre-kickoff bookmaker line, +// de-vigged to fair probabilities; +// - snapshots taken after kickoff are stored but flagged `late` and excluded +// from the head-to-head totals. + +const SNAPSHOT_BEFORE_MS = 15 * 60 * 1000; + +function latestPreKickoffMarket(num: number, kickoffT: number): MarketSnapshot | null { + const history = oddsFor(num).filter((o) => o.captured_at <= kickoffT + 60_000); + const last = history[history.length - 1]; + if (!last) return null; + const probs = deVig(last.home_ml, last.draw_ml, last.away_ml); + if (!probs) return null; + return { + provider: last.provider, + probs, + homeMl: last.home_ml!, + drawMl: last.draw_ml!, + awayMl: last.away_ml!, + }; +} + +/** Freeze forecasts for any fixture at/inside the snapshot window. Runs on every + * live tick — cheap (indexed point lookups, no network). */ +export function snapshotCheck(state: TournamentState, model: ModelEngine): number { + const now = Date.now(); + let taken = 0; + for (const f of state.allFixtures()) { + if (!f.home.team || !f.away.team) continue; + const kickoffT = new Date(f.kickoff).getTime(); + // Inside the pre-kickoff window — or the match already started regardless of + // the scheduled clock (reschedules); the `late` flag keeps totals honest. + if (now < kickoffT - SNAPSHOT_BEFORE_MS && f.status === 'scheduled') continue; + if (hasSnapshot(f.num)) continue; + const pred = model.current().matches.find((m) => m.num === f.num); + if (!pred) continue; // finished before we ever saw it — nothing honest to freeze + const late = f.status !== 'scheduled'; + saveSnapshot(f.num, late, { probs: pred.probs, lambdaHome: pred.lambdaHome, lambdaAway: pred.lambdaAway }, latestPreKickoffMarket(f.num, kickoffT)); + taken++; + } + return taken; +} + +export function buildScoreboard(state: TournamentState): ScoreboardData { + const byNum = new Map(state.allFixtures().map((f) => [f.num, f])); + const rows: ScoreboardRow[] = []; + let n = 0, mR = 0, kR = 0, mB = 0, kB = 0, mCloser = 0, kCloser = 0; + + for (const s of allSnapshots()) { + const f = byNum.get(s.fixture_num); + if (!f || !f.home.team || !f.away.team) continue; + const modelStored = JSON.parse(s.model_json) as { probs: MatchProbs }; + const market = s.market_json ? (JSON.parse(s.market_json) as MarketSnapshot) : null; + + let scored: ScoreboardRow['scored'] = null; + if (f.status === 'finished' && f.homeScore != null && f.awayScore != null) { + const o = outcomeOfScore(f.homeScore, f.awayScore); + scored = { + outcome: o, + modelRps: +rpsScore(modelStored.probs, o).toFixed(4), + marketRps: market ? +rpsScore(market.probs, o).toFixed(4) : null, + modelBrier: +brierScore(modelStored.probs, o).toFixed(4), + marketBrier: market ? +brierScore(market.probs, o).toFixed(4) : null, + }; + if (!s.late && market && scored.marketRps != null) { + n++; + mR += scored.modelRps; kR += scored.marketRps; + mB += scored.modelBrier; kB += scored.marketBrier!; + if (scored.modelRps < scored.marketRps) mCloser++; + else if (scored.marketRps < scored.modelRps) kCloser++; + } + } + + rows.push({ + num: f.num, + home: f.home.team, + away: f.away.team, + group: f.group, + kickoff: f.kickoff, + status: f.status, + homeScore: f.homeScore, + awayScore: f.awayScore, + takenAt: s.taken_at, + late: !!s.late, + model: modelStored.probs, + market, + scored, + }); + } + + rows.sort((a, b) => a.kickoff.localeCompare(b.kickoff)); + return { + rows, + totals: n + ? { + n, + modelRps: +(mR / n).toFixed(4), marketRps: +(kR / n).toFixed(4), + modelBrier: +(mB / n).toFixed(4), marketBrier: +(kB / n).toFixed(4), + modelCloser: mCloser, marketCloser: kCloser, + } + : null, + }; +} diff --git a/src/app/RootLayout.tsx b/src/app/RootLayout.tsx index 80d5232..a3d0129 100644 --- a/src/app/RootLayout.tsx +++ b/src/app/RootLayout.tsx @@ -1,6 +1,6 @@ import { useEffect } from 'react'; import { Link, Outlet } from '@tanstack/react-router'; -import { Gauge, GitMerge, Moon, Radio, Sparkles, Sun, Table, TrendingUp, Trophy } from 'lucide-react'; +import { Gauge, GitMerge, Moon, Radio, Scale, Sparkles, Sun, Table, TrendingUp, Trophy } from 'lucide-react'; import type { LucideIcon } from 'lucide-react'; import { useUiStore } from '@/stores/uiStore'; import { useTournamentStore } from '@/stores/tournamentStore'; @@ -11,6 +11,7 @@ const NAV: { to: string; label: string; exact: boolean; icon: LucideIcon }[] = [ { to: '/groups', label: 'Groups', exact: false, icon: Table }, { to: '/bracket', label: 'Bracket', exact: false, icon: GitMerge }, { to: '/predict', label: 'Predict', exact: false, icon: TrendingUp }, + { to: '/scoreboard', label: 'vs Market', exact: false, icon: Scale }, { to: '/methodology', label: 'Model', exact: false, icon: Gauge }, { to: '/story', label: 'Story', exact: false, icon: Sparkles }, ]; diff --git a/src/features/methodology/MethodologyPage.tsx b/src/features/methodology/MethodologyPage.tsx index 4b353d2..e2a3fc9 100644 --- a/src/features/methodology/MethodologyPage.tsx +++ b/src/features/methodology/MethodologyPage.tsx @@ -7,8 +7,8 @@ import type { BacktestReport } from '@/lib/types'; const STEPS = [ { icon: TrendingUp, title: 'Elo ratings', body: '150 years of international results (49,000 matches) build a strength rating for every nation, updated after each game and weighted by match importance.' }, - { icon: Activity, title: 'Expected goals', body: "Each team's goal expectation comes from the rating gap, calibrated on how Elo differences have actually translated into goals — plus host home advantage." }, - { icon: Gauge, title: 'Dixon-Coles', body: 'A bivariate-Poisson scoreline model (with the Dixon-Coles low-score correction) turns those goal expectations into the full distribution of scorelines and win/draw/loss.' }, + { icon: Activity, title: 'Two goal models', body: 'Elo-based goal expectations are blended with an independent attack/defence Dixon-Coles model (time-decayed, fit on the last 15 years) — an ensemble that beat both members in testing.' }, + { icon: Gauge, title: 'Dixon-Coles scorelines', body: 'A bivariate-Poisson scoreline model (with the Dixon-Coles low-score correction) turns the blended expectations into the full distribution of scorelines and win/draw/loss.' }, { icon: Dices, title: 'Monte Carlo', body: 'The whole 48-team tournament is simulated 20,000 times — sampling every remaining match — to produce championship and advancement odds that update after each result.' }, ]; @@ -72,8 +72,8 @@ export function MethodologyPage() { {bt ? ( <>

- Model parameters were fit on internationals before {bt.trainEnd.slice(0, 4)}, then tested walk-forward on{' '} - {bt.tested.toLocaleString()} matches it had never seen ({bt.trainEnd.slice(0, 4)}–{bt.testTo.slice(0, 4)}) — predicting each game using only prior data. + Strict three-way split: parameters fit before {bt.trainEnd.slice(0, 4)}, ensemble settings tuned on {bt.validation ? `${bt.validation.from.slice(0, 4)}–${bt.validation.to.slice(0, 4)}` : 'a validation window'}, and the numbers below come from{' '} + {bt.tested.toLocaleString()} untouched matches ({(bt.testFrom ?? bt.trainEnd).slice(0, 4)}–{bt.testTo.slice(0, 4)}) — each predicted walk-forward using only prior data.

@@ -81,10 +81,11 @@ export function MethodologyPage() {
-
Ranked Probability Score — lower is better, vs baselines
+
Ranked Probability Score — lower is better, vs candidates & baselines
- - + {(bt.variants ?? [{ name: 'This model', ...bt.model }]).map((v, i) => ( + + ))}
diff --git a/src/features/scoreboard/ScoreboardPage.tsx b/src/features/scoreboard/ScoreboardPage.tsx new file mode 100644 index 0000000..5d7925f --- /dev/null +++ b/src/features/scoreboard/ScoreboardPage.tsx @@ -0,0 +1,140 @@ +import { useEffect, useState } from 'react'; +import { Link } from '@tanstack/react-router'; +import { Scale } from 'lucide-react'; +import { PageHeader } from '@/components/ui/PageHeader'; +import { Card, CardBody, CardHeader } from '@/components/ui/Card'; +import { teamFlag } from '@/lib/teams'; +import { kickoffDay, kickoffTime } from '@/lib/format'; +import { cn } from '@/lib/cn'; +import type { MatchProbs, ScoreboardData, ScoreboardRow } from '@/lib/types'; + +const pct = (x: number) => `${Math.round(x * 100)}%`; + +function ProbTriple({ label, probs, tone }: { label: string; probs: MatchProbs; tone: 'model' | 'market' }) { + const color = tone === 'model' ? 'bg-accent' : 'bg-gold'; + return ( +
+ {label} +
+
+
+
+
+ + {pct(probs.home)} / {pct(probs.draw)} / {pct(probs.away)} + +
+ ); +} + +function Row({ r }: { r: ScoreboardRow }) { + const closer = + r.scored && r.scored.marketRps != null + ? r.scored.modelRps < r.scored.marketRps ? 'model' : r.scored.marketRps < r.scored.modelRps ? 'market' : 'tie' + : null; + return ( + + +
+ + {teamFlag(r.home)} {r.home} + {r.status !== 'scheduled' && r.homeScore != null ? `${r.homeScore}–${r.awayScore}` : 'vs'} + {r.away} {teamFlag(r.away)} + + + {r.group ? `Group ${r.group} · ` : ''}{kickoffDay(r.kickoff)} {kickoffTime(r.kickoff)} + {r.late && ' · snapshot late — excluded from totals'} + +
+ + {r.market ? ( + + ) : ( +

no bookmaker line captured before kickoff

+ )} + {r.scored && ( +
+ result: {r.scored.outcome} + model RPS {r.scored.modelRps} + {r.scored.marketRps != null && market RPS {r.scored.marketRps}} + {closer && closer !== 'tie' && ( + + {closer === 'model' ? 'model closer' : 'market closer'} + + )} +
+ )} + + + ); +} + +export function ScoreboardPage() { + const [data, setData] = useState(null); + + useEffect(() => { + let alive = true; + const load = () => fetch('/api/scoreboard').then((r) => r.json()).then((d: ScoreboardData) => alive && setData(d)).catch(() => {}); + void load(); + const id = setInterval(load, 60_000); + return () => { alive = false; clearInterval(id); }; + }, []); + + const t = data?.totals ?? null; + const lead = t ? (t.modelRps < t.marketRps ? 'model' : t.marketRps < t.modelRps ? 'market' : 'tie') : null; + + return ( +
+ + + + + + Running head-to-head + + + {t ? ( +
+
+
{t.modelRps.toFixed(4)}
+
our model · RPS
+
closer {t.modelCloser}×
+
+
+ {t.n} match{t.n === 1 ? '' : 'es'} scored +
{lead === 'tie' ? 'dead level' : lead === 'model' ? 'model ahead' : 'market ahead'}
+
+
+
{t.marketRps.toFixed(4)}
+
bookmaker · RPS
+
closer {t.marketCloser}×
+
+
+ ) : ( +

+ The head-to-head starts with the first final whistle — forecasts are frozen before each kickoff and scored when the match ends. +

+ )} +

+ Honest rules: both forecasts frozen pre-kickoff; bookmaker margin removed (de-vig); scored with the Ranked Probability + Score; late snapshots excluded. Beating the market over a small sample is noise — watch the gap, not the lead. +

+ + + + {data ? ( +
+ {data.rows.length === 0 && ( +

No frozen forecasts yet — the first snapshots are taken 15 minutes before kickoff.

+ )} + {data.rows.map((r) => )} +
+ ) : ( +
+ )} +
+ ); +} diff --git a/src/lib/model/scoring.ts b/src/lib/model/scoring.ts new file mode 100644 index 0000000..eee2913 --- /dev/null +++ b/src/lib/model/scoring.ts @@ -0,0 +1,29 @@ +// Proper scoring rules — ONE implementation shared by the offline backtest and +// the live Model-vs-Market scoreboard, so the numbers are always comparable. + +import type { MatchProbs } from '../types'; + +export type Outcome3 = 'home' | 'draw' | 'away'; + +export const outcomeOfScore = (hs: number, as: number): Outcome3 => + hs > as ? 'home' : hs < as ? 'away' : 'draw'; + +/** Multiclass Brier (sum of squared errors over the 3 outcomes; 0 best, 2 worst). */ +export function brierScore(p: MatchProbs, o: Outcome3): number { + return ( + (p.home - (o === 'home' ? 1 : 0)) ** 2 + + (p.draw - (o === 'draw' ? 1 : 0)) ** 2 + + (p.away - (o === 'away' ? 1 : 0)) ** 2 + ); +} + +/** Ranked Probability Score for the ordered outcomes home > draw > away (0 best). */ +export function rpsScore(p: MatchProbs, o: Outcome3): number { + const y1 = o === 'home' ? 1 : 0; + const y2 = o === 'away' ? 0 : 1; // cumulative: home+draw + return ((p.home - y1) ** 2 + (p.home + p.draw - y2) ** 2) / 2; +} + +export function logLoss(p: MatchProbs, o: Outcome3): number { + return -Math.log(Math.max(1e-12, p[o])); +} diff --git a/src/lib/types.ts b/src/lib/types.ts index 2c31512..b0cc9cf 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -165,16 +165,56 @@ export interface TeamProfile { qualifyOdds: number | null; } +// ---- Model-vs-Market scoreboard ---- + +export interface MarketSnapshot { + provider: string; + probs: MatchProbs; // de-vigged + homeMl: number; + drawMl: number; + awayMl: number; +} + +export interface ScoreboardRow { + num: number; + home: string; + away: string; + group: string | null; + kickoff: string; + status: MatchStatus; + homeScore: number | null; + awayScore: number | null; + takenAt: number; + /** Snapshot was taken after kickoff — excluded from the fair totals. */ + late: boolean; + model: MatchProbs; + market: MarketSnapshot | null; + /** Scores, present once the match finished (regulation result). */ + scored: { outcome: 'home' | 'draw' | 'away'; modelRps: number; marketRps: number | null; modelBrier: number; marketBrier: number | null } | null; +} + +export interface ScoreboardData { + rows: ScoreboardRow[]; + /** Aggregates over finished, fair (pre-kickoff, with-market) matches. */ + totals: { n: number; modelRps: number; marketRps: number; modelBrier: number; marketBrier: number; modelCloser: number; marketCloser: number } | null; +} + // ---- model backtest (Methodology page) ---- export interface BacktestScores { brier: number; logloss: number; rps: number; accuracy: number } export interface BacktestReport { generatedAt: string; trainEnd: string; + /** Test window start (validation ends here). */ + testFrom?: string; testTo: string; tested: number; params: { goalsPerElo: number; avgGoals: number; rho: number; homeAdvElo: number }; + /** Hyper-params tuned on the validation window (never on test). */ + validation?: { from: string; to: string; tuned: { xi: number; shrinkK: number; ensembleW: number } }; model: BacktestScores; + /** Bake-off: every candidate scored on the same untouched test window. */ + variants?: ({ name: string } & BacktestScores)[]; baselines: { uniform: BacktestScores; baseRate: BacktestScores; eloOnly: BacktestScores }; reliability: { predicted: number; observed: number; count: number }[]; ece: number; diff --git a/src/router.tsx b/src/router.tsx index 6f677c1..b922cfd 100644 --- a/src/router.tsx +++ b/src/router.tsx @@ -10,6 +10,7 @@ import { MatchPreviewPage } from './features/match/MatchPreviewPage'; import { TeamProfilePage } from './features/team/TeamProfilePage'; import { MethodologyPage } from './features/methodology/MethodologyPage'; import { TeamsPage } from './features/teams/TeamsPage'; +import { ScoreboardPage } from './features/scoreboard/ScoreboardPage'; const rootRoute = createRootRoute({ component: RootLayout, @@ -25,8 +26,9 @@ const matchRoute = createRoute({ getParentRoute: () => rootRoute, path: '/match/ const teamRoute = createRoute({ getParentRoute: () => rootRoute, path: '/team/$name', component: TeamProfilePage }); const methodologyRoute = createRoute({ getParentRoute: () => rootRoute, path: '/methodology', component: MethodologyPage }); const teamsRoute = createRoute({ getParentRoute: () => rootRoute, path: '/teams', component: TeamsPage }); +const scoreboardRoute = createRoute({ getParentRoute: () => rootRoute, path: '/scoreboard', component: ScoreboardPage }); -const routeTree = rootRoute.addChildren([indexRoute, groupsRoute, bracketRoute, predictRoute, storyRoute, matchRoute, teamRoute, methodologyRoute, teamsRoute]); +const routeTree = rootRoute.addChildren([indexRoute, groupsRoute, bracketRoute, predictRoute, storyRoute, matchRoute, teamRoute, methodologyRoute, teamsRoute, scoreboardRoute]); export const router = createRouter({ routeTree, defaultPreload: 'intent' });