v2 Phase 3: rigorous backtest + Methodology page

- scripts/buildBacktest.ts: honest walk-forward validation — params fit on pre-2018 internationals, tested out-of-sample on 7,988 matches (2018-2026) predicting each game from prior data only. Proper scoring (Brier/log-loss/RPS/ accuracy) vs uniform / base-rate / Elo-only baselines + a calibration analysis (reliability bins + ECE). Results: 60% accuracy, RPS 0.171 (beats all baselines), ECE 0.01 (excellent calibration). - Methodology page (/methodology): plain-language model walkthrough, the backtest scorecard, a custom-SVG reliability diagram, and honest limits. Transparency is the differentiator — no market odds, no overclaiming. - ReliabilityDiagram component; 'Model' nav entry. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-11 16:12:40 +02:00
parent 7f4838d032
commit 45c0a978fc
8 changed files with 391 additions and 3 deletions
@@ -1,6 +1,6 @@
 import { useEffect } from 'react';
 import { Link, Outlet } from '@tanstack/react-router';
-import { GitMerge, Moon, Radio, Sparkles, Sun, Table, TrendingUp, Trophy } from 'lucide-react';
+import { Gauge, GitMerge, Moon, Radio, Sparkles, Sun, Table, TrendingUp, Trophy } from 'lucide-react';
 import type { LucideIcon } from 'lucide-react';
 import { useUiStore } from '@/stores/uiStore';
 import { useTournamentStore } from '@/stores/tournamentStore';
@@ -11,6 +11,7 @@ const NAV: { to: string; label: string; exact: boolean; icon: LucideIcon }[] = [
  { to: '/groups', label: 'Groups', exact: false, icon: Table },
  { to: '/bracket', label: 'Bracket', exact: false, icon: GitMerge },
  { to: '/predict', label: 'Predict', exact: false, icon: TrendingUp },
+  { to: '/methodology', label: 'Model', exact: false, icon: Gauge },
  { to: '/story', label: 'Story', exact: false, icon: Sparkles },
 ];

@@ -0,0 +1,49 @@
+import { scaleLinear } from 'd3-scale';
+import type { BacktestReport } from '@/lib/types';
+
+const W = 360;
+const H = 360;
+const M = 34;
+
+/** Calibration plot: predicted probability (x) vs observed frequency (y). Points
+ *  on the diagonal = perfectly calibrated. Dot size ∝ sample count. */
+export function ReliabilityDiagram({ reliability, ece }: { reliability: BacktestReport['reliability']; ece: number }) {
+  const sx = scaleLinear().domain([0, 1]).range([M, W - 8]);
+  const sy = scaleLinear().domain([0, 1]).range([H - M, 8]);
+  const maxCount = Math.max(...reliability.map((b) => b.count), 1);
+  const pts = reliability.filter((b) => b.count > 0);
+
+  return (
+    <div>
+      <svg viewBox={`0 0 ${W} ${H}`} className="w-full max-w-sm">
+        {/* axes */}
+        <line x1={M} y1={H - M} x2={W - 8} y2={H - M} stroke="var(--app-line-strong)" strokeWidth={0.8} />
+        <line x1={M} y1={H - M} x2={M} y2={8} stroke="var(--app-line-strong)" strokeWidth={0.8} />
+        {/* perfect-calibration diagonal */}
+        <line x1={sx(0)} y1={sy(0)} x2={sx(1)} y2={sy(1)} stroke="var(--app-line)" strokeDasharray="4 4" strokeWidth={1} />
+        {[0.25, 0.5, 0.75, 1].map((t) => (
+          <g key={t}>
+            <text x={sx(t)} y={H - M + 14} textAnchor="middle" fontSize="10" fill="var(--app-faint)">{t}</text>
+            <text x={M - 6} y={sy(t) + 3} textAnchor="end" fontSize="10" fill="var(--app-faint)">{t}</text>
+          </g>
+        ))}
+        {/* model points + connecting line */}
+        <polyline
+          points={pts.map((b) => `${sx(b.predicted)},${sy(b.observed)}`).join(' ')}
+          fill="none" stroke="var(--app-accent)" strokeWidth={1.5} opacity={0.5}
+        />
+        {pts.map((b, i) => (
+          <circle key={i} cx={sx(b.predicted)} cy={sy(b.observed)} r={2 + 5 * Math.sqrt(b.count / maxCount)} fill="var(--app-accent)" fillOpacity={0.85}>
+            <title>{`predicted ${(b.predicted * 100).toFixed(0)}% → happened ${(b.observed * 100).toFixed(0)}% (${b.count})`}</title>
+          </circle>
+        ))}
+        <text x={(W + M) / 2} y={H - 4} textAnchor="middle" fontSize="11" fill="var(--app-muted)">predicted probability</text>
+        <text x={12} y={H / 2} textAnchor="middle" fontSize="11" fill="var(--app-muted)" transform={`rotate(-90 12 ${H / 2})`}>actually happened</text>
+      </svg>
+      <p className="mt-1 text-sm text-muted">
+        Points hug the diagonal — when the model says <span className="font-semibold text-ink">30%</span>, it happens about 30% of the time.
+        Calibration error (ECE) is just <span className="font-bold text-accent">{ece.toFixed(2)}</span> (lower is better; under 0.05 is excellent).
+      </p>
+    </div>
+  );
+}
@@ -0,0 +1,119 @@
+import { useEffect, useState } from 'react';
+import { Activity, Dices, Gauge, TrendingUp } from 'lucide-react';
+import { PageHeader } from '@/components/ui/PageHeader';
+import { Card, CardBody, CardHeader } from '@/components/ui/Card';
+import { ReliabilityDiagram } from '@/components/ReliabilityDiagram';
+import type { BacktestReport } from '@/lib/types';
+
+const STEPS = [
+  { icon: TrendingUp, title: 'Elo ratings', body: '150 years of international results (49,000 matches) build a strength rating for every nation, updated after each game and weighted by match importance.' },
+  { icon: Activity, title: 'Expected goals', body: "Each team's goal expectation comes from the rating gap, calibrated on how Elo differences have actually translated into goals — plus host home advantage." },
+  { icon: Gauge, title: 'Dixon-Coles', body: 'A bivariate-Poisson scoreline model (with the Dixon-Coles low-score correction) turns those goal expectations into the full distribution of scorelines and win/draw/loss.' },
+  { icon: Dices, title: 'Monte Carlo', body: 'The whole 48-team tournament is simulated 20,000 times — sampling every remaining match — to produce championship and advancement odds that update after each result.' },
+];
+
+function RpsBar({ label, rps, best, highlight }: { label: string; rps: number; best: number; highlight?: boolean }) {
+  // shorter bar = better (lower RPS). Scale relative to the worst (uniform ~0.24).
+  const width = Math.min(100, (rps / 0.26) * 100);
+  return (
+    <div className="flex items-center gap-3 text-sm">
+      <span className={`w-28 shrink-0 ${highlight ? 'font-bold text-ink' : 'text-muted'}`}>{label}</span>
+      <div className="h-4 flex-1 overflow-hidden rounded bg-elevated">
+        <div className={`h-full rounded ${highlight ? 'bg-gradient-to-r from-accent-deep to-accent' : 'bg-line-strong'}`} style={{ width: `${width}%` }} />
+      </div>
+      <span className={`tnum w-12 text-right ${highlight ? 'font-bold text-accent' : 'text-faint'}`}>{rps.toFixed(3)}</span>
+      {rps === best && <span className="text-[10px] font-bold uppercase text-accent">best</span>}
+    </div>
+  );
+}
+
+function Stat({ value, label }: { value: string; label: string }) {
+  return (
+    <div className="text-center">
+      <div className="font-display text-2xl font-extrabold text-ink">{value}</div>
+      <div className="text-[11px] uppercase tracking-wide text-faint">{label}</div>
+    </div>
+  );
+}
+
+export function MethodologyPage() {
+  const [bt, setBt] = useState<BacktestReport | null>(null);
+
+  useEffect(() => {
+    let alive = true;
+    fetch('/data/backtest.json').then((r) => r.json()).then((d: BacktestReport) => alive && setBt(d)).catch(() => {});
+    return () => { alive = false; };
+  }, []);
+
+  const bestRps = bt ? Math.min(bt.model.rps, bt.baselines.uniform.rps, bt.baselines.baseRate.rps, bt.baselines.eloOnly.rps) : 0;
+
+  return (
+    <div className="space-y-6">
+      <PageHeader title="Methodology" subtitle="How the model works — and, measured honestly, how good it actually is." />
+
+      <div className="grid gap-3 sm:grid-cols-2 lg:grid-cols-4">
+        {STEPS.map((s, i) => (
+          <Card key={s.title}>
+            <CardBody>
+              <div className="mb-2 flex items-center gap-2">
+                <span className="grid h-7 w-7 place-items-center rounded-md bg-accent-glow text-accent"><s.icon size={16} /></span>
+                <span className="text-xs font-bold text-faint">STEP {i + 1}</span>
+              </div>
+              <h3 className="font-display font-bold text-ink">{s.title}</h3>
+              <p className="mt-1 text-sm text-muted">{s.body}</p>
+            </CardBody>
+          </Card>
+        ))}
+      </div>
+
+      <Card>
+        <CardHeader><span className="font-display font-bold text-ink">How good is it? (out-of-sample backtest)</span></CardHeader>
+        <CardBody className="space-y-5">
+          {bt ? (
+            <>
+              <p className="text-sm text-muted">
+                Model parameters were fit on internationals before {bt.trainEnd.slice(0, 4)}, then tested <span className="font-semibold text-ink">walk-forward</span> on{' '}
+                <span className="font-semibold text-ink">{bt.tested.toLocaleString()}</span> matches it had never seen ({bt.trainEnd.slice(0, 4)}–{bt.testTo.slice(0, 4)}) — predicting each game using only prior data.
+              </p>
+              <div className="grid grid-cols-3 gap-3 rounded-lg border border-line bg-surface-2 py-3">
+                <Stat value={`${(bt.model.accuracy * 100).toFixed(0)}%`} label="outcome accuracy" />
+                <Stat value={bt.model.rps.toFixed(3)} label="ranked prob. score" />
+                <Stat value={`${(bt.worldCup.accuracy * 100).toFixed(0)}%`} label={`World Cup acc. (${bt.worldCup.tested})`} />
+              </div>
+              <div>
+                <div className="mb-2 text-xs uppercase tracking-wide text-faint">Ranked Probability Score — lower is better, vs baselines</div>
+                <div className="space-y-1.5">
+                  <RpsBar label="This model" rps={bt.model.rps} best={bestRps} highlight />
+                  <RpsBar label="Elo only" rps={bt.baselines.eloOnly.rps} best={bestRps} />
+                  <RpsBar label="Base rates" rps={bt.baselines.baseRate.rps} best={bestRps} />
+                  <RpsBar label="Coin flip" rps={bt.baselines.uniform.rps} best={bestRps} />
+                </div>
+              </div>
+            </>
+          ) : (
+            <div className="h-32 animate-pulse rounded bg-elevated" />
+          )}
+        </CardBody>
+      </Card>
+
+      <Card>
+        <CardHeader><span className="font-display font-bold text-ink">Is it calibrated?</span></CardHeader>
+        <CardBody>
+          {bt ? <ReliabilityDiagram reliability={bt.reliability} ece={bt.ece} /> : <div className="h-64 animate-pulse rounded bg-elevated" />}
+        </CardBody>
+      </Card>
+
+      <Card>
+        <CardHeader><span className="font-display font-bold text-ink">Honest limits</span></CardHeader>
+        <CardBody>
+          <ul className="space-y-2 text-sm text-muted">
+            <li>• These are <span className="font-semibold text-ink">model probabilities, not betting advice</span>. We don't use bookmaker odds, so we make no claim to beat the market — sharp betting markets remain the most accurate forecaster there is.</li>
+            <li>• International expected-goals and event data are sparse, so per-match tactical breakdowns are richer for some teams than others, and improve as live data accumulates during the tournament.</li>
+            <li>• Football is high-variance: a 60% favourite still loses plenty. Calibration means our 60% really is ~60% — not that the favourite always wins.</li>
+            <li>• Everything is transparent and reproducible from public data (results, ESPN, StatsBomb) — that openness is the point.</li>
+          </ul>
+        </CardBody>
+      </Card>
+    </div>
+  );
+}
@@ -160,6 +160,22 @@ export interface TeamProfile {
  qualifyOdds: number | null;
 }

+// ---- model backtest (Methodology page) ----
+
+export interface BacktestScores { brier: number; logloss: number; rps: number; accuracy: number }
+export interface BacktestReport {
+  generatedAt: string;
+  trainEnd: string;
+  testTo: string;
+  tested: number;
+  params: { goalsPerElo: number; avgGoals: number; rho: number; homeAdvElo: number };
+  model: BacktestScores;
+  baselines: { uniform: BacktestScores; baseRate: BacktestScores; eloOnly: BacktestScores };
+  reliability: { predicted: number; observed: number; count: number }[];
+  ece: number;
+  worldCup: { tested: number; accuracy: number };
+}
+
 // ---- data-story viz (StatsBomb open data, a fixed historical match) ----

 export interface VizShot {
@@ -8,6 +8,7 @@ import { PredictionsPage } from './features/predictions/PredictionsPage';
 import { StoryPage } from './features/story/StoryPage';
 import { MatchPreviewPage } from './features/match/MatchPreviewPage';
 import { TeamProfilePage } from './features/team/TeamProfilePage';
+import { MethodologyPage } from './features/methodology/MethodologyPage';

 const rootRoute = createRootRoute({
  component: RootLayout,
@@ -21,8 +22,9 @@ const predictRoute = createRoute({ getParentRoute: () => rootRoute, path: '/pred
 const storyRoute = createRoute({ getParentRoute: () => rootRoute, path: '/story', component: StoryPage });
 const matchRoute = createRoute({ getParentRoute: () => rootRoute, path: '/match/$num', component: MatchPreviewPage });
 const teamRoute = createRoute({ getParentRoute: () => rootRoute, path: '/team/$name', component: TeamProfilePage });
+const methodologyRoute = createRoute({ getParentRoute: () => rootRoute, path: '/methodology', component: MethodologyPage });

-const routeTree = rootRoute.addChildren([indexRoute, groupsRoute, bracketRoute, predictRoute, storyRoute, matchRoute, teamRoute]);
+const routeTree = rootRoute.addChildren([indexRoute, groupsRoute, bracketRoute, predictRoute, storyRoute, matchRoute, teamRoute, methodologyRoute]);

 export const router = createRouter({ routeTree, defaultPreload: 'intent' });