v2 Phase 3: rigorous backtest + Methodology page

- scripts/buildBacktest.ts: honest walk-forward validation — params fit on pre-2018 internationals, tested out-of-sample on 7,988 matches (2018-2026) predicting each game from prior data only. Proper scoring (Brier/log-loss/RPS/ accuracy) vs uniform / base-rate / Elo-only baselines + a calibration analysis (reliability bins + ECE). Results: 60% accuracy, RPS 0.171 (beats all baselines), ECE 0.01 (excellent calibration). - Methodology page (/methodology): plain-language model walkthrough, the backtest scorecard, a custom-SVG reliability diagram, and honest limits. Transparency is the differentiator — no market odds, no overclaiming. - ReliabilityDiagram component; 'Model' nav entry. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-11 16:12:40 +02:00
parent 7f4838d032
commit 45c0a978fc
8 changed files with 391 additions and 3 deletions
@@ -0,0 +1,49 @@
+import { scaleLinear } from 'd3-scale';
+import type { BacktestReport } from '@/lib/types';
+
+const W = 360;
+const H = 360;
+const M = 34;
+
+/** Calibration plot: predicted probability (x) vs observed frequency (y). Points
+ *  on the diagonal = perfectly calibrated. Dot size ∝ sample count. */
+export function ReliabilityDiagram({ reliability, ece }: { reliability: BacktestReport['reliability']; ece: number }) {
+  const sx = scaleLinear().domain([0, 1]).range([M, W - 8]);
+  const sy = scaleLinear().domain([0, 1]).range([H - M, 8]);
+  const maxCount = Math.max(...reliability.map((b) => b.count), 1);
+  const pts = reliability.filter((b) => b.count > 0);
+
+  return (
+    <div>
+      <svg viewBox={`0 0 ${W} ${H}`} className="w-full max-w-sm">
+        {/* axes */}
+        <line x1={M} y1={H - M} x2={W - 8} y2={H - M} stroke="var(--app-line-strong)" strokeWidth={0.8} />
+        <line x1={M} y1={H - M} x2={M} y2={8} stroke="var(--app-line-strong)" strokeWidth={0.8} />
+        {/* perfect-calibration diagonal */}
+        <line x1={sx(0)} y1={sy(0)} x2={sx(1)} y2={sy(1)} stroke="var(--app-line)" strokeDasharray="4 4" strokeWidth={1} />
+        {[0.25, 0.5, 0.75, 1].map((t) => (
+          <g key={t}>
+            <text x={sx(t)} y={H - M + 14} textAnchor="middle" fontSize="10" fill="var(--app-faint)">{t}</text>
+            <text x={M - 6} y={sy(t) + 3} textAnchor="end" fontSize="10" fill="var(--app-faint)">{t}</text>
+          </g>
+        ))}
+        {/* model points + connecting line */}
+        <polyline
+          points={pts.map((b) => `${sx(b.predicted)},${sy(b.observed)}`).join(' ')}
+          fill="none" stroke="var(--app-accent)" strokeWidth={1.5} opacity={0.5}
+        />
+        {pts.map((b, i) => (
+          <circle key={i} cx={sx(b.predicted)} cy={sy(b.observed)} r={2 + 5 * Math.sqrt(b.count / maxCount)} fill="var(--app-accent)" fillOpacity={0.85}>
+            <title>{`predicted ${(b.predicted * 100).toFixed(0)}% → happened ${(b.observed * 100).toFixed(0)}% (${b.count})`}</title>
+          </circle>
+        ))}
+        <text x={(W + M) / 2} y={H - 4} textAnchor="middle" fontSize="11" fill="var(--app-muted)">predicted probability</text>
+        <text x={12} y={H / 2} textAnchor="middle" fontSize="11" fill="var(--app-muted)" transform={`rotate(-90 12 ${H / 2})`}>actually happened</text>
+      </svg>
+      <p className="mt-1 text-sm text-muted">
+        Points hug the diagonal — when the model says <span className="font-semibold text-ink">30%</span>, it happens about 30% of the time.
+        Calibration error (ECE) is just <span className="font-bold text-accent">{ece.toFixed(2)}</span> (lower is better; under 0.05 is excellent).
+      </p>
+    </div>
+  );
+}