NilsBriggen
45c0a978fc
- scripts/buildBacktest.ts: honest walk-forward validation — params fit on pre-2018 internationals, tested out-of-sample on 7,988 matches (2018-2026) predicting each game from prior data only. Proper scoring (Brier/log-loss/RPS/ accuracy) vs uniform / base-rate / Elo-only baselines + a calibration analysis (reliability bins + ECE). Results: 60% accuracy, RPS 0.171 (beats all baselines), ECE 0.01 (excellent calibration). - Methodology page (/methodology): plain-language model walkthrough, the backtest scorecard, a custom-SVG reliability diagram, and honest limits. Transparency is the differentiator — no market odds, no overclaiming. - ReliabilityDiagram component; 'Model' nav entry. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
50 lines
2.6 KiB
TypeScript
50 lines
2.6 KiB
TypeScript
import { scaleLinear } from 'd3-scale';
|
|
import type { BacktestReport } from '@/lib/types';
|
|
|
|
const W = 360;
|
|
const H = 360;
|
|
const M = 34;
|
|
|
|
/** Calibration plot: predicted probability (x) vs observed frequency (y). Points
|
|
* on the diagonal = perfectly calibrated. Dot size ∝ sample count. */
|
|
export function ReliabilityDiagram({ reliability, ece }: { reliability: BacktestReport['reliability']; ece: number }) {
|
|
const sx = scaleLinear().domain([0, 1]).range([M, W - 8]);
|
|
const sy = scaleLinear().domain([0, 1]).range([H - M, 8]);
|
|
const maxCount = Math.max(...reliability.map((b) => b.count), 1);
|
|
const pts = reliability.filter((b) => b.count > 0);
|
|
|
|
return (
|
|
<div>
|
|
<svg viewBox={`0 0 ${W} ${H}`} className="w-full max-w-sm">
|
|
{/* axes */}
|
|
<line x1={M} y1={H - M} x2={W - 8} y2={H - M} stroke="var(--app-line-strong)" strokeWidth={0.8} />
|
|
<line x1={M} y1={H - M} x2={M} y2={8} stroke="var(--app-line-strong)" strokeWidth={0.8} />
|
|
{/* perfect-calibration diagonal */}
|
|
<line x1={sx(0)} y1={sy(0)} x2={sx(1)} y2={sy(1)} stroke="var(--app-line)" strokeDasharray="4 4" strokeWidth={1} />
|
|
{[0.25, 0.5, 0.75, 1].map((t) => (
|
|
<g key={t}>
|
|
<text x={sx(t)} y={H - M + 14} textAnchor="middle" fontSize="10" fill="var(--app-faint)">{t}</text>
|
|
<text x={M - 6} y={sy(t) + 3} textAnchor="end" fontSize="10" fill="var(--app-faint)">{t}</text>
|
|
</g>
|
|
))}
|
|
{/* model points + connecting line */}
|
|
<polyline
|
|
points={pts.map((b) => `${sx(b.predicted)},${sy(b.observed)}`).join(' ')}
|
|
fill="none" stroke="var(--app-accent)" strokeWidth={1.5} opacity={0.5}
|
|
/>
|
|
{pts.map((b, i) => (
|
|
<circle key={i} cx={sx(b.predicted)} cy={sy(b.observed)} r={2 + 5 * Math.sqrt(b.count / maxCount)} fill="var(--app-accent)" fillOpacity={0.85}>
|
|
<title>{`predicted ${(b.predicted * 100).toFixed(0)}% → happened ${(b.observed * 100).toFixed(0)}% (${b.count})`}</title>
|
|
</circle>
|
|
))}
|
|
<text x={(W + M) / 2} y={H - 4} textAnchor="middle" fontSize="11" fill="var(--app-muted)">predicted probability</text>
|
|
<text x={12} y={H / 2} textAnchor="middle" fontSize="11" fill="var(--app-muted)" transform={`rotate(-90 12 ${H / 2})`}>actually happened</text>
|
|
</svg>
|
|
<p className="mt-1 text-sm text-muted">
|
|
Points hug the diagonal — when the model says <span className="font-semibold text-ink">30%</span>, it happens about 30% of the time.
|
|
Calibration error (ECE) is just <span className="font-bold text-accent">{ece.toFixed(2)}</span> (lower is better; under 0.05 is excellent).
|
|
</p>
|
|
</div>
|
|
);
|
|
}
|