NilsBriggen
acd8c3e75d
Five two-year walk-forward folds (2014-2024), each with its own pre-fold parameter fit; selection by mean fold RPS; staged greedy over Team-DC decay/shrinkage/weight, fast-Elo blend, form multiplier, and new Elo-walk axes (home advantage, K scale); untouched 2024-26 final test. Result: the shipped v4 config holds — the staged champion gained 0.0003, under the 0.0005 ship bar, and the final test agrees. v4 is no longer a one-window result. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
382 lines
16 KiB
TypeScript
382 lines
16 KiB
TypeScript
// v5 hyperparameter search: rolling-origin cross-validation.
|
||
//
|
||
// The v4 protocol tuned on ONE validation window (2018–2022) — selection could
|
||
// overfit that window's quirks. v5 selects by MEAN walk-forward RPS across five
|
||
// rolling two-year folds, each with its own pre-fold parameter fit (no fold
|
||
// ever sees information from its future):
|
||
// F1 2014–16 · F2 2016–18 · F3 2018–20 · F4 2020–22 · F5 2022–24
|
||
// FINAL TEST: 2024-01-01 → 2026-06-01, untouched by selection, evaluated once.
|
||
//
|
||
// Search axes (staged greedy, like v4 — full grid is combinatorially silly):
|
||
// stage 1: Team-DC ξ (decay), k (shrinkage), w (ensemble weight)
|
||
// stage 2: fast-Elo blend (m, β)
|
||
// stage 3: recent-form goal multiplier γ
|
||
// stage 4: Elo-walk axes — HOME_ADV_ELO and K-scale σ
|
||
// Ship bar (pre-registered): champion must beat the current shipped config by
|
||
// ≥ MIN_GAIN mean-fold RPS. Output → data/cv-report.json (research artifact,
|
||
// not public; the public evidence stays scripts/buildBacktest.ts).
|
||
//
|
||
// Self-contained on purpose: this is a research harness, and keeping it apart
|
||
// from the publication script means it can never destabilize shipped evidence.
|
||
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
||
import { fileURLToPath } from 'node:url';
|
||
import { dirname, join } from 'node:path';
|
||
import { canonicalTeam } from '../src/lib/teams';
|
||
import { importanceWeight, eloDelta, expectedHome } from '../src/lib/model/elo';
|
||
import { lambdasFromElo, scoreMatrix, outcomeProbs, poissonPmf, type ModelParams } from '../src/lib/model/poisson';
|
||
import { fitTeamDc, teamDcLambdas, poolMatrices, type DcMatch, type TeamDcParams } from '../src/lib/model/teamDc';
|
||
|
||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||
const CSV = join(ROOT, 'data', 'raw', 'international-results.csv');
|
||
const OUT = join(ROOT, 'data', 'cv-report.json');
|
||
|
||
const START_ELO = 1500;
|
||
const REFIT_DAYS = 60;
|
||
const FIT_WINDOW_YEARS = 15;
|
||
const PARAM_YEARS = 12; // params fit on the 12 years before each fold/test
|
||
const DAY = 24 * 60 * 60 * 1000;
|
||
const MIN_GAIN = 0.0005;
|
||
|
||
const FOLDS = [
|
||
{ from: '2014-01-01', to: '2016-01-01' },
|
||
{ from: '2016-01-01', to: '2018-01-01' },
|
||
{ from: '2018-01-01', to: '2020-01-01' },
|
||
{ from: '2020-01-01', to: '2022-01-01' },
|
||
{ from: '2022-01-01', to: '2024-01-01' },
|
||
];
|
||
const TEST = { from: '2024-01-01', to: '2026-06-01' };
|
||
|
||
const XI_GRID = [0.25, 0.5, 0.75, 1, 1.5, 2];
|
||
const K_GRID = [3, 5, 10, 20];
|
||
const M_GRID = [2, 3];
|
||
const BETA_GRID = [0.1, 0.2, 0.3, 0.4];
|
||
const GAMMA_GRID = [0.05, 0.1, 0.15];
|
||
const HOMEADV_GRID = [50, 70, 90];
|
||
const KSCALE_GRID = [0.75, 1, 1.25];
|
||
const FORM_WINDOW = 10;
|
||
|
||
/** the currently shipped v4 config — the bar to clear (homeAdv 70, σ 1) */
|
||
const SHIPPED = { xi: 0.25, k: 3, w: 0.45, m: 3, beta: 0.3, gamma: 0, homeAdv: 70, kScale: 1 };
|
||
|
||
interface Row { date: string; home: string; away: string; hs: number; as: number; tournament: string; neutral: boolean; t: number }
|
||
type Probs = { h: number; d: number; a: number };
|
||
type Outcome = 'h' | 'd' | 'a';
|
||
|
||
function parse(): Row[] {
|
||
const lines = readFileSync(CSV, 'utf8').split('\n');
|
||
const rows: Row[] = [];
|
||
for (let i = 1; i < lines.length; i++) {
|
||
const c = lines[i]?.split(',');
|
||
if (!c || c.length < 9) continue;
|
||
const hs = Number(c[3]); const as = Number(c[4]);
|
||
if (!Number.isFinite(hs) || !Number.isFinite(as)) continue;
|
||
rows.push({
|
||
date: c[0]!, home: canonicalTeam(c[1]!), away: canonicalTeam(c[2]!), hs, as,
|
||
tournament: c[5]!, neutral: c[8]!.trim().toUpperCase() === 'TRUE',
|
||
t: new Date(c[0]!).getTime(),
|
||
});
|
||
}
|
||
rows.sort((a, b) => a.t - b.t);
|
||
return rows;
|
||
}
|
||
|
||
function dcTau(h: number, a: number, lh: number, la: number, rho: number): number {
|
||
if (h === 0 && a === 0) return 1 - lh * la * rho;
|
||
if (h === 0 && a === 1) return 1 + lh * rho;
|
||
if (h === 1 && a === 0) return 1 + la * rho;
|
||
if (h === 1 && a === 1) return 1 - rho;
|
||
return 1;
|
||
}
|
||
|
||
const outcomeOf = (hs: number, as: number): Outcome => (hs > as ? 'h' : hs < as ? 'a' : 'd');
|
||
const probsOf = (m: number[][]): Probs => { const p = outcomeProbs(m); return { h: p.home, d: p.draw, a: p.away }; };
|
||
function rps(p: Probs, o: Outcome): number {
|
||
const y1 = o === 'h' ? 1 : 0;
|
||
const y2 = o === 'a' ? 0 : 1;
|
||
return ((p.h - y1) ** 2 + (p.h + p.d - y2) ** 2) / 2;
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Elo walk variants: one pass per (homeAdv, kScale), storing pre-match slow +
|
||
// fast ratings and the pre-match form signal (mean Elo surprise, last 10
|
||
// non-friendly). Everything is strictly pre-match by construction.
|
||
// ---------------------------------------------------------------------------
|
||
interface Walk {
|
||
preH: Float64Array; preA: Float64Array; // slow Elo, pre-match
|
||
fast: Map<number, { h: Float64Array; a: Float64Array }>; // per m
|
||
formH: Float64Array; formA: Float64Array;
|
||
}
|
||
|
||
function runWalk(rows: Row[], homeAdvElo: number, kScale: number): Walk {
|
||
const n = rows.length;
|
||
const w: Walk = {
|
||
preH: new Float64Array(n), preA: new Float64Array(n),
|
||
fast: new Map(M_GRID.map((m) => [m, { h: new Float64Array(n), a: new Float64Array(n) }])),
|
||
formH: new Float64Array(n), formA: new Float64Array(n),
|
||
};
|
||
const elo = new Map<string, number>();
|
||
const eloFast = new Map<number, Map<string, number>>(M_GRID.map((m) => [m, new Map()]));
|
||
const formArr = new Map<string, number[]>();
|
||
const get = (map: Map<string, number>, t: string) => map.get(t) ?? START_ELO;
|
||
const formMean = (t: string) => {
|
||
const a = formArr.get(t);
|
||
return a && a.length ? a.reduce((s, x) => s + x, 0) / a.length : 0;
|
||
};
|
||
for (let i = 0; i < n; i++) {
|
||
const r = rows[i]!;
|
||
const adv = r.neutral ? 0 : homeAdvElo;
|
||
const eh = get(elo, r.home), ea = get(elo, r.away);
|
||
w.preH[i] = eh; w.preA[i] = ea;
|
||
for (const m of M_GRID) {
|
||
const fm = eloFast.get(m)!;
|
||
const arr = w.fast.get(m)!;
|
||
const feh = get(fm, r.home), fea = get(fm, r.away);
|
||
arr.h[i] = feh; arr.a[i] = fea;
|
||
const fd = eloDelta(r.hs, r.as, feh, fea, importanceWeight(r.tournament) * kScale * m, adv);
|
||
fm.set(r.home, feh + fd); fm.set(r.away, fea - fd);
|
||
}
|
||
w.formH[i] = formMean(r.home);
|
||
w.formA[i] = formMean(r.away);
|
||
if (r.tournament !== 'Friendly') {
|
||
const eH = expectedHome(eh, ea, adv);
|
||
const sH = r.hs > r.as ? 1 : r.hs < r.as ? 0 : 0.5;
|
||
const push = (t: string, v: number) => {
|
||
const a = formArr.get(t) ?? [];
|
||
a.push(v);
|
||
if (a.length > FORM_WINDOW) a.shift();
|
||
formArr.set(t, a);
|
||
};
|
||
push(r.home, sH - eH);
|
||
push(r.away, (1 - sH) - (1 - eH));
|
||
}
|
||
const d = eloDelta(r.hs, r.as, eh, ea, importanceWeight(r.tournament) * kScale, adv);
|
||
elo.set(r.home, eh + d); elo.set(r.away, ea - d);
|
||
}
|
||
return w;
|
||
}
|
||
|
||
/** Fit goalsPerElo/avgGoals/rho on the PARAM_YEARS before `fromT` (pre-fold only). */
|
||
function fitParams(rows: Row[], walk: Walk, homeAdvElo: number, fromT: number): ModelParams {
|
||
const startT = fromT - PARAM_YEARS * 365 * DAY;
|
||
let sum = 0, n = 0, sxy = 0, sxx = 0;
|
||
const calib: { d: number; h: number; a: number }[] = [];
|
||
for (let i = 0; i < rows.length; i++) {
|
||
const r = rows[i]!;
|
||
if (r.t < startT) continue;
|
||
if (r.t >= fromT) break;
|
||
const eff = walk.preH[i]! - walk.preA[i]! + (r.neutral ? 0 : homeAdvElo);
|
||
sum += r.hs + r.as; n++; sxy += eff * (r.hs - r.as); sxx += eff * eff;
|
||
calib.push({ d: eff, h: r.hs, a: r.as });
|
||
}
|
||
const avgGoals = sum / Math.max(1, n);
|
||
const goalsPerElo = sxy / Math.max(1e-9, sxx);
|
||
let rho0 = -0.05, bestLL = -Infinity;
|
||
for (let r0 = -0.2; r0 <= 0.1 + 1e-9; r0 += 0.02) {
|
||
let ll = 0;
|
||
for (const m of calib) {
|
||
const sup = goalsPerElo * m.d;
|
||
const lh = Math.min(8, Math.max(0.15, avgGoals / 2 + sup / 2));
|
||
const la = Math.min(8, Math.max(0.15, avgGoals / 2 - sup / 2));
|
||
ll += Math.log(Math.max(1e-12, poissonPmf(lh, m.h) * poissonPmf(la, m.a) * dcTau(m.h, m.a, lh, la, r0)));
|
||
}
|
||
if (ll > bestLL) { bestLL = ll; rho0 = +r0.toFixed(2); }
|
||
}
|
||
return { goalsPerElo, avgGoals, rho: rho0, homeAdvElo };
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Walk-forward Team-DC lambdas per fold per (ξ, k) — the expensive part, cached.
|
||
// ---------------------------------------------------------------------------
|
||
type Pred = { i: number; lDcH: number; lDcA: number };
|
||
|
||
function walkTeamDc(rows: Row[], xi: number, k: number, fromT: number, toT: number): Pred[] {
|
||
const preds: Pred[] = [];
|
||
let fit: TeamDcParams | null = null;
|
||
let fitAt = -Infinity;
|
||
const windowMs = FIT_WINDOW_YEARS * 365 * DAY;
|
||
for (let i = 0; i < rows.length; i++) {
|
||
const r = rows[i]!;
|
||
if (r.t < fromT) continue;
|
||
if (r.t >= toT) break;
|
||
if (r.t - fitAt > REFIT_DAYS * DAY) {
|
||
const train: DcMatch[] = [];
|
||
for (let j = 0; j < i; j++) {
|
||
const m = rows[j]!;
|
||
if (r.t - m.t > windowMs) continue;
|
||
train.push({
|
||
daysAgo: (r.t - m.t) / DAY, home: m.home, away: m.away,
|
||
homeGoals: m.hs, awayGoals: m.as, neutral: m.neutral,
|
||
});
|
||
}
|
||
fit = fitTeamDc(train, xi, k);
|
||
fitAt = r.t;
|
||
}
|
||
const { lambdaHome, lambdaAway } = teamDcLambdas(fit!, r.home, r.away, r.neutral);
|
||
preds.push({ i, lDcH: lambdaHome, lDcA: lambdaAway });
|
||
}
|
||
return preds;
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Scoring: mean RPS over one fold for a full config, sweeping w cheaply.
|
||
// ---------------------------------------------------------------------------
|
||
const W_STEPS: number[] = [];
|
||
for (let w = 0; w <= 1.0001; w += 0.05) W_STEPS.push(+w.toFixed(2));
|
||
|
||
const clampL = (x: number) => Math.min(8, Math.max(0.1, x));
|
||
|
||
interface Cfg { xi: number; k: number; w: number; m: number; beta: number; gamma: number; homeAdv: number; kScale: number }
|
||
|
||
function sweepW(
|
||
rows: Row[], preds: Pred[], walk: Walk, params: ModelParams,
|
||
beta: number, m: number, gamma: number,
|
||
): { w: number; rps: number }[] {
|
||
const acc = new Float64Array(W_STEPS.length);
|
||
const fast = walk.fast.get(m)!;
|
||
for (const p of preds) {
|
||
const r = rows[p.i]!;
|
||
const o = outcomeOf(r.hs, r.as);
|
||
const adv = r.neutral ? 0 : params.homeAdvElo;
|
||
const eh = (1 - beta) * walk.preH[p.i]! + beta * fast.h[p.i]!;
|
||
const ea = (1 - beta) * walk.preA[p.i]! + beta * fast.a[p.i]!;
|
||
let { lambdaHome: elh, lambdaAway: ela } = lambdasFromElo(eh, ea, params, adv);
|
||
let dlh = p.lDcH, dla = p.lDcA;
|
||
if (gamma > 0) {
|
||
const fh = Math.exp(gamma * walk.formH[p.i]!), fa = Math.exp(gamma * walk.formA[p.i]!);
|
||
elh = clampL(elh * fh); ela = clampL(ela * fa);
|
||
dlh = clampL(dlh * fh); dla = clampL(dla * fa);
|
||
}
|
||
const mE = scoreMatrix(elh, ela, params.rho);
|
||
const mD = scoreMatrix(dlh, dla, params.rho);
|
||
for (let wi = 0; wi < W_STEPS.length; wi++) {
|
||
const w = W_STEPS[wi]!;
|
||
const pooled = w === 0 ? mD : w === 1 ? mE : poolMatrices(mE, mD, w);
|
||
acc[wi] += rps(probsOf(pooled), o);
|
||
}
|
||
}
|
||
return W_STEPS.map((w, wi) => ({ w, rps: acc[wi]! / Math.max(1, preds.length) }));
|
||
}
|
||
|
||
function main(): void {
|
||
const t0 = Date.now();
|
||
const rows = parse();
|
||
console.log(`${rows.length} rows · ${FOLDS.length} folds · test ${TEST.from}→${TEST.to}`);
|
||
|
||
const foldTs = FOLDS.map((f) => ({ fromT: new Date(f.from).getTime(), toT: new Date(f.to).getTime() }));
|
||
|
||
// Elo walks + per-fold params, per (homeAdv, kScale) — cheap, precompute all.
|
||
const walks = new Map<string, Walk>();
|
||
const foldParams = new Map<string, ModelParams[]>(); // key → params per fold
|
||
for (const ha of HOMEADV_GRID) {
|
||
for (const ks of KSCALE_GRID) {
|
||
const key = `${ha}|${ks}`;
|
||
const w = runWalk(rows, ha, ks);
|
||
walks.set(key, w);
|
||
foldParams.set(key, foldTs.map((f) => fitParams(rows, w, ha, f.fromT)));
|
||
}
|
||
}
|
||
console.log(`walks + per-fold params ready (${((Date.now() - t0) / 1000).toFixed(0)}s)`);
|
||
|
||
const baseKey = `${SHIPPED.homeAdv}|${SHIPPED.kScale}`;
|
||
const baseWalk = walks.get(baseKey)!;
|
||
const baseParams = foldParams.get(baseKey)!;
|
||
|
||
/** mean fold RPS for (xi,k) preds under a config, returning per-w means */
|
||
const dcCache = new Map<string, Pred[][]>(); // `${xi}|${k}` → preds per fold
|
||
const foldPreds = (xi: number, k: number): Pred[][] => {
|
||
const key = `${xi}|${k}`;
|
||
let v = dcCache.get(key);
|
||
if (!v) {
|
||
v = foldTs.map((f) => walkTeamDc(rows, xi, k, f.fromT, f.toT));
|
||
dcCache.set(key, v);
|
||
}
|
||
return v;
|
||
};
|
||
const meanSweep = (xi: number, k: number, beta: number, m: number, gamma: number, walk = baseWalk, params = baseParams): { w: number; rps: number }[] => {
|
||
const folds = foldPreds(xi, k);
|
||
const acc = new Float64Array(W_STEPS.length);
|
||
for (let fi = 0; fi < folds.length; fi++) {
|
||
const res = sweepW(rows, folds[fi]!, walk, params[fi]!, beta, m, gamma);
|
||
for (let wi = 0; wi < W_STEPS.length; wi++) acc[wi] += res[wi]!.rps;
|
||
}
|
||
return W_STEPS.map((w, wi) => ({ w, rps: acc[wi]! / folds.length }));
|
||
};
|
||
|
||
// ---- control: shipped config under the CV protocol ----
|
||
const shippedMean = meanSweep(SHIPPED.xi, SHIPPED.k, SHIPPED.beta, SHIPPED.m, SHIPPED.gamma)
|
||
.find((x) => x.w === SHIPPED.w)!.rps;
|
||
console.log(`shipped v4 config mean-fold rps ${shippedMean.toFixed(4)}`);
|
||
|
||
// ---- stage 1: ξ, k, w ----
|
||
let best: Cfg & { rps: number } = { ...SHIPPED, rps: Infinity };
|
||
for (const xi of XI_GRID) {
|
||
for (const k of K_GRID) {
|
||
for (const { w, rps: r } of meanSweep(xi, k, 0, 2, 0)) {
|
||
if (r < best.rps) best = { ...SHIPPED, xi, k, w, m: 2, beta: 0, gamma: 0, rps: r };
|
||
}
|
||
console.log(` s1 ξ=${xi} k=${k} (best ξ=${best.xi} k=${best.k} w=${best.w} rps=${best.rps.toFixed(4)})`);
|
||
}
|
||
}
|
||
|
||
// ---- stage 2: fast-Elo m, β ----
|
||
for (const m of M_GRID) {
|
||
for (const beta of BETA_GRID) {
|
||
for (const { w, rps: r } of meanSweep(best.xi, best.k, beta, m, 0)) {
|
||
if (r < best.rps) best = { ...best, m, beta, w, rps: r };
|
||
}
|
||
}
|
||
}
|
||
console.log(` s2 best: m=${best.m} β=${best.beta} w=${best.w} rps=${best.rps.toFixed(4)}`);
|
||
|
||
// ---- stage 3: form γ ----
|
||
for (const gamma of GAMMA_GRID) {
|
||
for (const { w, rps: r } of meanSweep(best.xi, best.k, best.beta, best.m, gamma)) {
|
||
if (r < best.rps) best = { ...best, gamma, w, rps: r };
|
||
}
|
||
}
|
||
console.log(` s3 best: γ=${best.gamma} w=${best.w} rps=${best.rps.toFixed(4)}`);
|
||
|
||
// ---- stage 4: Elo-walk axes (homeAdv, kScale) ----
|
||
for (const ha of HOMEADV_GRID) {
|
||
for (const ks of KSCALE_GRID) {
|
||
if (ha === SHIPPED.homeAdv && ks === SHIPPED.kScale && best.rps < Infinity) continue;
|
||
const key = `${ha}|${ks}`;
|
||
for (const { w, rps: r } of meanSweep(best.xi, best.k, best.beta, best.m, best.gamma, walks.get(key)!, foldParams.get(key)!)) {
|
||
if (r < best.rps) best = { ...best, homeAdv: ha, kScale: ks, w, rps: r };
|
||
}
|
||
}
|
||
}
|
||
console.log(` s4 best: homeAdv=${best.homeAdv} σ=${best.kScale} w=${best.w} rps=${best.rps.toFixed(4)}`);
|
||
|
||
const gain = shippedMean - best.rps;
|
||
const ship = gain >= MIN_GAIN;
|
||
console.log(`champion mean-fold rps ${best.rps.toFixed(4)} vs shipped ${shippedMean.toFixed(4)} → gain ${gain.toFixed(4)} → ${ship ? 'SHIP' : 'KEEP shipped'}`);
|
||
|
||
// ---- final test (2024–26), evaluated once for champion + shipped ----
|
||
const testFromT = new Date(TEST.from).getTime();
|
||
const testToT = new Date(TEST.to).getTime();
|
||
const testEval = (cfg: Cfg): number => {
|
||
const key = `${cfg.homeAdv}|${cfg.kScale}`;
|
||
const walk = walks.get(key)!;
|
||
const params = fitParams(rows, walk, cfg.homeAdv, testFromT);
|
||
const preds = walkTeamDc(rows, cfg.xi, cfg.k, testFromT, testToT);
|
||
return sweepW(rows, preds, walk, params, cfg.beta, cfg.m, cfg.gamma).find((x) => x.w === cfg.w)!.rps;
|
||
};
|
||
const champTest = testEval(best);
|
||
const shippedTest = testEval(SHIPPED);
|
||
console.log(`TEST 2024–26: champion rps ${champTest.toFixed(4)} | shipped rps ${shippedTest.toFixed(4)}`);
|
||
|
||
mkdirSync(dirname(OUT), { recursive: true });
|
||
writeFileSync(OUT, JSON.stringify({
|
||
generatedAt: new Date().toISOString(),
|
||
protocol: { folds: FOLDS, test: TEST, paramYears: PARAM_YEARS, minGain: MIN_GAIN, selection: 'mean fold RPS, staged greedy' },
|
||
shipped: { ...SHIPPED, meanFoldRps: +shippedMean.toFixed(4), testRps: +shippedTest.toFixed(4) },
|
||
champion: { ...best, rps: undefined, meanFoldRps: +best.rps.toFixed(4), testRps: +champTest.toFixed(4) },
|
||
shipChange: ship,
|
||
runtimeSec: Math.round((Date.now() - t0) / 1000),
|
||
}, null, 2));
|
||
console.log(`report → data/cv-report.json (${Math.round((Date.now() - t0) / 1000)}s total)`);
|
||
}
|
||
|
||
main();
|