From acd8c3e75d41f05970b7820cfb30e4a7ca30363d Mon Sep 17 00:00:00 2001 From: Nils Briggen Date: Fri, 12 Jun 2026 00:41:37 +0200 Subject: [PATCH] v5 model protocol: rolling-origin CV search harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five two-year walk-forward folds (2014-2024), each with its own pre-fold parameter fit; selection by mean fold RPS; staged greedy over Team-DC decay/shrinkage/weight, fast-Elo blend, form multiplier, and new Elo-walk axes (home advantage, K scale); untouched 2024-26 final test. Result: the shipped v4 config holds — the staged champion gained 0.0003, under the 0.0005 ship bar, and the final test agrees. v4 is no longer a one-window result. Co-Authored-By: Claude Fable 5 --- data/cv-report.json | 60 +++++++ scripts/cvSearch.ts | 381 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 441 insertions(+) create mode 100644 data/cv-report.json create mode 100644 scripts/cvSearch.ts diff --git a/data/cv-report.json b/data/cv-report.json new file mode 100644 index 0000000..fef94b9 --- /dev/null +++ b/data/cv-report.json @@ -0,0 +1,60 @@ +{ + "generatedAt": "2026-06-11T22:41:08.860Z", + "protocol": { + "folds": [ + { + "from": "2014-01-01", + "to": "2016-01-01" + }, + { + "from": "2016-01-01", + "to": "2018-01-01" + }, + { + "from": "2018-01-01", + "to": "2020-01-01" + }, + { + "from": "2020-01-01", + "to": "2022-01-01" + }, + { + "from": "2022-01-01", + "to": "2024-01-01" + } + ], + "test": { + "from": "2024-01-01", + "to": "2026-06-01" + }, + "paramYears": 12, + "minGain": 0.0005, + "selection": "mean fold RPS, staged greedy" + }, + "shipped": { + "xi": 0.25, + "k": 3, + "w": 0.45, + "m": 3, + "beta": 0.3, + "gamma": 0, + "homeAdv": 70, + "kScale": 1, + "meanFoldRps": 0.1724, + "testRps": 0.1649 + }, + "champion": { + "xi": 0.25, + "k": 3, + "w": 0.4, + "m": 3, + "beta": 0.4, + "gamma": 0, + "homeAdv": 90, + "kScale": 1.25, + "meanFoldRps": 0.1721, + "testRps": 0.1647 + }, + "shipChange": false, + "runtimeSec": 72 +} \ No newline at end of file diff --git a/scripts/cvSearch.ts b/scripts/cvSearch.ts new file mode 100644 index 0000000..f1f3cc9 --- /dev/null +++ b/scripts/cvSearch.ts @@ -0,0 +1,381 @@ +// v5 hyperparameter search: rolling-origin cross-validation. +// +// The v4 protocol tuned on ONE validation window (2018–2022) — selection could +// overfit that window's quirks. v5 selects by MEAN walk-forward RPS across five +// rolling two-year folds, each with its own pre-fold parameter fit (no fold +// ever sees information from its future): +// F1 2014–16 · F2 2016–18 · F3 2018–20 · F4 2020–22 · F5 2022–24 +// FINAL TEST: 2024-01-01 → 2026-06-01, untouched by selection, evaluated once. +// +// Search axes (staged greedy, like v4 — full grid is combinatorially silly): +// stage 1: Team-DC ξ (decay), k (shrinkage), w (ensemble weight) +// stage 2: fast-Elo blend (m, β) +// stage 3: recent-form goal multiplier γ +// stage 4: Elo-walk axes — HOME_ADV_ELO and K-scale σ +// Ship bar (pre-registered): champion must beat the current shipped config by +// ≥ MIN_GAIN mean-fold RPS. Output → data/cv-report.json (research artifact, +// not public; the public evidence stays scripts/buildBacktest.ts). +// +// Self-contained on purpose: this is a research harness, and keeping it apart +// from the publication script means it can never destabilize shipped evidence. +import { readFileSync, writeFileSync, mkdirSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { canonicalTeam } from '../src/lib/teams'; +import { importanceWeight, eloDelta, expectedHome } from '../src/lib/model/elo'; +import { lambdasFromElo, scoreMatrix, outcomeProbs, poissonPmf, type ModelParams } from '../src/lib/model/poisson'; +import { fitTeamDc, teamDcLambdas, poolMatrices, type DcMatch, type TeamDcParams } from '../src/lib/model/teamDc'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); +const CSV = join(ROOT, 'data', 'raw', 'international-results.csv'); +const OUT = join(ROOT, 'data', 'cv-report.json'); + +const START_ELO = 1500; +const REFIT_DAYS = 60; +const FIT_WINDOW_YEARS = 15; +const PARAM_YEARS = 12; // params fit on the 12 years before each fold/test +const DAY = 24 * 60 * 60 * 1000; +const MIN_GAIN = 0.0005; + +const FOLDS = [ + { from: '2014-01-01', to: '2016-01-01' }, + { from: '2016-01-01', to: '2018-01-01' }, + { from: '2018-01-01', to: '2020-01-01' }, + { from: '2020-01-01', to: '2022-01-01' }, + { from: '2022-01-01', to: '2024-01-01' }, +]; +const TEST = { from: '2024-01-01', to: '2026-06-01' }; + +const XI_GRID = [0.25, 0.5, 0.75, 1, 1.5, 2]; +const K_GRID = [3, 5, 10, 20]; +const M_GRID = [2, 3]; +const BETA_GRID = [0.1, 0.2, 0.3, 0.4]; +const GAMMA_GRID = [0.05, 0.1, 0.15]; +const HOMEADV_GRID = [50, 70, 90]; +const KSCALE_GRID = [0.75, 1, 1.25]; +const FORM_WINDOW = 10; + +/** the currently shipped v4 config — the bar to clear (homeAdv 70, σ 1) */ +const SHIPPED = { xi: 0.25, k: 3, w: 0.45, m: 3, beta: 0.3, gamma: 0, homeAdv: 70, kScale: 1 }; + +interface Row { date: string; home: string; away: string; hs: number; as: number; tournament: string; neutral: boolean; t: number } +type Probs = { h: number; d: number; a: number }; +type Outcome = 'h' | 'd' | 'a'; + +function parse(): Row[] { + const lines = readFileSync(CSV, 'utf8').split('\n'); + const rows: Row[] = []; + for (let i = 1; i < lines.length; i++) { + const c = lines[i]?.split(','); + if (!c || c.length < 9) continue; + const hs = Number(c[3]); const as = Number(c[4]); + if (!Number.isFinite(hs) || !Number.isFinite(as)) continue; + rows.push({ + date: c[0]!, home: canonicalTeam(c[1]!), away: canonicalTeam(c[2]!), hs, as, + tournament: c[5]!, neutral: c[8]!.trim().toUpperCase() === 'TRUE', + t: new Date(c[0]!).getTime(), + }); + } + rows.sort((a, b) => a.t - b.t); + return rows; +} + +function dcTau(h: number, a: number, lh: number, la: number, rho: number): number { + if (h === 0 && a === 0) return 1 - lh * la * rho; + if (h === 0 && a === 1) return 1 + lh * rho; + if (h === 1 && a === 0) return 1 + la * rho; + if (h === 1 && a === 1) return 1 - rho; + return 1; +} + +const outcomeOf = (hs: number, as: number): Outcome => (hs > as ? 'h' : hs < as ? 'a' : 'd'); +const probsOf = (m: number[][]): Probs => { const p = outcomeProbs(m); return { h: p.home, d: p.draw, a: p.away }; }; +function rps(p: Probs, o: Outcome): number { + const y1 = o === 'h' ? 1 : 0; + const y2 = o === 'a' ? 0 : 1; + return ((p.h - y1) ** 2 + (p.h + p.d - y2) ** 2) / 2; +} + +// --------------------------------------------------------------------------- +// Elo walk variants: one pass per (homeAdv, kScale), storing pre-match slow + +// fast ratings and the pre-match form signal (mean Elo surprise, last 10 +// non-friendly). Everything is strictly pre-match by construction. +// --------------------------------------------------------------------------- +interface Walk { + preH: Float64Array; preA: Float64Array; // slow Elo, pre-match + fast: Map; // per m + formH: Float64Array; formA: Float64Array; +} + +function runWalk(rows: Row[], homeAdvElo: number, kScale: number): Walk { + const n = rows.length; + const w: Walk = { + preH: new Float64Array(n), preA: new Float64Array(n), + fast: new Map(M_GRID.map((m) => [m, { h: new Float64Array(n), a: new Float64Array(n) }])), + formH: new Float64Array(n), formA: new Float64Array(n), + }; + const elo = new Map(); + const eloFast = new Map>(M_GRID.map((m) => [m, new Map()])); + const formArr = new Map(); + const get = (map: Map, t: string) => map.get(t) ?? START_ELO; + const formMean = (t: string) => { + const a = formArr.get(t); + return a && a.length ? a.reduce((s, x) => s + x, 0) / a.length : 0; + }; + for (let i = 0; i < n; i++) { + const r = rows[i]!; + const adv = r.neutral ? 0 : homeAdvElo; + const eh = get(elo, r.home), ea = get(elo, r.away); + w.preH[i] = eh; w.preA[i] = ea; + for (const m of M_GRID) { + const fm = eloFast.get(m)!; + const arr = w.fast.get(m)!; + const feh = get(fm, r.home), fea = get(fm, r.away); + arr.h[i] = feh; arr.a[i] = fea; + const fd = eloDelta(r.hs, r.as, feh, fea, importanceWeight(r.tournament) * kScale * m, adv); + fm.set(r.home, feh + fd); fm.set(r.away, fea - fd); + } + w.formH[i] = formMean(r.home); + w.formA[i] = formMean(r.away); + if (r.tournament !== 'Friendly') { + const eH = expectedHome(eh, ea, adv); + const sH = r.hs > r.as ? 1 : r.hs < r.as ? 0 : 0.5; + const push = (t: string, v: number) => { + const a = formArr.get(t) ?? []; + a.push(v); + if (a.length > FORM_WINDOW) a.shift(); + formArr.set(t, a); + }; + push(r.home, sH - eH); + push(r.away, (1 - sH) - (1 - eH)); + } + const d = eloDelta(r.hs, r.as, eh, ea, importanceWeight(r.tournament) * kScale, adv); + elo.set(r.home, eh + d); elo.set(r.away, ea - d); + } + return w; +} + +/** Fit goalsPerElo/avgGoals/rho on the PARAM_YEARS before `fromT` (pre-fold only). */ +function fitParams(rows: Row[], walk: Walk, homeAdvElo: number, fromT: number): ModelParams { + const startT = fromT - PARAM_YEARS * 365 * DAY; + let sum = 0, n = 0, sxy = 0, sxx = 0; + const calib: { d: number; h: number; a: number }[] = []; + for (let i = 0; i < rows.length; i++) { + const r = rows[i]!; + if (r.t < startT) continue; + if (r.t >= fromT) break; + const eff = walk.preH[i]! - walk.preA[i]! + (r.neutral ? 0 : homeAdvElo); + sum += r.hs + r.as; n++; sxy += eff * (r.hs - r.as); sxx += eff * eff; + calib.push({ d: eff, h: r.hs, a: r.as }); + } + const avgGoals = sum / Math.max(1, n); + const goalsPerElo = sxy / Math.max(1e-9, sxx); + let rho0 = -0.05, bestLL = -Infinity; + for (let r0 = -0.2; r0 <= 0.1 + 1e-9; r0 += 0.02) { + let ll = 0; + for (const m of calib) { + const sup = goalsPerElo * m.d; + const lh = Math.min(8, Math.max(0.15, avgGoals / 2 + sup / 2)); + const la = Math.min(8, Math.max(0.15, avgGoals / 2 - sup / 2)); + ll += Math.log(Math.max(1e-12, poissonPmf(lh, m.h) * poissonPmf(la, m.a) * dcTau(m.h, m.a, lh, la, r0))); + } + if (ll > bestLL) { bestLL = ll; rho0 = +r0.toFixed(2); } + } + return { goalsPerElo, avgGoals, rho: rho0, homeAdvElo }; +} + +// --------------------------------------------------------------------------- +// Walk-forward Team-DC lambdas per fold per (ξ, k) — the expensive part, cached. +// --------------------------------------------------------------------------- +type Pred = { i: number; lDcH: number; lDcA: number }; + +function walkTeamDc(rows: Row[], xi: number, k: number, fromT: number, toT: number): Pred[] { + const preds: Pred[] = []; + let fit: TeamDcParams | null = null; + let fitAt = -Infinity; + const windowMs = FIT_WINDOW_YEARS * 365 * DAY; + for (let i = 0; i < rows.length; i++) { + const r = rows[i]!; + if (r.t < fromT) continue; + if (r.t >= toT) break; + if (r.t - fitAt > REFIT_DAYS * DAY) { + const train: DcMatch[] = []; + for (let j = 0; j < i; j++) { + const m = rows[j]!; + if (r.t - m.t > windowMs) continue; + train.push({ + daysAgo: (r.t - m.t) / DAY, home: m.home, away: m.away, + homeGoals: m.hs, awayGoals: m.as, neutral: m.neutral, + }); + } + fit = fitTeamDc(train, xi, k); + fitAt = r.t; + } + const { lambdaHome, lambdaAway } = teamDcLambdas(fit!, r.home, r.away, r.neutral); + preds.push({ i, lDcH: lambdaHome, lDcA: lambdaAway }); + } + return preds; +} + +// --------------------------------------------------------------------------- +// Scoring: mean RPS over one fold for a full config, sweeping w cheaply. +// --------------------------------------------------------------------------- +const W_STEPS: number[] = []; +for (let w = 0; w <= 1.0001; w += 0.05) W_STEPS.push(+w.toFixed(2)); + +const clampL = (x: number) => Math.min(8, Math.max(0.1, x)); + +interface Cfg { xi: number; k: number; w: number; m: number; beta: number; gamma: number; homeAdv: number; kScale: number } + +function sweepW( + rows: Row[], preds: Pred[], walk: Walk, params: ModelParams, + beta: number, m: number, gamma: number, +): { w: number; rps: number }[] { + const acc = new Float64Array(W_STEPS.length); + const fast = walk.fast.get(m)!; + for (const p of preds) { + const r = rows[p.i]!; + const o = outcomeOf(r.hs, r.as); + const adv = r.neutral ? 0 : params.homeAdvElo; + const eh = (1 - beta) * walk.preH[p.i]! + beta * fast.h[p.i]!; + const ea = (1 - beta) * walk.preA[p.i]! + beta * fast.a[p.i]!; + let { lambdaHome: elh, lambdaAway: ela } = lambdasFromElo(eh, ea, params, adv); + let dlh = p.lDcH, dla = p.lDcA; + if (gamma > 0) { + const fh = Math.exp(gamma * walk.formH[p.i]!), fa = Math.exp(gamma * walk.formA[p.i]!); + elh = clampL(elh * fh); ela = clampL(ela * fa); + dlh = clampL(dlh * fh); dla = clampL(dla * fa); + } + const mE = scoreMatrix(elh, ela, params.rho); + const mD = scoreMatrix(dlh, dla, params.rho); + for (let wi = 0; wi < W_STEPS.length; wi++) { + const w = W_STEPS[wi]!; + const pooled = w === 0 ? mD : w === 1 ? mE : poolMatrices(mE, mD, w); + acc[wi] += rps(probsOf(pooled), o); + } + } + return W_STEPS.map((w, wi) => ({ w, rps: acc[wi]! / Math.max(1, preds.length) })); +} + +function main(): void { + const t0 = Date.now(); + const rows = parse(); + console.log(`${rows.length} rows · ${FOLDS.length} folds · test ${TEST.from}→${TEST.to}`); + + const foldTs = FOLDS.map((f) => ({ fromT: new Date(f.from).getTime(), toT: new Date(f.to).getTime() })); + + // Elo walks + per-fold params, per (homeAdv, kScale) — cheap, precompute all. + const walks = new Map(); + const foldParams = new Map(); // key → params per fold + for (const ha of HOMEADV_GRID) { + for (const ks of KSCALE_GRID) { + const key = `${ha}|${ks}`; + const w = runWalk(rows, ha, ks); + walks.set(key, w); + foldParams.set(key, foldTs.map((f) => fitParams(rows, w, ha, f.fromT))); + } + } + console.log(`walks + per-fold params ready (${((Date.now() - t0) / 1000).toFixed(0)}s)`); + + const baseKey = `${SHIPPED.homeAdv}|${SHIPPED.kScale}`; + const baseWalk = walks.get(baseKey)!; + const baseParams = foldParams.get(baseKey)!; + + /** mean fold RPS for (xi,k) preds under a config, returning per-w means */ + const dcCache = new Map(); // `${xi}|${k}` → preds per fold + const foldPreds = (xi: number, k: number): Pred[][] => { + const key = `${xi}|${k}`; + let v = dcCache.get(key); + if (!v) { + v = foldTs.map((f) => walkTeamDc(rows, xi, k, f.fromT, f.toT)); + dcCache.set(key, v); + } + return v; + }; + const meanSweep = (xi: number, k: number, beta: number, m: number, gamma: number, walk = baseWalk, params = baseParams): { w: number; rps: number }[] => { + const folds = foldPreds(xi, k); + const acc = new Float64Array(W_STEPS.length); + for (let fi = 0; fi < folds.length; fi++) { + const res = sweepW(rows, folds[fi]!, walk, params[fi]!, beta, m, gamma); + for (let wi = 0; wi < W_STEPS.length; wi++) acc[wi] += res[wi]!.rps; + } + return W_STEPS.map((w, wi) => ({ w, rps: acc[wi]! / folds.length })); + }; + + // ---- control: shipped config under the CV protocol ---- + const shippedMean = meanSweep(SHIPPED.xi, SHIPPED.k, SHIPPED.beta, SHIPPED.m, SHIPPED.gamma) + .find((x) => x.w === SHIPPED.w)!.rps; + console.log(`shipped v4 config mean-fold rps ${shippedMean.toFixed(4)}`); + + // ---- stage 1: ξ, k, w ---- + let best: Cfg & { rps: number } = { ...SHIPPED, rps: Infinity }; + for (const xi of XI_GRID) { + for (const k of K_GRID) { + for (const { w, rps: r } of meanSweep(xi, k, 0, 2, 0)) { + if (r < best.rps) best = { ...SHIPPED, xi, k, w, m: 2, beta: 0, gamma: 0, rps: r }; + } + console.log(` s1 ξ=${xi} k=${k} (best ξ=${best.xi} k=${best.k} w=${best.w} rps=${best.rps.toFixed(4)})`); + } + } + + // ---- stage 2: fast-Elo m, β ---- + for (const m of M_GRID) { + for (const beta of BETA_GRID) { + for (const { w, rps: r } of meanSweep(best.xi, best.k, beta, m, 0)) { + if (r < best.rps) best = { ...best, m, beta, w, rps: r }; + } + } + } + console.log(` s2 best: m=${best.m} β=${best.beta} w=${best.w} rps=${best.rps.toFixed(4)}`); + + // ---- stage 3: form γ ---- + for (const gamma of GAMMA_GRID) { + for (const { w, rps: r } of meanSweep(best.xi, best.k, best.beta, best.m, gamma)) { + if (r < best.rps) best = { ...best, gamma, w, rps: r }; + } + } + console.log(` s3 best: γ=${best.gamma} w=${best.w} rps=${best.rps.toFixed(4)}`); + + // ---- stage 4: Elo-walk axes (homeAdv, kScale) ---- + for (const ha of HOMEADV_GRID) { + for (const ks of KSCALE_GRID) { + if (ha === SHIPPED.homeAdv && ks === SHIPPED.kScale && best.rps < Infinity) continue; + const key = `${ha}|${ks}`; + for (const { w, rps: r } of meanSweep(best.xi, best.k, best.beta, best.m, best.gamma, walks.get(key)!, foldParams.get(key)!)) { + if (r < best.rps) best = { ...best, homeAdv: ha, kScale: ks, w, rps: r }; + } + } + } + console.log(` s4 best: homeAdv=${best.homeAdv} σ=${best.kScale} w=${best.w} rps=${best.rps.toFixed(4)}`); + + const gain = shippedMean - best.rps; + const ship = gain >= MIN_GAIN; + console.log(`champion mean-fold rps ${best.rps.toFixed(4)} vs shipped ${shippedMean.toFixed(4)} → gain ${gain.toFixed(4)} → ${ship ? 'SHIP' : 'KEEP shipped'}`); + + // ---- final test (2024–26), evaluated once for champion + shipped ---- + const testFromT = new Date(TEST.from).getTime(); + const testToT = new Date(TEST.to).getTime(); + const testEval = (cfg: Cfg): number => { + const key = `${cfg.homeAdv}|${cfg.kScale}`; + const walk = walks.get(key)!; + const params = fitParams(rows, walk, cfg.homeAdv, testFromT); + const preds = walkTeamDc(rows, cfg.xi, cfg.k, testFromT, testToT); + return sweepW(rows, preds, walk, params, cfg.beta, cfg.m, cfg.gamma).find((x) => x.w === cfg.w)!.rps; + }; + const champTest = testEval(best); + const shippedTest = testEval(SHIPPED); + console.log(`TEST 2024–26: champion rps ${champTest.toFixed(4)} | shipped rps ${shippedTest.toFixed(4)}`); + + mkdirSync(dirname(OUT), { recursive: true }); + writeFileSync(OUT, JSON.stringify({ + generatedAt: new Date().toISOString(), + protocol: { folds: FOLDS, test: TEST, paramYears: PARAM_YEARS, minGain: MIN_GAIN, selection: 'mean fold RPS, staged greedy' }, + shipped: { ...SHIPPED, meanFoldRps: +shippedMean.toFixed(4), testRps: +shippedTest.toFixed(4) }, + champion: { ...best, rps: undefined, meanFoldRps: +best.rps.toFixed(4), testRps: +champTest.toFixed(4) }, + shipChange: ship, + runtimeSec: Math.round((Date.now() - t0) / 1000), + }, null, 2)); + console.log(`report → data/cv-report.json (${Math.round((Date.now() - t0) / 1000)}s total)`); +} + +main();