v5 model protocol: rolling-origin CV search harness

Five two-year walk-forward folds (2014-2024), each with its own
pre-fold parameter fit; selection by mean fold RPS; staged greedy over
Team-DC decay/shrinkage/weight, fast-Elo blend, form multiplier, and
new Elo-walk axes (home advantage, K scale); untouched 2024-26 final
test. Result: the shipped v4 config holds — the staged champion gained
0.0003, under the 0.0005 ship bar, and the final test agrees. v4 is no
longer a one-window result.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

data/cv-report.json

@@ -0,0 +1,60 @@
+{
+  "generatedAt": "2026-06-11T22:41:08.860Z",
+  "protocol": {
+    "folds": [
+      {
+        "from": "2014-01-01",
+        "to": "2016-01-01"
+      },
+      {
+        "from": "2016-01-01",
+        "to": "2018-01-01"
+      },
+      {
+        "from": "2018-01-01",
+        "to": "2020-01-01"
+      },
+      {
+        "from": "2020-01-01",
+        "to": "2022-01-01"
+      },
+      {
+        "from": "2022-01-01",
+        "to": "2024-01-01"
+      }
+    ],
+    "test": {
+      "from": "2024-01-01",
+      "to": "2026-06-01"
+    },
+    "paramYears": 12,
+    "minGain": 0.0005,
+    "selection": "mean fold RPS, staged greedy"
+  },
+  "shipped": {
+    "xi": 0.25,
+    "k": 3,
+    "w": 0.45,
+    "m": 3,
+    "beta": 0.3,
+    "gamma": 0,
+    "homeAdv": 70,
+    "kScale": 1,
+    "meanFoldRps": 0.1724,
+    "testRps": 0.1649
+  },
+  "champion": {
+    "xi": 0.25,
+    "k": 3,
+    "w": 0.4,
+    "m": 3,
+    "beta": 0.4,
+    "gamma": 0,
+    "homeAdv": 90,
+    "kScale": 1.25,
+    "meanFoldRps": 0.1721,
+    "testRps": 0.1647
+  },
+  "shipChange": false,
+  "runtimeSec": 72
+}

scripts/cvSearch.ts

@@ -0,0 +1,381 @@
+// v5 hyperparameter search: rolling-origin cross-validation.
+//
+// The v4 protocol tuned on ONE validation window (2018–2022) — selection could
+// overfit that window's quirks. v5 selects by MEAN walk-forward RPS across five
+// rolling two-year folds, each with its own pre-fold parameter fit (no fold
+// ever sees information from its future):
+//   F1 2014–16 · F2 2016–18 · F3 2018–20 · F4 2020–22 · F5 2022–24
+// FINAL TEST: 2024-01-01 → 2026-06-01, untouched by selection, evaluated once.
+//
+// Search axes (staged greedy, like v4 — full grid is combinatorially silly):
+//   stage 1: Team-DC ξ (decay), k (shrinkage), w (ensemble weight)
+//   stage 2: fast-Elo blend (m, β)
+//   stage 3: recent-form goal multiplier γ
+//   stage 4: Elo-walk axes — HOME_ADV_ELO and K-scale σ
+// Ship bar (pre-registered): champion must beat the current shipped config by
+// ≥ MIN_GAIN mean-fold RPS. Output → data/cv-report.json (research artifact,
+// not public; the public evidence stays scripts/buildBacktest.ts).
+//
+// Self-contained on purpose: this is a research harness, and keeping it apart
+// from the publication script means it can never destabilize shipped evidence.
+import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+import { canonicalTeam } from '../src/lib/teams';
+import { importanceWeight, eloDelta, expectedHome } from '../src/lib/model/elo';
+import { lambdasFromElo, scoreMatrix, outcomeProbs, poissonPmf, type ModelParams } from '../src/lib/model/poisson';
+import { fitTeamDc, teamDcLambdas, poolMatrices, type DcMatch, type TeamDcParams } from '../src/lib/model/teamDc';
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+const CSV = join(ROOT, 'data', 'raw', 'international-results.csv');
+const OUT = join(ROOT, 'data', 'cv-report.json');
+
+const START_ELO = 1500;
+const REFIT_DAYS = 60;
+const FIT_WINDOW_YEARS = 15;
+const PARAM_YEARS = 12; // params fit on the 12 years before each fold/test
+const DAY = 24 * 60 * 60 * 1000;
+const MIN_GAIN = 0.0005;
+
+const FOLDS = [
+  { from: '2014-01-01', to: '2016-01-01' },
+  { from: '2016-01-01', to: '2018-01-01' },
+  { from: '2018-01-01', to: '2020-01-01' },
+  { from: '2020-01-01', to: '2022-01-01' },
+  { from: '2022-01-01', to: '2024-01-01' },
+];
+const TEST = { from: '2024-01-01', to: '2026-06-01' };
+
+const XI_GRID = [0.25, 0.5, 0.75, 1, 1.5, 2];
+const K_GRID = [3, 5, 10, 20];
+const M_GRID = [2, 3];
+const BETA_GRID = [0.1, 0.2, 0.3, 0.4];
+const GAMMA_GRID = [0.05, 0.1, 0.15];
+const HOMEADV_GRID = [50, 70, 90];
+const KSCALE_GRID = [0.75, 1, 1.25];
+const FORM_WINDOW = 10;
+
+/** the currently shipped v4 config — the bar to clear (homeAdv 70, σ 1) */
+const SHIPPED = { xi: 0.25, k: 3, w: 0.45, m: 3, beta: 0.3, gamma: 0, homeAdv: 70, kScale: 1 };
+
+interface Row { date: string; home: string; away: string; hs: number; as: number; tournament: string; neutral: boolean; t: number }
+type Probs = { h: number; d: number; a: number };
+type Outcome = 'h' | 'd' | 'a';
+
+function parse(): Row[] {
+  const lines = readFileSync(CSV, 'utf8').split('\n');
+  const rows: Row[] = [];
+  for (let i = 1; i < lines.length; i++) {
+    const c = lines[i]?.split(',');
+    if (!c || c.length < 9) continue;
+    const hs = Number(c[3]); const as = Number(c[4]);
+    if (!Number.isFinite(hs) || !Number.isFinite(as)) continue;
+    rows.push({
+      date: c[0]!, home: canonicalTeam(c[1]!), away: canonicalTeam(c[2]!), hs, as,
+      tournament: c[5]!, neutral: c[8]!.trim().toUpperCase() === 'TRUE',
+      t: new Date(c[0]!).getTime(),
+    });
+  }
+  rows.sort((a, b) => a.t - b.t);
+  return rows;
+}
+
+function dcTau(h: number, a: number, lh: number, la: number, rho: number): number {
+  if (h === 0 && a === 0) return 1 - lh * la * rho;
+  if (h === 0 && a === 1) return 1 + lh * rho;
+  if (h === 1 && a === 0) return 1 + la * rho;
+  if (h === 1 && a === 1) return 1 - rho;
+  return 1;
+}
+
+const outcomeOf = (hs: number, as: number): Outcome => (hs > as ? 'h' : hs < as ? 'a' : 'd');
+const probsOf = (m: number[][]): Probs => { const p = outcomeProbs(m); return { h: p.home, d: p.draw, a: p.away }; };
+function rps(p: Probs, o: Outcome): number {
+  const y1 = o === 'h' ? 1 : 0;
+  const y2 = o === 'a' ? 0 : 1;
+  return ((p.h - y1) ** 2 + (p.h + p.d - y2) ** 2) / 2;
+}
+
+// ---------------------------------------------------------------------------
+// Elo walk variants: one pass per (homeAdv, kScale), storing pre-match slow +
+// fast ratings and the pre-match form signal (mean Elo surprise, last 10
+// non-friendly). Everything is strictly pre-match by construction.
+// ---------------------------------------------------------------------------
+interface Walk {
+  preH: Float64Array; preA: Float64Array; // slow Elo, pre-match
+  fast: Map<number, { h: Float64Array; a: Float64Array }>; // per m
+  formH: Float64Array; formA: Float64Array;
+}
+
+function runWalk(rows: Row[], homeAdvElo: number, kScale: number): Walk {
+  const n = rows.length;
+  const w: Walk = {
+    preH: new Float64Array(n), preA: new Float64Array(n),
+    fast: new Map(M_GRID.map((m) => [m, { h: new Float64Array(n), a: new Float64Array(n) }])),
+    formH: new Float64Array(n), formA: new Float64Array(n),
+  };
+  const elo = new Map<string, number>();
+  const eloFast = new Map<number, Map<string, number>>(M_GRID.map((m) => [m, new Map()]));
+  const formArr = new Map<string, number[]>();
+  const get = (map: Map<string, number>, t: string) => map.get(t) ?? START_ELO;
+  const formMean = (t: string) => {
+    const a = formArr.get(t);
+    return a && a.length ? a.reduce((s, x) => s + x, 0) / a.length : 0;
+  };
+  for (let i = 0; i < n; i++) {
+    const r = rows[i]!;
+    const adv = r.neutral ? 0 : homeAdvElo;
+    const eh = get(elo, r.home), ea = get(elo, r.away);
+    w.preH[i] = eh; w.preA[i] = ea;
+    for (const m of M_GRID) {
+      const fm = eloFast.get(m)!;
+      const arr = w.fast.get(m)!;
+      const feh = get(fm, r.home), fea = get(fm, r.away);
+      arr.h[i] = feh; arr.a[i] = fea;
+      const fd = eloDelta(r.hs, r.as, feh, fea, importanceWeight(r.tournament) * kScale * m, adv);
+      fm.set(r.home, feh + fd); fm.set(r.away, fea - fd);
+    }
+    w.formH[i] = formMean(r.home);
+    w.formA[i] = formMean(r.away);
+    if (r.tournament !== 'Friendly') {
+      const eH = expectedHome(eh, ea, adv);
+      const sH = r.hs > r.as ? 1 : r.hs < r.as ? 0 : 0.5;
+      const push = (t: string, v: number) => {
+        const a = formArr.get(t) ?? [];
+        a.push(v);
+        if (a.length > FORM_WINDOW) a.shift();
+        formArr.set(t, a);
+      };
+      push(r.home, sH - eH);
+      push(r.away, (1 - sH) - (1 - eH));
+    }
+    const d = eloDelta(r.hs, r.as, eh, ea, importanceWeight(r.tournament) * kScale, adv);
+    elo.set(r.home, eh + d); elo.set(r.away, ea - d);
+  }
+  return w;
+}
+
+/** Fit goalsPerElo/avgGoals/rho on the PARAM_YEARS before `fromT` (pre-fold only). */
+function fitParams(rows: Row[], walk: Walk, homeAdvElo: number, fromT: number): ModelParams {
+  const startT = fromT - PARAM_YEARS * 365 * DAY;
+  let sum = 0, n = 0, sxy = 0, sxx = 0;
+  const calib: { d: number; h: number; a: number }[] = [];
+  for (let i = 0; i < rows.length; i++) {
+    const r = rows[i]!;
+    if (r.t < startT) continue;
+    if (r.t >= fromT) break;
+    const eff = walk.preH[i]! - walk.preA[i]! + (r.neutral ? 0 : homeAdvElo);
+    sum += r.hs + r.as; n++; sxy += eff * (r.hs - r.as); sxx += eff * eff;
+    calib.push({ d: eff, h: r.hs, a: r.as });
+  }
+  const avgGoals = sum / Math.max(1, n);
+  const goalsPerElo = sxy / Math.max(1e-9, sxx);
+  let rho0 = -0.05, bestLL = -Infinity;
+  for (let r0 = -0.2; r0 <= 0.1 + 1e-9; r0 += 0.02) {
+    let ll = 0;
+    for (const m of calib) {
+      const sup = goalsPerElo * m.d;
+      const lh = Math.min(8, Math.max(0.15, avgGoals / 2 + sup / 2));
+      const la = Math.min(8, Math.max(0.15, avgGoals / 2 - sup / 2));
+      ll += Math.log(Math.max(1e-12, poissonPmf(lh, m.h) * poissonPmf(la, m.a) * dcTau(m.h, m.a, lh, la, r0)));
+    }
+    if (ll > bestLL) { bestLL = ll; rho0 = +r0.toFixed(2); }
+  }
+  return { goalsPerElo, avgGoals, rho: rho0, homeAdvElo };
+}
+
+// ---------------------------------------------------------------------------
+// Walk-forward Team-DC lambdas per fold per (ξ, k) — the expensive part, cached.
+// ---------------------------------------------------------------------------
+type Pred = { i: number; lDcH: number; lDcA: number };
+
+function walkTeamDc(rows: Row[], xi: number, k: number, fromT: number, toT: number): Pred[] {
+  const preds: Pred[] = [];
+  let fit: TeamDcParams | null = null;
+  let fitAt = -Infinity;
+  const windowMs = FIT_WINDOW_YEARS * 365 * DAY;
+  for (let i = 0; i < rows.length; i++) {
+    const r = rows[i]!;
+    if (r.t < fromT) continue;
+    if (r.t >= toT) break;
+    if (r.t - fitAt > REFIT_DAYS * DAY) {
+      const train: DcMatch[] = [];
+      for (let j = 0; j < i; j++) {
+        const m = rows[j]!;
+        if (r.t - m.t > windowMs) continue;
+        train.push({
+          daysAgo: (r.t - m.t) / DAY, home: m.home, away: m.away,
+          homeGoals: m.hs, awayGoals: m.as, neutral: m.neutral,
+        });
+      }
+      fit = fitTeamDc(train, xi, k);
+      fitAt = r.t;
+    }
+    const { lambdaHome, lambdaAway } = teamDcLambdas(fit!, r.home, r.away, r.neutral);
+    preds.push({ i, lDcH: lambdaHome, lDcA: lambdaAway });
+  }
+  return preds;
+}
+
+// ---------------------------------------------------------------------------
+// Scoring: mean RPS over one fold for a full config, sweeping w cheaply.
+// ---------------------------------------------------------------------------
+const W_STEPS: number[] = [];
+for (let w = 0; w <= 1.0001; w += 0.05) W_STEPS.push(+w.toFixed(2));
+
+const clampL = (x: number) => Math.min(8, Math.max(0.1, x));
+
+interface Cfg { xi: number; k: number; w: number; m: number; beta: number; gamma: number; homeAdv: number; kScale: number }
+
+function sweepW(
+  rows: Row[], preds: Pred[], walk: Walk, params: ModelParams,
+  beta: number, m: number, gamma: number,
+): { w: number; rps: number }[] {
+  const acc = new Float64Array(W_STEPS.length);
+  const fast = walk.fast.get(m)!;
+  for (const p of preds) {
+    const r = rows[p.i]!;
+    const o = outcomeOf(r.hs, r.as);
+    const adv = r.neutral ? 0 : params.homeAdvElo;
+    const eh = (1 - beta) * walk.preH[p.i]! + beta * fast.h[p.i]!;
+    const ea = (1 - beta) * walk.preA[p.i]! + beta * fast.a[p.i]!;
+    let { lambdaHome: elh, lambdaAway: ela } = lambdasFromElo(eh, ea, params, adv);
+    let dlh = p.lDcH, dla = p.lDcA;
+    if (gamma > 0) {
+      const fh = Math.exp(gamma * walk.formH[p.i]!), fa = Math.exp(gamma * walk.formA[p.i]!);
+      elh = clampL(elh * fh); ela = clampL(ela * fa);
+      dlh = clampL(dlh * fh); dla = clampL(dla * fa);
+    }
+    const mE = scoreMatrix(elh, ela, params.rho);
+    const mD = scoreMatrix(dlh, dla, params.rho);
+    for (let wi = 0; wi < W_STEPS.length; wi++) {
+      const w = W_STEPS[wi]!;
+      const pooled = w === 0 ? mD : w === 1 ? mE : poolMatrices(mE, mD, w);
+      acc[wi] += rps(probsOf(pooled), o);
+    }
+  }
+  return W_STEPS.map((w, wi) => ({ w, rps: acc[wi]! / Math.max(1, preds.length) }));
+}
+
+function main(): void {
+  const t0 = Date.now();
+  const rows = parse();
+  console.log(`${rows.length} rows · ${FOLDS.length} folds · test ${TEST.from}→${TEST.to}`);
+
+  const foldTs = FOLDS.map((f) => ({ fromT: new Date(f.from).getTime(), toT: new Date(f.to).getTime() }));
+
+  // Elo walks + per-fold params, per (homeAdv, kScale) — cheap, precompute all.
+  const walks = new Map<string, Walk>();
+  const foldParams = new Map<string, ModelParams[]>(); // key → params per fold
+  for (const ha of HOMEADV_GRID) {
+    for (const ks of KSCALE_GRID) {
+      const key = `${ha}|${ks}`;
+      const w = runWalk(rows, ha, ks);
+      walks.set(key, w);
+      foldParams.set(key, foldTs.map((f) => fitParams(rows, w, ha, f.fromT)));
+    }
+  }
+  console.log(`walks + per-fold params ready (${((Date.now() - t0) / 1000).toFixed(0)}s)`);
+
+  const baseKey = `${SHIPPED.homeAdv}|${SHIPPED.kScale}`;
+  const baseWalk = walks.get(baseKey)!;
+  const baseParams = foldParams.get(baseKey)!;
+
+  /** mean fold RPS for (xi,k) preds under a config, returning per-w means */
+  const dcCache = new Map<string, Pred[][]>(); // `${xi}|${k}` → preds per fold
+  const foldPreds = (xi: number, k: number): Pred[][] => {
+    const key = `${xi}|${k}`;
+    let v = dcCache.get(key);
+    if (!v) {
+      v = foldTs.map((f) => walkTeamDc(rows, xi, k, f.fromT, f.toT));
+      dcCache.set(key, v);
+    }
+    return v;
+  };
+  const meanSweep = (xi: number, k: number, beta: number, m: number, gamma: number, walk = baseWalk, params = baseParams): { w: number; rps: number }[] => {
+    const folds = foldPreds(xi, k);
+    const acc = new Float64Array(W_STEPS.length);
+    for (let fi = 0; fi < folds.length; fi++) {
+      const res = sweepW(rows, folds[fi]!, walk, params[fi]!, beta, m, gamma);
+      for (let wi = 0; wi < W_STEPS.length; wi++) acc[wi] += res[wi]!.rps;
+    }
+    return W_STEPS.map((w, wi) => ({ w, rps: acc[wi]! / folds.length }));
+  };
+
+  // ---- control: shipped config under the CV protocol ----
+  const shippedMean = meanSweep(SHIPPED.xi, SHIPPED.k, SHIPPED.beta, SHIPPED.m, SHIPPED.gamma)
+    .find((x) => x.w === SHIPPED.w)!.rps;
+  console.log(`shipped v4 config mean-fold rps ${shippedMean.toFixed(4)}`);
+
+  // ---- stage 1: ξ, k, w ----
+  let best: Cfg & { rps: number } = { ...SHIPPED, rps: Infinity };
+  for (const xi of XI_GRID) {
+    for (const k of K_GRID) {
+      for (const { w, rps: r } of meanSweep(xi, k, 0, 2, 0)) {
+        if (r < best.rps) best = { ...SHIPPED, xi, k, w, m: 2, beta: 0, gamma: 0, rps: r };
+      }
+      console.log(`  s1 ξ=${xi} k=${k} (best ξ=${best.xi} k=${best.k} w=${best.w} rps=${best.rps.toFixed(4)})`);
+    }
+  }
+
+  // ---- stage 2: fast-Elo m, β ----
+  for (const m of M_GRID) {
+    for (const beta of BETA_GRID) {
+      for (const { w, rps: r } of meanSweep(best.xi, best.k, beta, m, 0)) {
+        if (r < best.rps) best = { ...best, m, beta, w, rps: r };
+      }
+    }
+  }
+  console.log(`  s2 best: m=${best.m} β=${best.beta} w=${best.w} rps=${best.rps.toFixed(4)}`);
+
+  // ---- stage 3: form γ ----
+  for (const gamma of GAMMA_GRID) {
+    for (const { w, rps: r } of meanSweep(best.xi, best.k, best.beta, best.m, gamma)) {
+      if (r < best.rps) best = { ...best, gamma, w, rps: r };
+    }
+  }
+  console.log(`  s3 best: γ=${best.gamma} w=${best.w} rps=${best.rps.toFixed(4)}`);
+
+  // ---- stage 4: Elo-walk axes (homeAdv, kScale) ----
+  for (const ha of HOMEADV_GRID) {
+    for (const ks of KSCALE_GRID) {
+      if (ha === SHIPPED.homeAdv && ks === SHIPPED.kScale && best.rps < Infinity) continue;
+      const key = `${ha}|${ks}`;
+      for (const { w, rps: r } of meanSweep(best.xi, best.k, best.beta, best.m, best.gamma, walks.get(key)!, foldParams.get(key)!)) {
+        if (r < best.rps) best = { ...best, homeAdv: ha, kScale: ks, w, rps: r };
+      }
+    }
+  }
+  console.log(`  s4 best: homeAdv=${best.homeAdv} σ=${best.kScale} w=${best.w} rps=${best.rps.toFixed(4)}`);
+
+  const gain = shippedMean - best.rps;
+  const ship = gain >= MIN_GAIN;
+  console.log(`champion mean-fold rps ${best.rps.toFixed(4)} vs shipped ${shippedMean.toFixed(4)} → gain ${gain.toFixed(4)} → ${ship ? 'SHIP' : 'KEEP shipped'}`);
+
+  // ---- final test (2024–26), evaluated once for champion + shipped ----
+  const testFromT = new Date(TEST.from).getTime();
+  const testToT = new Date(TEST.to).getTime();
+  const testEval = (cfg: Cfg): number => {
+    const key = `${cfg.homeAdv}|${cfg.kScale}`;
+    const walk = walks.get(key)!;
+    const params = fitParams(rows, walk, cfg.homeAdv, testFromT);
+    const preds = walkTeamDc(rows, cfg.xi, cfg.k, testFromT, testToT);
+    return sweepW(rows, preds, walk, params, cfg.beta, cfg.m, cfg.gamma).find((x) => x.w === cfg.w)!.rps;
+  };
+  const champTest = testEval(best);
+  const shippedTest = testEval(SHIPPED);
+  console.log(`TEST 2024–26: champion rps ${champTest.toFixed(4)} | shipped rps ${shippedTest.toFixed(4)}`);
+
+  mkdirSync(dirname(OUT), { recursive: true });
+  writeFileSync(OUT, JSON.stringify({
+    generatedAt: new Date().toISOString(),
+    protocol: { folds: FOLDS, test: TEST, paramYears: PARAM_YEARS, minGain: MIN_GAIN, selection: 'mean fold RPS, staged greedy' },
+    shipped: { ...SHIPPED, meanFoldRps: +shippedMean.toFixed(4), testRps: +shippedTest.toFixed(4) },
+    champion: { ...best, rps: undefined, meanFoldRps: +best.rps.toFixed(4), testRps: +champTest.toFixed(4) },
+    shipChange: ship,
+    runtimeSec: Math.round((Date.now() - t0) / 1000),
+  }, null, 2));
+  console.log(`report → data/cv-report.json (${Math.round((Date.now() - t0) / 1000)}s total)`);
+}
+
+main();