// v4 bake-off backtest with a strict three-way split — the evidence behind the // shipped model. NO information leaks forward: // params (pre-2018) : Elo→goals calibration, Dixon-Coles rho // validate (2018–2022) : tune Team-DC decay ξ, shrinkage k, ensemble weight w, // plus the v4 recency candidates — a fast-Elo blend // (second walk at K×m, blended in with weight β) and a // recent-form goal multiplier exp(γ·form), where form is // each side's mean Elo surprise (actual − expected) over // its last 10 non-friendly matches // test (2022–2026) : untouched final numbers for every variant + baselines // Decision rule: a recency candidate ships only if it beats the previous shipped // config by ≥ MIN_VAL_GAIN RPS on validation. Test is evaluated once, for honesty. // Output → public/data/backtest.json (Methodology page). import { readFileSync, writeFileSync, mkdirSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { dirname, join } from 'node:path'; import { canonicalTeam } from '../src/lib/teams'; import { HOME_ADV_ELO, importanceWeight, eloDelta, expectedHome } from '../src/lib/model/elo'; import { lambdasFromElo, scoreMatrix, outcomeProbs, poissonPmf, type ModelParams } from '../src/lib/model/poisson'; import { fitTeamDc, teamDcLambdas, poolMatrices, type DcMatch, type TeamDcParams } from '../src/lib/model/teamDc'; const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); const CSV = join(ROOT, 'data', 'raw', 'international-results.csv'); const OUT_DIR = join(ROOT, 'public', 'data'); const START_ELO = 1500; const PARAM_FROM = '2006-01-01'; const VAL_FROM = '2018-01-01'; const TEST_FROM = '2022-01-01'; const TEST_TO = '2026-06-01'; const REFIT_DAYS = 60; const FIT_WINDOW_YEARS = 15; const XI_GRID = [0.25, 0.5, 0.75, 1, 1.5, 2, 2.5]; const K_GRID = [3, 5, 10, 20]; const M_GRID = [2, 3]; // fast-Elo K multiplier const BETA_GRID = [0, 0.1, 0.2, 0.3]; // fast-Elo blend weight (0 = control) const GAMMA_GRID = [0, 0.05, 0.1, 0.15]; // recent-form goal multiplier (0 = control) const FORM_WINDOW = 10; // last N non-friendly matches feeding the form signal const MIN_VAL_GAIN = 0.0005; // candidate must beat the previous config by this much /** the previously shipped v3 config — the bar every candidate has to clear */ const PREV = { xi: 0.5, k: 5, w: 0.75, m: 2, beta: 0, gamma: 0 }; /** the v4 winner (from the full grid run) — keep in sync with buildRatings.ts */ const SHIPPED = { xi: 0.25, k: 3, w: 0.45, m: 3, beta: 0.3, gamma: 0 }; /** BACKTEST_QUICK=1 (Docker builds): skip the grid search and just score the * shipped + previous configs — the published test numbers are identical, the * hyperparam hunt only happens locally. */ const QUICK = process.env.BACKTEST_QUICK === '1'; interface Row { date: string; home: string; away: string; hs: number; as: number; tournament: string; neutral: boolean; t: number } type Probs = { h: number; d: number; a: number }; type Outcome = 'h' | 'd' | 'a'; const DAY = 24 * 60 * 60 * 1000; function parse(): Row[] { const lines = readFileSync(CSV, 'utf8').split('\n'); const rows: Row[] = []; for (let i = 1; i < lines.length; i++) { const c = lines[i]?.split(','); if (!c || c.length < 9) continue; const hs = Number(c[3]); const as = Number(c[4]); if (!Number.isFinite(hs) || !Number.isFinite(as)) continue; rows.push({ date: c[0]!, home: canonicalTeam(c[1]!), away: canonicalTeam(c[2]!), hs, as, tournament: c[5]!, neutral: c[8]!.trim().toUpperCase() === 'TRUE', t: new Date(c[0]!).getTime(), }); } rows.sort((a, b) => a.t - b.t); return rows; } function dcTau(h: number, a: number, lh: number, la: number, rho: number): number { if (h === 0 && a === 0) return 1 - lh * la * rho; if (h === 0 && a === 1) return 1 + lh * rho; if (h === 1 && a === 0) return 1 + la * rho; if (h === 1 && a === 1) return 1 - rho; return 1; } const outcomeOf = (hs: number, as: number): Outcome => (hs > as ? 'h' : hs < as ? 'a' : 'd'); const probsOf = (m: number[][]): Probs => { const p = outcomeProbs(m); return { h: p.home, d: p.draw, a: p.away }; }; function brier(p: Probs, o: Outcome): number { return (p.h - (o === 'h' ? 1 : 0)) ** 2 + (p.d - (o === 'd' ? 1 : 0)) ** 2 + (p.a - (o === 'a' ? 1 : 0)) ** 2; } const logloss = (p: Probs, o: Outcome): number => -Math.log(Math.max(1e-12, p[o])); function rps(p: Probs, o: Outcome): number { const y1 = o === 'h' ? 1 : 0; const y2 = o === 'a' ? 0 : 1; return ((p.h - y1) ** 2 + (p.h + p.d - y2) ** 2) / 2; } const argmax = (p: Probs): Outcome => (p.h >= p.d && p.h >= p.a ? 'h' : p.d >= p.a ? 'd' : 'a'); function scoreSet(preds: { p: Probs; o: Outcome }[]) { let b = 0, l = 0, r = 0, correct = 0; for (const { p, o } of preds) { b += brier(p, o); l += logloss(p, o); r += rps(p, o); if (argmax(p) === o) correct++; } const n = Math.max(1, preds.length); return { brier: +(b / n).toFixed(4), logloss: +(l / n).toFixed(4), rps: +(r / n).toFixed(4), accuracy: +(correct / n).toFixed(4), }; } function main(): void { const rows = parse(); const tValFrom = new Date(VAL_FROM).getTime(); const tTestFrom = new Date(TEST_FROM).getTime(); const tTestTo = new Date(TEST_TO).getTime(); // ---------- pass 1: Elo walks + form signal + Elo-DC param fit on pre-2018 ---------- const elo = new Map(); const get = (t: string) => elo.get(t) ?? START_ELO; let sumTotal = 0, nCal = 0, sxy = 0, sxx = 0; const calib: { d: number; h: number; a: number }[] = []; /** pre-match Elo per row index (so later passes never see post-match info) */ const preElo: { eh: number; ea: number }[] = new Array(rows.length); /** pre-match fast Elo (same walk at K×m) per multiplier, for the β blend */ const eloFast = new Map>(M_GRID.map((m) => [m, new Map()])); const preEloFast = new Map(M_GRID.map((m) => [m, new Array(rows.length)])); /** pre-match recent form per side: mean Elo surprise (s − E) over the last * FORM_WINDOW non-friendly matches — strictly pre-match, like preElo */ const formArr = new Map(); const formH = new Float64Array(rows.length); const formA = new Float64Array(rows.length); const formMean = (t: string): number => { const a = formArr.get(t); return a && a.length ? a.reduce((s, x) => s + x, 0) / a.length : 0; }; for (let i = 0; i < rows.length; i++) { const r = rows[i]!; const eh = get(r.home), ea = get(r.away); preElo[i] = { eh, ea }; const homeAdv = r.neutral ? 0 : HOME_ADV_ELO; if (r.date >= PARAM_FROM && r.t < tValFrom) { const eff = eh - ea + homeAdv; sumTotal += r.hs + r.as; nCal++; sxy += eff * (r.hs - r.as); sxx += eff * eff; calib.push({ d: eff, h: r.hs, a: r.as }); } for (const m of M_GRID) { const fm = eloFast.get(m)!; const feh = fm.get(r.home) ?? START_ELO, fea = fm.get(r.away) ?? START_ELO; preEloFast.get(m)![i] = { eh: feh, ea: fea }; const fd = eloDelta(r.hs, r.as, feh, fea, importanceWeight(r.tournament) * m, homeAdv); fm.set(r.home, feh + fd); fm.set(r.away, fea - fd); } formH[i] = formMean(r.home); formA[i] = formMean(r.away); if (r.tournament !== 'Friendly') { const eHome = expectedHome(eh, ea, homeAdv); const sHome = r.hs > r.as ? 1 : r.hs < r.as ? 0 : 0.5; const push = (t: string, v: number): void => { const a = formArr.get(t) ?? []; a.push(v); if (a.length > FORM_WINDOW) a.shift(); formArr.set(t, a); }; push(r.home, sHome - eHome); push(r.away, (1 - sHome) - (1 - eHome)); } const d = eloDelta(r.hs, r.as, eh, ea, importanceWeight(r.tournament), homeAdv); elo.set(r.home, eh + d); elo.set(r.away, ea - d); } const avgGoals = sumTotal / Math.max(1, nCal); const goalsPerElo = sxy / Math.max(1e-9, sxx); let rho = -0.05, bestLL = -Infinity; for (let r0 = -0.2; r0 <= 0.1 + 1e-9; r0 += 0.02) { let ll = 0; for (const m of calib) { const sup = goalsPerElo * m.d; const lh = Math.min(8, Math.max(0.15, avgGoals / 2 + sup / 2)); const la = Math.min(8, Math.max(0.15, avgGoals / 2 - sup / 2)); ll += Math.log(Math.max(1e-12, poissonPmf(lh, m.h) * poissonPmf(la, m.a) * dcTau(m.h, m.a, lh, la, r0))); } if (ll > bestLL) { bestLL = ll; rho = +r0.toFixed(2); } } const params: ModelParams = { goalsPerElo, avgGoals, rho, homeAdvElo: HOME_ADV_ELO }; // ---------- prediction helpers (all read strictly pre-match state) ---------- /** Elo-member lambdas, with optional fast-Elo blend R = (1−β)·slow + β·fast. */ const eloLambdas = (i: number, beta: number, m: number): { lh: number; la: number } => { const slow = preElo[i]!; const fast = beta > 0 ? preEloFast.get(m)![i]! : slow; const eh = (1 - beta) * slow.eh + beta * fast.eh; const ea = (1 - beta) * slow.ea + beta * fast.ea; const { lambdaHome, lambdaAway } = lambdasFromElo(eh, ea, params, rows[i]!.neutral ? 0 : HOME_ADV_ELO); return { lh: lambdaHome, la: lambdaAway }; }; const clampL = (x: number): number => Math.min(8, Math.max(0.1, x)); /** Recent-form goal multiplier exp(γ·form), applied to both ensemble members. */ const withForm = (lh: number, la: number, i: number, gamma: number): { lh: number; la: number } => gamma === 0 ? { lh, la } : { lh: clampL(lh * Math.exp(gamma * formH[i]!)), la: clampL(la * Math.exp(gamma * formA[i]!)) }; // ---------- walk-forward Team-DC over a window, collecting lambdas ---------- type Pred = { i: number; lDcH: number; lDcA: number }; const walkTeamDc = (xi: number, k: number, fromT: number, toT: number, refitDays = REFIT_DAYS): Pred[] => { const preds: Pred[] = []; let fit: TeamDcParams | null = null; let fitAt = -Infinity; const windowMs = FIT_WINDOW_YEARS * 365 * DAY; for (let i = 0; i < rows.length; i++) { const r = rows[i]!; if (r.t < fromT) continue; if (r.t >= toT) break; if (r.t - fitAt > refitDays * DAY) { const train: DcMatch[] = []; for (let j = 0; j < i; j++) { const m = rows[j]!; if (r.t - m.t > windowMs) continue; train.push({ daysAgo: (r.t - m.t) / DAY, home: m.home, away: m.away, homeGoals: m.hs, awayGoals: m.as, neutral: m.neutral, }); } fit = fitTeamDc(train, xi, k); fitAt = r.t; } const { lambdaHome, lambdaAway } = teamDcLambdas(fit!, r.home, r.away, r.neutral); preds.push({ i, lDcH: lambdaHome, lDcA: lambdaAway }); } return preds; }; /** Sweep the ensemble weight w for a fixed (β, m, γ) over cached walk preds. */ const W_STEPS: number[] = []; for (let w = 0; w <= 1.0001; w += 0.05) W_STEPS.push(+w.toFixed(2)); const sweepW = (preds: Pred[], beta: number, m: number, gamma: number): { w: number; rps: number }[] => { const acc = new Float64Array(W_STEPS.length); for (const p of preds) { const o = outcomeOf(rows[p.i]!.hs, rows[p.i]!.as); const el = eloLambdas(p.i, beta, m); const ef = withForm(el.lh, el.la, p.i, gamma); const df = withForm(p.lDcH, p.lDcA, p.i, gamma); const mE = scoreMatrix(ef.lh, ef.la, rho); const mD = scoreMatrix(df.lh, df.la, rho); for (let wi = 0; wi < W_STEPS.length; wi++) { const w = W_STEPS[wi]!; const pooled = w === 0 ? mD : w === 1 ? mE : poolMatrices(mE, mD, w); acc[wi] += rps(probsOf(pooled), o); } } return W_STEPS.map((w, wi) => ({ w, rps: acc[wi]! / preds.length })); }; let best = { xi: PREV.xi, k: PREV.k, w: PREV.w, m: PREV.m, beta: 0, gamma: 0, rps: Infinity }; let prevValRps = Infinity; // validation RPS of the previously shipped config if (QUICK) { // ---------- quick mode: score only shipped + previous on validation ---------- console.log('quick mode — scoring shipped + previous configs on validation…'); const predsShipped = walkTeamDc(SHIPPED.xi, SHIPPED.k, tValFrom, tTestFrom); const predsPrev = SHIPPED.xi === PREV.xi && SHIPPED.k === PREV.k ? predsShipped : walkTeamDc(PREV.xi, PREV.k, tValFrom, tTestFrom); const shippedRps = sweepW(predsShipped, SHIPPED.beta, SHIPPED.m, SHIPPED.gamma).find((x) => x.w === SHIPPED.w)!.rps; prevValRps = sweepW(predsPrev, PREV.beta, PREV.m, PREV.gamma).find((x) => x.w === PREV.w)!.rps; best = { ...SHIPPED, rps: shippedRps }; } else { // ---------- validation stage 1: tune ξ, k, w (β=0, γ=0) ---------- console.log('validation stage 1 — ξ/k/w grid (β=0, γ=0)…'); const valWalks = new Map(); for (const xi of XI_GRID) { for (const k of K_GRID) { const preds = walkTeamDc(xi, k, tValFrom, tTestFrom); valWalks.set(`${xi}|${k}`, preds); for (const { w, rps: r } of sweepW(preds, 0, PREV.m, 0)) { if (xi === PREV.xi && k === PREV.k && w === PREV.w) prevValRps = r; if (r < best.rps) best = { xi, k, w, m: PREV.m, beta: 0, gamma: 0, rps: r }; } console.log(` ξ=${xi} k=${k} done (best so far: ξ=${best.xi} k=${best.k} w=${best.w} rps=${best.rps.toFixed(4)})`); } } // ---------- validation stage 2: fast-Elo blend (m, β) at the stage-1 ξ/k ---------- console.log('validation stage 2 — fast-Elo blend (m, β)…'); const stagePreds = valWalks.get(`${best.xi}|${best.k}`)!; for (const m of M_GRID) { for (const beta of BETA_GRID) { if (beta === 0) continue; // control already covered by stage 1 for (const { w, rps: r } of sweepW(stagePreds, beta, m, 0)) { if (r < best.rps) best = { ...best, m, beta, w, rps: r }; } } console.log(` m=${m} swept (best: β=${best.beta} m=${best.m} w=${best.w} rps=${best.rps.toFixed(4)})`); } // ---------- validation stage 3: recent-form multiplier γ at the winners ---------- console.log('validation stage 3 — form multiplier γ…'); for (const gamma of GAMMA_GRID) { if (gamma === 0) continue; // control covered above for (const { w, rps: r } of sweepW(stagePreds, best.beta, best.m, gamma)) { if (r < best.rps) best = { ...best, gamma, w, rps: r }; } } } console.log(` champion: ξ=${best.xi} k=${best.k} w=${best.w} β=${best.beta} m=${best.m} γ=${best.gamma} rps=${best.rps.toFixed(4)}`); // ---------- decision rule: ship only a validated improvement ---------- const gain = prevValRps - best.rps; const shipChange = gain >= MIN_VAL_GAIN; const shipped = shipChange ? best : { ...PREV, rps: prevValRps }; console.log(`previous config validation rps=${prevValRps.toFixed(4)} | champion gain=${gain.toFixed(4)} → ${shipChange ? 'SHIP candidate' : 'KEEP previous (bar not met)'}`); // ---------- in-tournament refit cadence check (validation only) ---------- console.log('validating 7-day Team-DC refit cadence…'); const preds7 = walkTeamDc(shipped.xi, shipped.k, tValFrom, tTestFrom, 7); const refit7Rps = sweepW(preds7, shipped.beta, shipped.m, shipped.gamma).find((x) => x.w === shipped.w)!.rps; const refit7Ok = refit7Rps <= shipped.rps + MIN_VAL_GAIN; console.log(` refit=7d rps=${refit7Rps.toFixed(4)} vs ${REFIT_DAYS}d rps=${shipped.rps.toFixed(4)} → nightly refit ${refit7Ok ? 'ENABLED' : 'disabled'}`); // ---------- test window: shipped + candidate/previous + members + baselines ---------- console.log(`testing on ${TEST_FROM}–${TEST_TO} with shipped ξ=${shipped.xi} k=${shipped.k} w=${shipped.w} β=${shipped.beta} m=${shipped.m} γ=${shipped.gamma}…`); const testPreds = walkTeamDc(shipped.xi, shipped.k, tTestFrom, tTestTo); const testPredsAlt = shipped.xi === PREV.xi && shipped.k === PREV.k ? testPreds : walkTeamDc(shipChange ? PREV.xi : best.xi, shipChange ? PREV.k : best.k, tTestFrom, tTestTo); const altCfg = shipChange ? { ...PREV } : { xi: best.xi, k: best.k, w: best.w, m: best.m, beta: best.beta, gamma: best.gamma }; const altByIdx = new Map(testPredsAlt.map((p) => [p.i, p])); const setEnsemble: { p: Probs; o: Outcome }[] = []; const setAlt: { p: Probs; o: Outcome }[] = []; const setElo: { p: Probs; o: Outcome }[] = []; const setDc: { p: Probs; o: Outcome }[] = []; const setUniform: { p: Probs; o: Outcome }[] = []; const setRate: { p: Probs; o: Outcome }[] = []; const reliabilityRaw: { p: number; y: number }[] = []; let wcCorrect = 0, wcN = 0; // outcome base rates over the test window let th = 0, td = 0, ta = 0, tn = 0; for (const p of testPreds) { const o = outcomeOf(rows[p.i]!.hs, rows[p.i]!.as); if (o === 'h') th++; else if (o === 'd') td++; else ta++; tn++; } const rate: Probs = { h: th / tn, d: td / tn, a: ta / tn }; /** pooled outcome probs for one test row under a full config */ const pooledProbs = (p: Pred, cfg: { w: number; m: number; beta: number; gamma: number }): Probs => { const el = eloLambdas(p.i, cfg.beta, cfg.m); const ef = withForm(el.lh, el.la, p.i, cfg.gamma); const df = withForm(p.lDcH, p.lDcA, p.i, cfg.gamma); const mE = scoreMatrix(ef.lh, ef.la, rho); const mD = scoreMatrix(df.lh, df.la, rho); return probsOf(cfg.w === 0 ? mD : cfg.w === 1 ? mE : poolMatrices(mE, mD, cfg.w)); }; for (const p of testPreds) { const r = rows[p.i]!; const o = outcomeOf(r.hs, r.as); const el = eloLambdas(p.i, 0, PREV.m); const pe = probsOf(scoreMatrix(el.lh, el.la, rho)); const pd = probsOf(scoreMatrix(p.lDcH, p.lDcA, rho)); const pp = pooledProbs(p, shipped); const alt = altByIdx.get(p.i); if (alt) setAlt.push({ p: pooledProbs(alt, altCfg), o }); setElo.push({ p: pe, o }); setDc.push({ p: pd, o }); setEnsemble.push({ p: pp, o }); setUniform.push({ p: { h: 1 / 3, d: 1 / 3, a: 1 / 3 }, o }); setRate.push({ p: rate, o }); reliabilityRaw.push({ p: pp.h, y: o === 'h' ? 1 : 0 }, { p: pp.d, y: o === 'd' ? 1 : 0 }, { p: pp.a, y: o === 'a' ? 1 : 0 }); if (r.tournament.includes('FIFA World Cup') && !r.tournament.includes('qualification')) { wcN++; if (argmax(pp) === o) wcCorrect++; } } const bins = Array.from({ length: 10 }, () => ({ sp: 0, sy: 0, n: 0 })); for (const { p, y } of reliabilityRaw) { const idx = Math.min(9, Math.floor(p * 10)); bins[idx]!.sp += p; bins[idx]!.sy += y; bins[idx]!.n++; } const reliability = bins.map((b) => ({ predicted: b.n ? b.sp / b.n : 0, observed: b.n ? b.sy / b.n : 0, count: b.n })); const totalN = reliabilityRaw.length; const ece = +bins.reduce((s, b) => s + (b.n ? (b.n / totalN) * Math.abs(b.sp / b.n - b.sy / b.n) : 0), 0).toFixed(3); const altName = shipChange ? 'Previous ensemble (v3)' : 'Recency candidate (failed validation bar)'; const out = { generatedAt: new Date().toISOString(), paramFrom: PARAM_FROM, trainEnd: VAL_FROM, validation: { from: VAL_FROM, to: TEST_FROM, tuned: { xi: shipped.xi, shrinkK: shipped.k, ensembleW: shipped.w, fastM: shipped.m, fastBeta: shipped.beta, formGamma: shipped.gamma, }, }, testFrom: TEST_FROM, testTo: TEST_TO, tested: testPreds.length, params, model: scoreSet(setEnsemble), // the SHIPPED model = ensemble variants: [ { name: 'Ensemble (shipped)', ...scoreSet(setEnsemble) }, { name: altName, ...scoreSet(setAlt) }, { name: 'Elo–Dixon-Coles', ...scoreSet(setElo) }, { name: 'Team Dixon-Coles', ...scoreSet(setDc) }, ], baselines: { uniform: scoreSet(setUniform), baseRate: scoreSet(setRate), eloOnly: scoreSet(setElo) }, reliability, ece, worldCup: { tested: wcN, accuracy: wcN ? +(wcCorrect / wcN).toFixed(4) : 0 }, // the v4 recency decision, recorded for honesty + reproducibility recency: { minValGain: MIN_VAL_GAIN, previous: { ...PREV, valRps: +prevValRps.toFixed(4) }, champion: { xi: best.xi, k: best.k, w: best.w, m: best.m, beta: best.beta, gamma: best.gamma, valRps: +best.rps.toFixed(4) }, shippedChange: shipChange, nightlyRefit: { refitDays: 7, valRps: +refit7Rps.toFixed(4), baselineValRps: +shipped.rps.toFixed(4), enabled: refit7Ok }, }, }; mkdirSync(OUT_DIR, { recursive: true }); writeFileSync(join(OUT_DIR, 'backtest.json'), JSON.stringify(out)); console.log(`\nbacktest → public/data/backtest.json (${out.tested} test matches, ${TEST_FROM}→${TEST_TO})`); for (const v of out.variants) console.log(` ${v.name.padEnd(20)} rps ${v.rps} brier ${v.brier} logloss ${v.logloss} acc ${(v.accuracy * 100).toFixed(1)}%`); console.log(` baselines: uniform rps ${out.baselines.uniform.rps} | baseRate rps ${out.baselines.baseRate.rps}`); console.log(` ECE ${ece} | WC accuracy ${(out.worldCup.accuracy * 100).toFixed(1)}% (${wcN})`); } main();