cup26/scripts/buildBacktest.ts

// v3 bake-off backtest with a strict three-way split — the evidence behind the
// shipped model. NO information leaks forward:
//   params   (pre-2018)  : Elo→goals calibration, Dixon-Coles rho
//   validate (2018–2022) : tune Team-DC decay ξ, shrinkage k, ensemble weight w
//   test     (2022–2026) : untouched final numbers for every variant + baselines
// Output → public/data/backtest.json (Methodology page).
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
import { canonicalTeam } from '../src/lib/teams';
import { HOME_ADV_ELO, importanceWeight, eloDelta } from '../src/lib/model/elo';
import { lambdasFromElo, scoreMatrix, outcomeProbs, poissonPmf, type ModelParams } from '../src/lib/model/poisson';
import { fitTeamDc, teamDcLambdas, poolMatrices, type DcMatch, type TeamDcParams } from '../src/lib/model/teamDc';

const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const CSV = join(ROOT, 'data', 'raw', 'international-results.csv');
const OUT_DIR = join(ROOT, 'public', 'data');

const START_ELO = 1500;
const PARAM_FROM = '2006-01-01';
const VAL_FROM = '2018-01-01';
const TEST_FROM = '2022-01-01';
const TEST_TO = '2026-06-01';
const REFIT_DAYS = 60;
const FIT_WINDOW_YEARS = 15;

const XI_GRID = [0.5, 1, 1.5, 2, 2.5];
const K_GRID = [5, 10, 20];

interface Row { date: string; home: string; away: string; hs: number; as: number; tournament: string; neutral: boolean; t: number }
type Probs = { h: number; d: number; a: number };
type Outcome = 'h' | 'd' | 'a';

const DAY = 24 * 60 * 60 * 1000;

function parse(): Row[] {
  const lines = readFileSync(CSV, 'utf8').split('\n');
  const rows: Row[] = [];
  for (let i = 1; i < lines.length; i++) {
    const c = lines[i]?.split(',');
    if (!c || c.length < 9) continue;
    const hs = Number(c[3]); const as = Number(c[4]);
    if (!Number.isFinite(hs) || !Number.isFinite(as)) continue;
    rows.push({
      date: c[0]!, home: canonicalTeam(c[1]!), away: canonicalTeam(c[2]!), hs, as,
      tournament: c[5]!, neutral: c[8]!.trim().toUpperCase() === 'TRUE',
      t: new Date(c[0]!).getTime(),
    });
  }
  rows.sort((a, b) => a.t - b.t);
  return rows;
}

function dcTau(h: number, a: number, lh: number, la: number, rho: number): number {
  if (h === 0 && a === 0) return 1 - lh * la * rho;
  if (h === 0 && a === 1) return 1 + lh * rho;
  if (h === 1 && a === 0) return 1 + la * rho;
  if (h === 1 && a === 1) return 1 - rho;
  return 1;
}

const outcomeOf = (hs: number, as: number): Outcome => (hs > as ? 'h' : hs < as ? 'a' : 'd');
const probsOf = (m: number[][]): Probs => { const p = outcomeProbs(m); return { h: p.home, d: p.draw, a: p.away }; };

function brier(p: Probs, o: Outcome): number {
  return (p.h - (o === 'h' ? 1 : 0)) ** 2 + (p.d - (o === 'd' ? 1 : 0)) ** 2 + (p.a - (o === 'a' ? 1 : 0)) ** 2;
}
const logloss = (p: Probs, o: Outcome): number => -Math.log(Math.max(1e-12, p[o]));
function rps(p: Probs, o: Outcome): number {
  const y1 = o === 'h' ? 1 : 0;
  const y2 = o === 'a' ? 0 : 1;
  return ((p.h - y1) ** 2 + (p.h + p.d - y2) ** 2) / 2;
}
const argmax = (p: Probs): Outcome => (p.h >= p.d && p.h >= p.a ? 'h' : p.d >= p.a ? 'd' : 'a');

function scoreSet(preds: { p: Probs; o: Outcome }[]) {
  let b = 0, l = 0, r = 0, correct = 0;
  for (const { p, o } of preds) { b += brier(p, o); l += logloss(p, o); r += rps(p, o); if (argmax(p) === o) correct++; }
  const n = Math.max(1, preds.length);
  return {
    brier: +(b / n).toFixed(4), logloss: +(l / n).toFixed(4),
    rps: +(r / n).toFixed(4), accuracy: +(correct / n).toFixed(4),
  };
}

function main(): void {
  const rows = parse();
  const tValFrom = new Date(VAL_FROM).getTime();
  const tTestFrom = new Date(TEST_FROM).getTime();
  const tTestTo = new Date(TEST_TO).getTime();

  // ---------- pass 1: Elo walk + Elo-DC param fit on pre-2018 ----------
  const elo = new Map<string, number>();
  const get = (t: string) => elo.get(t) ?? START_ELO;
  let sumTotal = 0, nCal = 0, sxy = 0, sxx = 0;
  const calib: { d: number; h: number; a: number }[] = [];
  /** pre-match Elo per row index (so later passes never see post-match info) */
  const preElo: { eh: number; ea: number }[] = new Array(rows.length);

  for (let i = 0; i < rows.length; i++) {
    const r = rows[i]!;
    const eh = get(r.home), ea = get(r.away);
    preElo[i] = { eh, ea };
    const homeAdv = r.neutral ? 0 : HOME_ADV_ELO;
    if (r.date >= PARAM_FROM && r.t < tValFrom) {
      const eff = eh - ea + homeAdv;
      sumTotal += r.hs + r.as; nCal++; sxy += eff * (r.hs - r.as); sxx += eff * eff;
      calib.push({ d: eff, h: r.hs, a: r.as });
    }
    const d = eloDelta(r.hs, r.as, eh, ea, importanceWeight(r.tournament), homeAdv);
    elo.set(r.home, eh + d); elo.set(r.away, ea - d);
  }

  const avgGoals = sumTotal / Math.max(1, nCal);
  const goalsPerElo = sxy / Math.max(1e-9, sxx);
  let rho = -0.05, bestLL = -Infinity;
  for (let r0 = -0.2; r0 <= 0.1 + 1e-9; r0 += 0.02) {
    let ll = 0;
    for (const m of calib) {
      const sup = goalsPerElo * m.d;
      const lh = Math.min(8, Math.max(0.15, avgGoals / 2 + sup / 2));
      const la = Math.min(8, Math.max(0.15, avgGoals / 2 - sup / 2));
      ll += Math.log(Math.max(1e-12, poissonPmf(lh, m.h) * poissonPmf(la, m.a) * dcTau(m.h, m.a, lh, la, r0)));
    }
    if (ll > bestLL) { bestLL = ll; rho = +r0.toFixed(2); }
  }
  const params: ModelParams = { goalsPerElo, avgGoals, rho, homeAdvElo: HOME_ADV_ELO };

  const eloMatrix = (i: number): number[][] => {
    const r = rows[i]!;
    const { lambdaHome, lambdaAway } = lambdasFromElo(preElo[i]!.eh, preElo[i]!.ea, params, r.neutral ? 0 : HOME_ADV_ELO);
    return scoreMatrix(lambdaHome, lambdaAway, rho);
  };

  // ---------- walk-forward Team-DC over a window, collecting matrices ----------
  type Pred = { i: number; mDc: number[][] };
  const walkTeamDc = (xi: number, k: number, fromT: number, toT: number): Pred[] => {
    const preds: Pred[] = [];
    let fit: TeamDcParams | null = null;
    let fitAt = -Infinity;
    const windowMs = FIT_WINDOW_YEARS * 365 * DAY;
    for (let i = 0; i < rows.length; i++) {
      const r = rows[i]!;
      if (r.t < fromT) continue;
      if (r.t >= toT) break;
      if (r.t - fitAt > REFIT_DAYS * DAY) {
        const train: DcMatch[] = [];
        for (let j = 0; j < i; j++) {
          const m = rows[j]!;
          if (r.t - m.t > windowMs) continue;
          train.push({
            daysAgo: (r.t - m.t) / DAY, home: m.home, away: m.away,
            homeGoals: m.hs, awayGoals: m.as, neutral: m.neutral,
          });
        }
        fit = fitTeamDc(train, xi, k);
        fitAt = r.t;
      }
      const { lambdaHome, lambdaAway } = teamDcLambdas(fit!, r.home, r.away, r.neutral);
      preds.push({ i, mDc: scoreMatrix(lambdaHome, lambdaAway, rho) });
    }
    return preds;
  };

  // ---------- validation: tune ξ, k, then w ----------
  console.log('tuning on validation (2018–2022)…');
  let best = { xi: 1, k: 10, w: 0.5, rps: Infinity };
  for (const xi of XI_GRID) {
    for (const k of K_GRID) {
      const preds = walkTeamDc(xi, k, tValFrom, tTestFrom);
      // pure Team-DC score (w=0 candidate) and weight sweep against Elo
      for (let w = 0; w <= 1.0001; w += 0.05) {
        let r = 0;
        for (const p of preds) {
          const pooled = w === 0 ? p.mDc : w === 1 ? eloMatrix(p.i) : poolMatrices(eloMatrix(p.i), p.mDc, w);
          r += rps(probsOf(pooled), outcomeOf(rows[p.i]!.hs, rows[p.i]!.as));
        }
        r /= preds.length;
        if (r < best.rps) best = { xi, k, w: +w.toFixed(2), rps: r };
      }
      console.log(`  ξ=${xi} k=${k} done (best so far: ξ=${best.xi} k=${best.k} w=${best.w} rps=${best.rps.toFixed(4)})`);
    }
  }

  // ---------- test window: final scores for every variant ----------
  console.log(`testing on ${TEST_FROM}–${TEST_TO} with ξ=${best.xi} k=${best.k} w=${best.w}…`);
  const testPreds = walkTeamDc(best.xi, best.k, tTestFrom, tTestTo);

  const setEnsemble: { p: Probs; o: Outcome }[] = [];
  const setElo: { p: Probs; o: Outcome }[] = [];
  const setDc: { p: Probs; o: Outcome }[] = [];
  const setUniform: { p: Probs; o: Outcome }[] = [];
  const setRate: { p: Probs; o: Outcome }[] = [];
  const reliabilityRaw: { p: number; y: number }[] = [];
  let wcCorrect = 0, wcN = 0;

  // outcome base rates over the test window
  let th = 0, td = 0, ta = 0, tn = 0;
  for (const p of testPreds) {
    const o = outcomeOf(rows[p.i]!.hs, rows[p.i]!.as);
    if (o === 'h') th++; else if (o === 'd') td++; else ta++; tn++;
  }
  const rate: Probs = { h: th / tn, d: td / tn, a: ta / tn };

  for (const p of testPreds) {
    const r = rows[p.i]!;
    const o = outcomeOf(r.hs, r.as);
    const mE = eloMatrix(p.i);
    const pooled = poolMatrices(mE, p.mDc, best.w);
    const pe = probsOf(mE), pd = probsOf(p.mDc), pp = probsOf(pooled);
    setElo.push({ p: pe, o });
    setDc.push({ p: pd, o });
    setEnsemble.push({ p: pp, o });
    setUniform.push({ p: { h: 1 / 3, d: 1 / 3, a: 1 / 3 }, o });
    setRate.push({ p: rate, o });
    reliabilityRaw.push({ p: pp.h, y: o === 'h' ? 1 : 0 }, { p: pp.d, y: o === 'd' ? 1 : 0 }, { p: pp.a, y: o === 'a' ? 1 : 0 });
    if (r.tournament.includes('FIFA World Cup') && !r.tournament.includes('qualification')) {
      wcN++; if (argmax(pp) === o) wcCorrect++;
    }
  }

  const bins = Array.from({ length: 10 }, () => ({ sp: 0, sy: 0, n: 0 }));
  for (const { p, y } of reliabilityRaw) {
    const idx = Math.min(9, Math.floor(p * 10));
    bins[idx]!.sp += p; bins[idx]!.sy += y; bins[idx]!.n++;
  }
  const reliability = bins.map((b) => ({ predicted: b.n ? b.sp / b.n : 0, observed: b.n ? b.sy / b.n : 0, count: b.n }));
  const totalN = reliabilityRaw.length;
  const ece = +bins.reduce((s, b) => s + (b.n ? (b.n / totalN) * Math.abs(b.sp / b.n - b.sy / b.n) : 0), 0).toFixed(3);

  const out = {
    generatedAt: new Date().toISOString(),
    paramFrom: PARAM_FROM,
    trainEnd: VAL_FROM,
    validation: { from: VAL_FROM, to: TEST_FROM, tuned: { xi: best.xi, shrinkK: best.k, ensembleW: best.w } },
    testFrom: TEST_FROM,
    testTo: TEST_TO,
    tested: testPreds.length,
    params,
    model: scoreSet(setEnsemble), // the SHIPPED model = ensemble
    variants: [
      { name: 'Ensemble (shipped)', ...scoreSet(setEnsemble) },
      { name: 'Elo–Dixon-Coles', ...scoreSet(setElo) },
      { name: 'Team Dixon-Coles', ...scoreSet(setDc) },
    ],
    baselines: { uniform: scoreSet(setUniform), baseRate: scoreSet(setRate), eloOnly: scoreSet(setElo) },
    reliability,
    ece,
    worldCup: { tested: wcN, accuracy: wcN ? +(wcCorrect / wcN).toFixed(4) : 0 },
  };

  mkdirSync(OUT_DIR, { recursive: true });
  writeFileSync(join(OUT_DIR, 'backtest.json'), JSON.stringify(out));
  console.log(`\nbacktest → public/data/backtest.json (${out.tested} test matches, ${TEST_FROM}→${TEST_TO})`);
  for (const v of out.variants) console.log(`  ${v.name.padEnd(20)} rps ${v.rps}  brier ${v.brier}  logloss ${v.logloss}  acc ${(v.accuracy * 100).toFixed(1)}%`);
  console.log(`  baselines: uniform rps ${out.baselines.uniform.rps} | baseRate rps ${out.baselines.baseRate.rps}`);
  console.log(`  ECE ${ece} | WC accuracy ${(out.worldCup.accuracy * 100).toFixed(1)}% (${wcN})`);
}

main();