cup26/scripts/buildRatings.ts

// international-results.csv → public/data/ratings.json
// Walks every men's international 1872→now in date order, maintaining a
// World-Football-Elo rating per team. Then calibrates the goals model
// (goals-per-Elo slope, mean goals) on the modern era and MLE-fits Dixon-Coles
// rho. The runtime re-uses these ratings + params to predict and simulate.
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
import { canonicalTeam, ALL_TEAMS } from '../src/lib/teams';
import { HOME_ADV_ELO, importanceWeight, eloDelta, expectedHome } from '../src/lib/model/elo';
import { poissonPmf } from '../src/lib/model/poisson';
import { fitTeamDc, type DcMatch } from '../src/lib/model/teamDc';

// Ensemble hyper-params — tuned on the 2018–2022 validation window and verified
// out-of-sample on 2022–2026 by scripts/buildBacktest.ts. Keep in sync with it.
// v4 recency config: more ensemble weight on Team-DC, plus a fast Elo walk
// (K×3) blended into the Elo member at β=0.3, so recent results count harder.
const TEAMDC_XI = 0.25;
const TEAMDC_SHRINK_K = 3;
const ENSEMBLE_W = 0.45; // log-pool weight on the Elo-DC member
const FAST_M = 3; // fast-Elo K multiplier
const FAST_BETA = 0.3; // blend weight on the fast walk
const FIT_WINDOW_YEARS = 15;

const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const CSV = join(ROOT, 'data', 'raw', 'international-results.csv');
const OUT_DIR = join(ROOT, 'public', 'data');

const START_ELO = 1500;
const CALIB_FROM = '2010-01-01'; // modern-era window for the goals calibration

interface Row {
  date: string; home: string; away: string; hs: number; as: number;
  tournament: string; neutral: boolean;
}

function parseCsv(text: string): Row[] {
  const lines = text.split('\n');
  const rows: Row[] = [];
  for (let i = 1; i < lines.length; i++) {
    const line = lines[i];
    if (!line) continue;
    const c = line.split(','); // this dataset has no embedded commas
    if (c.length < 9) continue;
    const hs = Number(c[3]);
    const as = Number(c[4]);
    if (!Number.isFinite(hs) || !Number.isFinite(as)) continue; // skip NA / future fixtures
    rows.push({
      date: c[0]!, home: c[1]!, away: c[2]!, hs, as,
      tournament: c[5]!, neutral: c[8]!.trim().toUpperCase() === 'TRUE',
    });
  }
  rows.sort((a, b) => a.date.localeCompare(b.date));
  return rows;
}

function dcTau(h: number, a: number, lh: number, la: number, rho: number): number {
  if (h === 0 && a === 0) return 1 - lh * la * rho;
  if (h === 0 && a === 1) return 1 + lh * rho;
  if (h === 1 && a === 0) return 1 + la * rho;
  if (h === 1 && a === 1) return 1 - rho;
  return 1;
}

function main(): void {
  const rows = parseCsv(readFileSync(CSV, 'utf8'));
  const elo = new Map<string, number>();
  const get = (t: string) => elo.get(t) ?? START_ELO;
  const eloFast = new Map<string, number>();
  const getFast = (t: string) => eloFast.get(t) ?? START_ELO;

  // Calibration accumulators (modern era only).
  let sumTotal = 0, nCalib = 0;
  let sxy = 0, sxx = 0; // regression of goal-diff on effective Elo diff
  const calib: { d: number; h: number; a: number }[] = [];

  let lastDate = '';
  for (const r of rows) {
    const home = canonicalTeam(r.home);
    const away = canonicalTeam(r.away);
    const eh = get(home);
    const ea = get(away);
    const homeAdv = r.neutral ? 0 : HOME_ADV_ELO;
    const effDiff = eh - ea + homeAdv;

    if (r.date >= CALIB_FROM) {
      const gd = r.hs - r.as;
      sumTotal += r.hs + r.as; nCalib++;
      sxy += effDiff * gd; sxx += effDiff * effDiff;
      calib.push({ d: effDiff, h: r.hs, a: r.as });
    }

    const k = importanceWeight(r.tournament);
    const delta = eloDelta(r.hs, r.as, eh, ea, k, homeAdv);
    elo.set(home, eh + delta);
    elo.set(away, ea - delta);
    const feh = getFast(home), fea = getFast(away);
    const fastDelta = eloDelta(r.hs, r.as, feh, fea, k * FAST_M, homeAdv);
    eloFast.set(home, feh + fastDelta);
    eloFast.set(away, fea - fastDelta);
    lastDate = r.date;
  }

  const avgGoals = sumTotal / Math.max(1, nCalib);
  const goalsPerElo = sxy / Math.max(1e-9, sxx);

  // MLE-fit Dixon-Coles rho on the modern era (classic DC likelihood: the four
  // low-score cells, no renormalization). Grid search keeps it simple + robust.
  let bestRho = 0, bestLL = -Infinity;
  for (let rho = -0.2; rho <= 0.1 + 1e-9; rho += 0.01) {
    let ll = 0;
    for (const m of calib) {
      const sup = goalsPerElo * m.d;
      const lh = Math.min(8, Math.max(0.15, avgGoals / 2 + sup / 2));
      const la = Math.min(8, Math.max(0.15, avgGoals / 2 - sup / 2));
      const p = poissonPmf(lh, m.h) * poissonPmf(la, m.a) * dcTau(m.h, m.a, lh, la, rho);
      ll += Math.log(Math.max(1e-12, p));
    }
    if (ll > bestLL) { bestLL = ll; bestRho = rho; }
  }
  const rho = Math.round(bestRho * 100) / 100;

  // Sanity: home win rate the model expects vs reality, on the calib window.
  let expHome = 0, n2 = 0;
  for (const r of rows) {
    if (r.date < CALIB_FROM) continue;
    const homeAdv = r.neutral ? 0 : HOME_ADV_ELO;
    // recompute pre-match would need stored ratings; use final as a rough proxy
    expHome += expectedHome(get(canonicalTeam(r.home)), get(canonicalTeam(r.away)), homeAdv);
    n2++;
  }

  const ratings: Record<string, number> = {};
  for (const [team, r] of [...elo.entries()].sort((a, b) => b[1] - a[1])) {
    ratings[team] = Math.round(r * 10) / 10;
  }
  // Ensure every WC team has a rating (default for any never seen).
  for (const t of ALL_TEAMS) if (!(t in ratings)) ratings[t] = START_ELO;
  const ratingsFast: Record<string, number> = {};
  for (const t of Object.keys(ratings)) ratingsFast[t] = Math.round(getFast(t) * 10) / 10;

  // ---- Team-DC ensemble member: fit on the recent window through today ----
  const nowT = new Date(lastDate).getTime();
  const windowMs = FIT_WINDOW_YEARS * 365 * 24 * 60 * 60 * 1000;
  const dcTrain: DcMatch[] = [];
  for (const r of rows) {
    const t = new Date(r.date).getTime();
    if (nowT - t > windowMs) continue;
    dcTrain.push({
      daysAgo: (nowT - t) / (24 * 60 * 60 * 1000),
      home: canonicalTeam(r.home), away: canonicalTeam(r.away),
      homeGoals: r.hs, awayGoals: r.as, neutral: r.neutral,
    });
  }
  const teamDc = fitTeamDc(dcTrain, TEAMDC_XI, TEAMDC_SHRINK_K);
  // Trim to relevant nations: WC qualifiers + any team rated (keeps file small).
  const keep = new Set<string>([...ALL_TEAMS, ...Object.keys(ratings)]);
  teamDc.attack = Object.fromEntries(Object.entries(teamDc.attack).filter(([t]) => keep.has(t)).map(([t, v]) => [t, Math.round(v * 1e4) / 1e4]));
  teamDc.defence = Object.fromEntries(Object.entries(teamDc.defence).filter(([t]) => keep.has(t)).map(([t, v]) => [t, Math.round(v * 1e4) / 1e4]));
  teamDc.mu = Math.round(teamDc.mu * 1e4) / 1e4;
  teamDc.homeMult = Math.round(teamDc.homeMult * 1e4) / 1e4;

  // Isotonic calibration maps — research artifact from scripts/cvSearch.ts
  // (committed; only valid while the shipped config matches its fit config).
  let calibration: unknown = null;
  const calFile = join(ROOT, 'data', 'calibration-maps.json');
  if (existsSync(calFile)) {
    const cal = JSON.parse(readFileSync(calFile, 'utf8')) as {
      config: { xi: number; k: number; w: number; m: number; beta: number };
      maps: unknown;
    };
    const c = cal.config;
    if (c.xi === TEAMDC_XI && c.k === TEAMDC_SHRINK_K && c.w === ENSEMBLE_W && c.m === FAST_M && c.beta === FAST_BETA) {
      calibration = cal.maps;
    } else {
      console.warn('calibration-maps.json was fit for a different config — skipping (rerun scripts/cvSearch.ts)');
    }
  }

  const out = {
    generatedAt: new Date().toISOString(),
    asOf: lastDate,
    matchesProcessed: rows.length,
    calibrationFrom: CALIB_FROM,
    calibrationMatches: nCalib,
    params: {
      goalsPerElo: Math.round(goalsPerElo * 1e6) / 1e6,
      avgGoals: Math.round(avgGoals * 1000) / 1000,
      rho,
      homeAdvElo: HOME_ADV_ELO,
    },
    teamDc,
    ensembleW: ENSEMBLE_W,
    ratingsFast,
    fastBeta: FAST_BETA,
    fastM: FAST_M,
    teamDcFit: { xi: TEAMDC_XI, shrinkK: TEAMDC_SHRINK_K, windowYears: FIT_WINDOW_YEARS },
    ...(calibration ? { calibration } : {}),
    ratings,
  };

  mkdirSync(OUT_DIR, { recursive: true });
  writeFileSync(join(OUT_DIR, 'ratings.json'), JSON.stringify(out));

  // Compact training rows for the server's nightly in-tournament Team-DC refit
  // (absolute dates, so daysAgo can be recomputed as the tournament progresses).
  const dcRows = rows
    .filter((r) => nowT - new Date(r.date).getTime() <= windowMs)
    .map((r) => [r.date, canonicalTeam(r.home), canonicalTeam(r.away), r.hs, r.as, r.neutral ? 1 : 0]);
  writeFileSync(join(OUT_DIR, 'dcTrain.json'), JSON.stringify({ asOf: lastDate, rows: dcRows }));
  console.log(`dcTrain → public/data/dcTrain.json (${dcRows.length} rows, ${FIT_WINDOW_YEARS}y window)`);

  const top = Object.entries(ratings).slice(0, 12).map(([t, r]) => `${t} ${Math.round(r)}`).join(', ');
  console.log(`ratings → public/data/ratings.json  (${rows.length} matches, asOf ${lastDate})`);
  console.log(`params: goalsPerElo=${out.params.goalsPerElo} avgGoals=${out.params.avgGoals} rho=${rho} homeAdvElo=${HOME_ADV_ELO}`);
  console.log(`top: ${top}`);
}

main();