NilsBriggen
716dab2f23
BACKTEST_QUICK=1 skips the hyperparameter grid and scores only the shipped + previous configs — same test metrics, a fraction of the build time on the 1-CPU VPS. The full grid search stays a local activity. Verified byte-level: quick output matches the grid run. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
429 lines
21 KiB
TypeScript
429 lines
21 KiB
TypeScript
// v4 bake-off backtest with a strict three-way split — the evidence behind the
|
||
// shipped model. NO information leaks forward:
|
||
// params (pre-2018) : Elo→goals calibration, Dixon-Coles rho
|
||
// validate (2018–2022) : tune Team-DC decay ξ, shrinkage k, ensemble weight w,
|
||
// plus the v4 recency candidates — a fast-Elo blend
|
||
// (second walk at K×m, blended in with weight β) and a
|
||
// recent-form goal multiplier exp(γ·form), where form is
|
||
// each side's mean Elo surprise (actual − expected) over
|
||
// its last 10 non-friendly matches
|
||
// test (2022–2026) : untouched final numbers for every variant + baselines
|
||
// Decision rule: a recency candidate ships only if it beats the previous shipped
|
||
// config by ≥ MIN_VAL_GAIN RPS on validation. Test is evaluated once, for honesty.
|
||
// Output → public/data/backtest.json (Methodology page).
|
||
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
||
import { fileURLToPath } from 'node:url';
|
||
import { dirname, join } from 'node:path';
|
||
import { canonicalTeam } from '../src/lib/teams';
|
||
import { HOME_ADV_ELO, importanceWeight, eloDelta, expectedHome } from '../src/lib/model/elo';
|
||
import { lambdasFromElo, scoreMatrix, outcomeProbs, poissonPmf, type ModelParams } from '../src/lib/model/poisson';
|
||
import { fitTeamDc, teamDcLambdas, poolMatrices, type DcMatch, type TeamDcParams } from '../src/lib/model/teamDc';
|
||
|
||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||
const CSV = join(ROOT, 'data', 'raw', 'international-results.csv');
|
||
const OUT_DIR = join(ROOT, 'public', 'data');
|
||
|
||
const START_ELO = 1500;
|
||
const PARAM_FROM = '2006-01-01';
|
||
const VAL_FROM = '2018-01-01';
|
||
const TEST_FROM = '2022-01-01';
|
||
const TEST_TO = '2026-06-01';
|
||
const REFIT_DAYS = 60;
|
||
const FIT_WINDOW_YEARS = 15;
|
||
|
||
const XI_GRID = [0.25, 0.5, 0.75, 1, 1.5, 2, 2.5];
|
||
const K_GRID = [3, 5, 10, 20];
|
||
const M_GRID = [2, 3]; // fast-Elo K multiplier
|
||
const BETA_GRID = [0, 0.1, 0.2, 0.3]; // fast-Elo blend weight (0 = control)
|
||
const GAMMA_GRID = [0, 0.05, 0.1, 0.15]; // recent-form goal multiplier (0 = control)
|
||
const FORM_WINDOW = 10; // last N non-friendly matches feeding the form signal
|
||
const MIN_VAL_GAIN = 0.0005; // candidate must beat the previous config by this much
|
||
/** the previously shipped v3 config — the bar every candidate has to clear */
|
||
const PREV = { xi: 0.5, k: 5, w: 0.75, m: 2, beta: 0, gamma: 0 };
|
||
/** the v4 winner (from the full grid run) — keep in sync with buildRatings.ts */
|
||
const SHIPPED = { xi: 0.25, k: 3, w: 0.45, m: 3, beta: 0.3, gamma: 0 };
|
||
/** BACKTEST_QUICK=1 (Docker builds): skip the grid search and just score the
|
||
* shipped + previous configs — the published test numbers are identical, the
|
||
* hyperparam hunt only happens locally. */
|
||
const QUICK = process.env.BACKTEST_QUICK === '1';
|
||
|
||
interface Row { date: string; home: string; away: string; hs: number; as: number; tournament: string; neutral: boolean; t: number }
|
||
type Probs = { h: number; d: number; a: number };
|
||
type Outcome = 'h' | 'd' | 'a';
|
||
|
||
const DAY = 24 * 60 * 60 * 1000;
|
||
|
||
function parse(): Row[] {
|
||
const lines = readFileSync(CSV, 'utf8').split('\n');
|
||
const rows: Row[] = [];
|
||
for (let i = 1; i < lines.length; i++) {
|
||
const c = lines[i]?.split(',');
|
||
if (!c || c.length < 9) continue;
|
||
const hs = Number(c[3]); const as = Number(c[4]);
|
||
if (!Number.isFinite(hs) || !Number.isFinite(as)) continue;
|
||
rows.push({
|
||
date: c[0]!, home: canonicalTeam(c[1]!), away: canonicalTeam(c[2]!), hs, as,
|
||
tournament: c[5]!, neutral: c[8]!.trim().toUpperCase() === 'TRUE',
|
||
t: new Date(c[0]!).getTime(),
|
||
});
|
||
}
|
||
rows.sort((a, b) => a.t - b.t);
|
||
return rows;
|
||
}
|
||
|
||
function dcTau(h: number, a: number, lh: number, la: number, rho: number): number {
|
||
if (h === 0 && a === 0) return 1 - lh * la * rho;
|
||
if (h === 0 && a === 1) return 1 + lh * rho;
|
||
if (h === 1 && a === 0) return 1 + la * rho;
|
||
if (h === 1 && a === 1) return 1 - rho;
|
||
return 1;
|
||
}
|
||
|
||
const outcomeOf = (hs: number, as: number): Outcome => (hs > as ? 'h' : hs < as ? 'a' : 'd');
|
||
const probsOf = (m: number[][]): Probs => { const p = outcomeProbs(m); return { h: p.home, d: p.draw, a: p.away }; };
|
||
|
||
function brier(p: Probs, o: Outcome): number {
|
||
return (p.h - (o === 'h' ? 1 : 0)) ** 2 + (p.d - (o === 'd' ? 1 : 0)) ** 2 + (p.a - (o === 'a' ? 1 : 0)) ** 2;
|
||
}
|
||
const logloss = (p: Probs, o: Outcome): number => -Math.log(Math.max(1e-12, p[o]));
|
||
function rps(p: Probs, o: Outcome): number {
|
||
const y1 = o === 'h' ? 1 : 0;
|
||
const y2 = o === 'a' ? 0 : 1;
|
||
return ((p.h - y1) ** 2 + (p.h + p.d - y2) ** 2) / 2;
|
||
}
|
||
const argmax = (p: Probs): Outcome => (p.h >= p.d && p.h >= p.a ? 'h' : p.d >= p.a ? 'd' : 'a');
|
||
|
||
function scoreSet(preds: { p: Probs; o: Outcome }[]) {
|
||
let b = 0, l = 0, r = 0, correct = 0;
|
||
for (const { p, o } of preds) { b += brier(p, o); l += logloss(p, o); r += rps(p, o); if (argmax(p) === o) correct++; }
|
||
const n = Math.max(1, preds.length);
|
||
return {
|
||
brier: +(b / n).toFixed(4), logloss: +(l / n).toFixed(4),
|
||
rps: +(r / n).toFixed(4), accuracy: +(correct / n).toFixed(4),
|
||
};
|
||
}
|
||
|
||
function main(): void {
|
||
const rows = parse();
|
||
const tValFrom = new Date(VAL_FROM).getTime();
|
||
const tTestFrom = new Date(TEST_FROM).getTime();
|
||
const tTestTo = new Date(TEST_TO).getTime();
|
||
|
||
// ---------- pass 1: Elo walks + form signal + Elo-DC param fit on pre-2018 ----------
|
||
const elo = new Map<string, number>();
|
||
const get = (t: string) => elo.get(t) ?? START_ELO;
|
||
let sumTotal = 0, nCal = 0, sxy = 0, sxx = 0;
|
||
const calib: { d: number; h: number; a: number }[] = [];
|
||
/** pre-match Elo per row index (so later passes never see post-match info) */
|
||
const preElo: { eh: number; ea: number }[] = new Array(rows.length);
|
||
/** pre-match fast Elo (same walk at K×m) per multiplier, for the β blend */
|
||
const eloFast = new Map<number, Map<string, number>>(M_GRID.map((m) => [m, new Map()]));
|
||
const preEloFast = new Map<number, { eh: number; ea: number }[]>(M_GRID.map((m) => [m, new Array(rows.length)]));
|
||
/** pre-match recent form per side: mean Elo surprise (s − E) over the last
|
||
* FORM_WINDOW non-friendly matches — strictly pre-match, like preElo */
|
||
const formArr = new Map<string, number[]>();
|
||
const formH = new Float64Array(rows.length);
|
||
const formA = new Float64Array(rows.length);
|
||
const formMean = (t: string): number => {
|
||
const a = formArr.get(t);
|
||
return a && a.length ? a.reduce((s, x) => s + x, 0) / a.length : 0;
|
||
};
|
||
|
||
for (let i = 0; i < rows.length; i++) {
|
||
const r = rows[i]!;
|
||
const eh = get(r.home), ea = get(r.away);
|
||
preElo[i] = { eh, ea };
|
||
const homeAdv = r.neutral ? 0 : HOME_ADV_ELO;
|
||
if (r.date >= PARAM_FROM && r.t < tValFrom) {
|
||
const eff = eh - ea + homeAdv;
|
||
sumTotal += r.hs + r.as; nCal++; sxy += eff * (r.hs - r.as); sxx += eff * eff;
|
||
calib.push({ d: eff, h: r.hs, a: r.as });
|
||
}
|
||
for (const m of M_GRID) {
|
||
const fm = eloFast.get(m)!;
|
||
const feh = fm.get(r.home) ?? START_ELO, fea = fm.get(r.away) ?? START_ELO;
|
||
preEloFast.get(m)![i] = { eh: feh, ea: fea };
|
||
const fd = eloDelta(r.hs, r.as, feh, fea, importanceWeight(r.tournament) * m, homeAdv);
|
||
fm.set(r.home, feh + fd); fm.set(r.away, fea - fd);
|
||
}
|
||
formH[i] = formMean(r.home);
|
||
formA[i] = formMean(r.away);
|
||
if (r.tournament !== 'Friendly') {
|
||
const eHome = expectedHome(eh, ea, homeAdv);
|
||
const sHome = r.hs > r.as ? 1 : r.hs < r.as ? 0 : 0.5;
|
||
const push = (t: string, v: number): void => {
|
||
const a = formArr.get(t) ?? [];
|
||
a.push(v);
|
||
if (a.length > FORM_WINDOW) a.shift();
|
||
formArr.set(t, a);
|
||
};
|
||
push(r.home, sHome - eHome);
|
||
push(r.away, (1 - sHome) - (1 - eHome));
|
||
}
|
||
const d = eloDelta(r.hs, r.as, eh, ea, importanceWeight(r.tournament), homeAdv);
|
||
elo.set(r.home, eh + d); elo.set(r.away, ea - d);
|
||
}
|
||
|
||
const avgGoals = sumTotal / Math.max(1, nCal);
|
||
const goalsPerElo = sxy / Math.max(1e-9, sxx);
|
||
let rho = -0.05, bestLL = -Infinity;
|
||
for (let r0 = -0.2; r0 <= 0.1 + 1e-9; r0 += 0.02) {
|
||
let ll = 0;
|
||
for (const m of calib) {
|
||
const sup = goalsPerElo * m.d;
|
||
const lh = Math.min(8, Math.max(0.15, avgGoals / 2 + sup / 2));
|
||
const la = Math.min(8, Math.max(0.15, avgGoals / 2 - sup / 2));
|
||
ll += Math.log(Math.max(1e-12, poissonPmf(lh, m.h) * poissonPmf(la, m.a) * dcTau(m.h, m.a, lh, la, r0)));
|
||
}
|
||
if (ll > bestLL) { bestLL = ll; rho = +r0.toFixed(2); }
|
||
}
|
||
const params: ModelParams = { goalsPerElo, avgGoals, rho, homeAdvElo: HOME_ADV_ELO };
|
||
|
||
// ---------- prediction helpers (all read strictly pre-match state) ----------
|
||
/** Elo-member lambdas, with optional fast-Elo blend R = (1−β)·slow + β·fast. */
|
||
const eloLambdas = (i: number, beta: number, m: number): { lh: number; la: number } => {
|
||
const slow = preElo[i]!;
|
||
const fast = beta > 0 ? preEloFast.get(m)![i]! : slow;
|
||
const eh = (1 - beta) * slow.eh + beta * fast.eh;
|
||
const ea = (1 - beta) * slow.ea + beta * fast.ea;
|
||
const { lambdaHome, lambdaAway } = lambdasFromElo(eh, ea, params, rows[i]!.neutral ? 0 : HOME_ADV_ELO);
|
||
return { lh: lambdaHome, la: lambdaAway };
|
||
};
|
||
const clampL = (x: number): number => Math.min(8, Math.max(0.1, x));
|
||
/** Recent-form goal multiplier exp(γ·form), applied to both ensemble members. */
|
||
const withForm = (lh: number, la: number, i: number, gamma: number): { lh: number; la: number } =>
|
||
gamma === 0
|
||
? { lh, la }
|
||
: { lh: clampL(lh * Math.exp(gamma * formH[i]!)), la: clampL(la * Math.exp(gamma * formA[i]!)) };
|
||
|
||
// ---------- walk-forward Team-DC over a window, collecting lambdas ----------
|
||
type Pred = { i: number; lDcH: number; lDcA: number };
|
||
const walkTeamDc = (xi: number, k: number, fromT: number, toT: number, refitDays = REFIT_DAYS): Pred[] => {
|
||
const preds: Pred[] = [];
|
||
let fit: TeamDcParams | null = null;
|
||
let fitAt = -Infinity;
|
||
const windowMs = FIT_WINDOW_YEARS * 365 * DAY;
|
||
for (let i = 0; i < rows.length; i++) {
|
||
const r = rows[i]!;
|
||
if (r.t < fromT) continue;
|
||
if (r.t >= toT) break;
|
||
if (r.t - fitAt > refitDays * DAY) {
|
||
const train: DcMatch[] = [];
|
||
for (let j = 0; j < i; j++) {
|
||
const m = rows[j]!;
|
||
if (r.t - m.t > windowMs) continue;
|
||
train.push({
|
||
daysAgo: (r.t - m.t) / DAY, home: m.home, away: m.away,
|
||
homeGoals: m.hs, awayGoals: m.as, neutral: m.neutral,
|
||
});
|
||
}
|
||
fit = fitTeamDc(train, xi, k);
|
||
fitAt = r.t;
|
||
}
|
||
const { lambdaHome, lambdaAway } = teamDcLambdas(fit!, r.home, r.away, r.neutral);
|
||
preds.push({ i, lDcH: lambdaHome, lDcA: lambdaAway });
|
||
}
|
||
return preds;
|
||
};
|
||
|
||
/** Sweep the ensemble weight w for a fixed (β, m, γ) over cached walk preds. */
|
||
const W_STEPS: number[] = [];
|
||
for (let w = 0; w <= 1.0001; w += 0.05) W_STEPS.push(+w.toFixed(2));
|
||
const sweepW = (preds: Pred[], beta: number, m: number, gamma: number): { w: number; rps: number }[] => {
|
||
const acc = new Float64Array(W_STEPS.length);
|
||
for (const p of preds) {
|
||
const o = outcomeOf(rows[p.i]!.hs, rows[p.i]!.as);
|
||
const el = eloLambdas(p.i, beta, m);
|
||
const ef = withForm(el.lh, el.la, p.i, gamma);
|
||
const df = withForm(p.lDcH, p.lDcA, p.i, gamma);
|
||
const mE = scoreMatrix(ef.lh, ef.la, rho);
|
||
const mD = scoreMatrix(df.lh, df.la, rho);
|
||
for (let wi = 0; wi < W_STEPS.length; wi++) {
|
||
const w = W_STEPS[wi]!;
|
||
const pooled = w === 0 ? mD : w === 1 ? mE : poolMatrices(mE, mD, w);
|
||
acc[wi] += rps(probsOf(pooled), o);
|
||
}
|
||
}
|
||
return W_STEPS.map((w, wi) => ({ w, rps: acc[wi]! / preds.length }));
|
||
};
|
||
|
||
let best = { xi: PREV.xi, k: PREV.k, w: PREV.w, m: PREV.m, beta: 0, gamma: 0, rps: Infinity };
|
||
let prevValRps = Infinity; // validation RPS of the previously shipped config
|
||
|
||
if (QUICK) {
|
||
// ---------- quick mode: score only shipped + previous on validation ----------
|
||
console.log('quick mode — scoring shipped + previous configs on validation…');
|
||
const predsShipped = walkTeamDc(SHIPPED.xi, SHIPPED.k, tValFrom, tTestFrom);
|
||
const predsPrev = SHIPPED.xi === PREV.xi && SHIPPED.k === PREV.k
|
||
? predsShipped
|
||
: walkTeamDc(PREV.xi, PREV.k, tValFrom, tTestFrom);
|
||
const shippedRps = sweepW(predsShipped, SHIPPED.beta, SHIPPED.m, SHIPPED.gamma).find((x) => x.w === SHIPPED.w)!.rps;
|
||
prevValRps = sweepW(predsPrev, PREV.beta, PREV.m, PREV.gamma).find((x) => x.w === PREV.w)!.rps;
|
||
best = { ...SHIPPED, rps: shippedRps };
|
||
} else {
|
||
// ---------- validation stage 1: tune ξ, k, w (β=0, γ=0) ----------
|
||
console.log('validation stage 1 — ξ/k/w grid (β=0, γ=0)…');
|
||
const valWalks = new Map<string, Pred[]>();
|
||
for (const xi of XI_GRID) {
|
||
for (const k of K_GRID) {
|
||
const preds = walkTeamDc(xi, k, tValFrom, tTestFrom);
|
||
valWalks.set(`${xi}|${k}`, preds);
|
||
for (const { w, rps: r } of sweepW(preds, 0, PREV.m, 0)) {
|
||
if (xi === PREV.xi && k === PREV.k && w === PREV.w) prevValRps = r;
|
||
if (r < best.rps) best = { xi, k, w, m: PREV.m, beta: 0, gamma: 0, rps: r };
|
||
}
|
||
console.log(` ξ=${xi} k=${k} done (best so far: ξ=${best.xi} k=${best.k} w=${best.w} rps=${best.rps.toFixed(4)})`);
|
||
}
|
||
}
|
||
|
||
// ---------- validation stage 2: fast-Elo blend (m, β) at the stage-1 ξ/k ----------
|
||
console.log('validation stage 2 — fast-Elo blend (m, β)…');
|
||
const stagePreds = valWalks.get(`${best.xi}|${best.k}`)!;
|
||
for (const m of M_GRID) {
|
||
for (const beta of BETA_GRID) {
|
||
if (beta === 0) continue; // control already covered by stage 1
|
||
for (const { w, rps: r } of sweepW(stagePreds, beta, m, 0)) {
|
||
if (r < best.rps) best = { ...best, m, beta, w, rps: r };
|
||
}
|
||
}
|
||
console.log(` m=${m} swept (best: β=${best.beta} m=${best.m} w=${best.w} rps=${best.rps.toFixed(4)})`);
|
||
}
|
||
|
||
// ---------- validation stage 3: recent-form multiplier γ at the winners ----------
|
||
console.log('validation stage 3 — form multiplier γ…');
|
||
for (const gamma of GAMMA_GRID) {
|
||
if (gamma === 0) continue; // control covered above
|
||
for (const { w, rps: r } of sweepW(stagePreds, best.beta, best.m, gamma)) {
|
||
if (r < best.rps) best = { ...best, gamma, w, rps: r };
|
||
}
|
||
}
|
||
}
|
||
console.log(` champion: ξ=${best.xi} k=${best.k} w=${best.w} β=${best.beta} m=${best.m} γ=${best.gamma} rps=${best.rps.toFixed(4)}`);
|
||
|
||
// ---------- decision rule: ship only a validated improvement ----------
|
||
const gain = prevValRps - best.rps;
|
||
const shipChange = gain >= MIN_VAL_GAIN;
|
||
const shipped = shipChange ? best : { ...PREV, rps: prevValRps };
|
||
console.log(`previous config validation rps=${prevValRps.toFixed(4)} | champion gain=${gain.toFixed(4)} → ${shipChange ? 'SHIP candidate' : 'KEEP previous (bar not met)'}`);
|
||
|
||
// ---------- in-tournament refit cadence check (validation only) ----------
|
||
console.log('validating 7-day Team-DC refit cadence…');
|
||
const preds7 = walkTeamDc(shipped.xi, shipped.k, tValFrom, tTestFrom, 7);
|
||
const refit7Rps = sweepW(preds7, shipped.beta, shipped.m, shipped.gamma).find((x) => x.w === shipped.w)!.rps;
|
||
const refit7Ok = refit7Rps <= shipped.rps + MIN_VAL_GAIN;
|
||
console.log(` refit=7d rps=${refit7Rps.toFixed(4)} vs ${REFIT_DAYS}d rps=${shipped.rps.toFixed(4)} → nightly refit ${refit7Ok ? 'ENABLED' : 'disabled'}`);
|
||
|
||
// ---------- test window: shipped + candidate/previous + members + baselines ----------
|
||
console.log(`testing on ${TEST_FROM}–${TEST_TO} with shipped ξ=${shipped.xi} k=${shipped.k} w=${shipped.w} β=${shipped.beta} m=${shipped.m} γ=${shipped.gamma}…`);
|
||
const testPreds = walkTeamDc(shipped.xi, shipped.k, tTestFrom, tTestTo);
|
||
const testPredsAlt = shipped.xi === PREV.xi && shipped.k === PREV.k
|
||
? testPreds
|
||
: walkTeamDc(shipChange ? PREV.xi : best.xi, shipChange ? PREV.k : best.k, tTestFrom, tTestTo);
|
||
const altCfg = shipChange ? { ...PREV } : { xi: best.xi, k: best.k, w: best.w, m: best.m, beta: best.beta, gamma: best.gamma };
|
||
const altByIdx = new Map(testPredsAlt.map((p) => [p.i, p]));
|
||
|
||
const setEnsemble: { p: Probs; o: Outcome }[] = [];
|
||
const setAlt: { p: Probs; o: Outcome }[] = [];
|
||
const setElo: { p: Probs; o: Outcome }[] = [];
|
||
const setDc: { p: Probs; o: Outcome }[] = [];
|
||
const setUniform: { p: Probs; o: Outcome }[] = [];
|
||
const setRate: { p: Probs; o: Outcome }[] = [];
|
||
const reliabilityRaw: { p: number; y: number }[] = [];
|
||
let wcCorrect = 0, wcN = 0;
|
||
|
||
// outcome base rates over the test window
|
||
let th = 0, td = 0, ta = 0, tn = 0;
|
||
for (const p of testPreds) {
|
||
const o = outcomeOf(rows[p.i]!.hs, rows[p.i]!.as);
|
||
if (o === 'h') th++; else if (o === 'd') td++; else ta++; tn++;
|
||
}
|
||
const rate: Probs = { h: th / tn, d: td / tn, a: ta / tn };
|
||
|
||
/** pooled outcome probs for one test row under a full config */
|
||
const pooledProbs = (p: Pred, cfg: { w: number; m: number; beta: number; gamma: number }): Probs => {
|
||
const el = eloLambdas(p.i, cfg.beta, cfg.m);
|
||
const ef = withForm(el.lh, el.la, p.i, cfg.gamma);
|
||
const df = withForm(p.lDcH, p.lDcA, p.i, cfg.gamma);
|
||
const mE = scoreMatrix(ef.lh, ef.la, rho);
|
||
const mD = scoreMatrix(df.lh, df.la, rho);
|
||
return probsOf(cfg.w === 0 ? mD : cfg.w === 1 ? mE : poolMatrices(mE, mD, cfg.w));
|
||
};
|
||
|
||
for (const p of testPreds) {
|
||
const r = rows[p.i]!;
|
||
const o = outcomeOf(r.hs, r.as);
|
||
const el = eloLambdas(p.i, 0, PREV.m);
|
||
const pe = probsOf(scoreMatrix(el.lh, el.la, rho));
|
||
const pd = probsOf(scoreMatrix(p.lDcH, p.lDcA, rho));
|
||
const pp = pooledProbs(p, shipped);
|
||
const alt = altByIdx.get(p.i);
|
||
if (alt) setAlt.push({ p: pooledProbs(alt, altCfg), o });
|
||
setElo.push({ p: pe, o });
|
||
setDc.push({ p: pd, o });
|
||
setEnsemble.push({ p: pp, o });
|
||
setUniform.push({ p: { h: 1 / 3, d: 1 / 3, a: 1 / 3 }, o });
|
||
setRate.push({ p: rate, o });
|
||
reliabilityRaw.push({ p: pp.h, y: o === 'h' ? 1 : 0 }, { p: pp.d, y: o === 'd' ? 1 : 0 }, { p: pp.a, y: o === 'a' ? 1 : 0 });
|
||
if (r.tournament.includes('FIFA World Cup') && !r.tournament.includes('qualification')) {
|
||
wcN++; if (argmax(pp) === o) wcCorrect++;
|
||
}
|
||
}
|
||
|
||
const bins = Array.from({ length: 10 }, () => ({ sp: 0, sy: 0, n: 0 }));
|
||
for (const { p, y } of reliabilityRaw) {
|
||
const idx = Math.min(9, Math.floor(p * 10));
|
||
bins[idx]!.sp += p; bins[idx]!.sy += y; bins[idx]!.n++;
|
||
}
|
||
const reliability = bins.map((b) => ({ predicted: b.n ? b.sp / b.n : 0, observed: b.n ? b.sy / b.n : 0, count: b.n }));
|
||
const totalN = reliabilityRaw.length;
|
||
const ece = +bins.reduce((s, b) => s + (b.n ? (b.n / totalN) * Math.abs(b.sp / b.n - b.sy / b.n) : 0), 0).toFixed(3);
|
||
|
||
const altName = shipChange
|
||
? 'Previous ensemble (v3)'
|
||
: 'Recency candidate (failed validation bar)';
|
||
const out = {
|
||
generatedAt: new Date().toISOString(),
|
||
paramFrom: PARAM_FROM,
|
||
trainEnd: VAL_FROM,
|
||
validation: {
|
||
from: VAL_FROM, to: TEST_FROM,
|
||
tuned: {
|
||
xi: shipped.xi, shrinkK: shipped.k, ensembleW: shipped.w,
|
||
fastM: shipped.m, fastBeta: shipped.beta, formGamma: shipped.gamma,
|
||
},
|
||
},
|
||
testFrom: TEST_FROM,
|
||
testTo: TEST_TO,
|
||
tested: testPreds.length,
|
||
params,
|
||
model: scoreSet(setEnsemble), // the SHIPPED model = ensemble
|
||
variants: [
|
||
{ name: 'Ensemble (shipped)', ...scoreSet(setEnsemble) },
|
||
{ name: altName, ...scoreSet(setAlt) },
|
||
{ name: 'Elo–Dixon-Coles', ...scoreSet(setElo) },
|
||
{ name: 'Team Dixon-Coles', ...scoreSet(setDc) },
|
||
],
|
||
baselines: { uniform: scoreSet(setUniform), baseRate: scoreSet(setRate), eloOnly: scoreSet(setElo) },
|
||
reliability,
|
||
ece,
|
||
worldCup: { tested: wcN, accuracy: wcN ? +(wcCorrect / wcN).toFixed(4) : 0 },
|
||
// the v4 recency decision, recorded for honesty + reproducibility
|
||
recency: {
|
||
minValGain: MIN_VAL_GAIN,
|
||
previous: { ...PREV, valRps: +prevValRps.toFixed(4) },
|
||
champion: { xi: best.xi, k: best.k, w: best.w, m: best.m, beta: best.beta, gamma: best.gamma, valRps: +best.rps.toFixed(4) },
|
||
shippedChange: shipChange,
|
||
nightlyRefit: { refitDays: 7, valRps: +refit7Rps.toFixed(4), baselineValRps: +shipped.rps.toFixed(4), enabled: refit7Ok },
|
||
},
|
||
};
|
||
|
||
mkdirSync(OUT_DIR, { recursive: true });
|
||
writeFileSync(join(OUT_DIR, 'backtest.json'), JSON.stringify(out));
|
||
console.log(`\nbacktest → public/data/backtest.json (${out.tested} test matches, ${TEST_FROM}→${TEST_TO})`);
|
||
for (const v of out.variants) console.log(` ${v.name.padEnd(20)} rps ${v.rps} brier ${v.brier} logloss ${v.logloss} acc ${(v.accuracy * 100).toFixed(1)}%`);
|
||
console.log(` baselines: uniform rps ${out.baselines.uniform.rps} | baseRate rps ${out.baselines.baseRate.rps}`);
|
||
console.log(` ECE ${ece} | WC accuracy ${(out.worldCup.accuracy * 100).toFixed(1)}% (${wcN})`);
|
||
}
|
||
|
||
main();
|