Isotonic calibration layer — validated leave-one-fold-out, shipped
The CV harness gained a calibration experiment: per-outcome isotonic maps (pool-adjacent-violators) fit walk-forward on the five folds and judged leave-one-fold-out against a pre-registered bar. It cleared it: LOFO ECE improves 16% (0.0146 → 0.0122) with RPS slightly better too (0.1724 → 0.1722). The final maps (fit on all folds) are committed as a research artifact, embedded into ratings.json by buildRatings when the shipped config matches, and applied in matchMatrix by scaling the score-matrix outcome regions — scorelines, Monte Carlo and displayed probabilities all stay mutually consistent. Without the field, output is bit-identical (golden tests). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
File diff suppressed because it is too large
Load Diff
+12
-2
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"generatedAt": "2026-06-11T22:41:08.860Z",
|
||||
"generatedAt": "2026-06-11T23:06:22.047Z",
|
||||
"protocol": {
|
||||
"folds": [
|
||||
{
|
||||
@@ -56,5 +56,15 @@
|
||||
"testRps": 0.1647
|
||||
},
|
||||
"shipChange": false,
|
||||
"runtimeSec": 72
|
||||
"isotonic": {
|
||||
"bar": "LOFO RPS worsens ≤0.0002 AND LOFO ECE improves ≥10%",
|
||||
"lofo": {
|
||||
"rawRps": 0.1724,
|
||||
"calRps": 0.1722,
|
||||
"rawEce": 0.0146,
|
||||
"calEce": 0.0122
|
||||
},
|
||||
"ship": true
|
||||
},
|
||||
"runtimeSec": 73
|
||||
}
|
||||
+19
-1
@@ -3,7 +3,7 @@
|
||||
// World-Football-Elo rating per team. Then calibrates the goals model
|
||||
// (goals-per-Elo slope, mean goals) on the modern era and MLE-fits Dixon-Coles
|
||||
// rho. The runtime re-uses these ratings + params to predict and simulate.
|
||||
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
||||
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { canonicalTeam, ALL_TEAMS } from '../src/lib/teams';
|
||||
@@ -160,6 +160,23 @@ function main(): void {
|
||||
teamDc.mu = Math.round(teamDc.mu * 1e4) / 1e4;
|
||||
teamDc.homeMult = Math.round(teamDc.homeMult * 1e4) / 1e4;
|
||||
|
||||
// Isotonic calibration maps — research artifact from scripts/cvSearch.ts
|
||||
// (committed; only valid while the shipped config matches its fit config).
|
||||
let calibration: unknown = null;
|
||||
const calFile = join(ROOT, 'data', 'calibration-maps.json');
|
||||
if (existsSync(calFile)) {
|
||||
const cal = JSON.parse(readFileSync(calFile, 'utf8')) as {
|
||||
config: { xi: number; k: number; w: number; m: number; beta: number };
|
||||
maps: unknown;
|
||||
};
|
||||
const c = cal.config;
|
||||
if (c.xi === TEAMDC_XI && c.k === TEAMDC_SHRINK_K && c.w === ENSEMBLE_W && c.m === FAST_M && c.beta === FAST_BETA) {
|
||||
calibration = cal.maps;
|
||||
} else {
|
||||
console.warn('calibration-maps.json was fit for a different config — skipping (rerun scripts/cvSearch.ts)');
|
||||
}
|
||||
}
|
||||
|
||||
const out = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
asOf: lastDate,
|
||||
@@ -178,6 +195,7 @@ function main(): void {
|
||||
fastBeta: FAST_BETA,
|
||||
fastM: FAST_M,
|
||||
teamDcFit: { xi: TEAMDC_XI, shrinkK: TEAMDC_SHRINK_K, windowYears: FIT_WINDOW_YEARS },
|
||||
...(calibration ? { calibration } : {}),
|
||||
ratings,
|
||||
};
|
||||
|
||||
|
||||
@@ -257,6 +257,69 @@ function sweepW(
|
||||
return W_STEPS.map((w, wi) => ({ w, rps: acc[wi]! / Math.max(1, preds.length) }));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Isotonic regression (PAV) for the calibration experiment: monotone map from
|
||||
// predicted probability to observed frequency, fit per outcome class.
|
||||
// ---------------------------------------------------------------------------
|
||||
interface IsoMap { x: number[]; y: number[] }
|
||||
|
||||
function fitIsotonic(pairs: { p: number; y: number }[]): IsoMap {
|
||||
const sorted = [...pairs].sort((a, b) => a.p - b.p);
|
||||
// pool-adjacent-violators on means, weighted by block size
|
||||
const blocks: { sum: number; n: number; minP: number; maxP: number }[] = [];
|
||||
for (const { p, y } of sorted) {
|
||||
blocks.push({ sum: y, n: 1, minP: p, maxP: p });
|
||||
while (blocks.length > 1) {
|
||||
const b = blocks[blocks.length - 1]!;
|
||||
const a = blocks[blocks.length - 2]!;
|
||||
if (a.sum / a.n <= b.sum / b.n) break;
|
||||
blocks.splice(blocks.length - 2, 2, { sum: a.sum + b.sum, n: a.n + b.n, minP: a.minP, maxP: b.maxP });
|
||||
}
|
||||
}
|
||||
const x: number[] = [0];
|
||||
const y: number[] = [0];
|
||||
for (const b of blocks) {
|
||||
x.push((b.minP + b.maxP) / 2);
|
||||
y.push(b.sum / b.n);
|
||||
}
|
||||
x.push(1); y.push(1);
|
||||
// enforce strictly increasing x for interpolation
|
||||
for (let i = 1; i < x.length; i++) if (x[i]! <= x[i - 1]!) x[i] = x[i - 1]! + 1e-9;
|
||||
return { x, y };
|
||||
}
|
||||
|
||||
function applyIso(map: IsoMap, p: number): number {
|
||||
const { x, y } = map;
|
||||
if (p <= x[0]!) return y[0]!;
|
||||
for (let i = 1; i < x.length; i++) {
|
||||
if (p <= x[i]!) {
|
||||
const f = (p - x[i - 1]!) / (x[i]! - x[i - 1]!);
|
||||
return y[i - 1]! + f * (y[i]! - y[i - 1]!);
|
||||
}
|
||||
}
|
||||
return y[y.length - 1]!;
|
||||
}
|
||||
|
||||
function calibrate(p: Probs, maps: { h: IsoMap; d: IsoMap; a: IsoMap }): Probs {
|
||||
const h = Math.max(1e-6, applyIso(maps.h, p.h));
|
||||
const d = Math.max(1e-6, applyIso(maps.d, p.d));
|
||||
const a = Math.max(1e-6, applyIso(maps.a, p.a));
|
||||
const z = h + d + a;
|
||||
return { h: h / z, d: d / z, a: a / z };
|
||||
}
|
||||
|
||||
function eceOf(preds: { p: Probs; o: Outcome }[]): number {
|
||||
const bins = Array.from({ length: 10 }, () => ({ sp: 0, sy: 0, n: 0 }));
|
||||
for (const { p, o } of preds) {
|
||||
for (const [v, hit] of [[p.h, o === 'h'], [p.d, o === 'd'], [p.a, o === 'a']] as [number, boolean][]) {
|
||||
const i = Math.min(9, Math.floor(v * 10));
|
||||
bins[i]!.sp += v; bins[i]!.sy += hit ? 1 : 0; bins[i]!.n++;
|
||||
}
|
||||
}
|
||||
const total = preds.length * 3;
|
||||
return bins.reduce((s, b) => s + (b.n ? (b.n / total) * Math.abs(b.sp / b.n - b.sy / b.n) : 0), 0);
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const t0 = Date.now();
|
||||
const rows = parse();
|
||||
@@ -366,6 +429,82 @@ function main(): void {
|
||||
const shippedTest = testEval(SHIPPED);
|
||||
console.log(`TEST 2024–26: champion rps ${champTest.toFixed(4)} | shipped rps ${shippedTest.toFixed(4)}`);
|
||||
|
||||
// ---- isotonic calibration experiment (leave-one-fold-out, shipped config) ----
|
||||
// Bar (pre-registered): mean LOFO RPS must not worsen by >0.0002 AND mean
|
||||
// LOFO ECE must improve by ≥10% relative — otherwise the layer stays out.
|
||||
console.log('isotonic calibration experiment (LOFO)…');
|
||||
const probsFor = (preds: Pred[]): { p: Probs; o: Outcome }[] => {
|
||||
const out: { p: Probs; o: Outcome }[] = [];
|
||||
for (const p of preds) {
|
||||
const r = rows[p.i]!;
|
||||
out.push({ p: pooledFor(p), o: outcomeOf(r.hs, r.as) });
|
||||
}
|
||||
return out;
|
||||
};
|
||||
const pooledFor = (p: Pred): Probs => {
|
||||
const el = eloLambdasFor(p, SHIPPED);
|
||||
const mE = scoreMatrix(el.lh, el.la, baseParamsForRow(p).rho);
|
||||
const mD = scoreMatrix(p.lDcH, p.lDcA, baseParamsForRow(p).rho);
|
||||
return probsOf(SHIPPED.w === 0 ? mD : SHIPPED.w === 1 ? mE : poolMatrices(mE, mD, SHIPPED.w));
|
||||
};
|
||||
// helpers bound to the shipped config + per-fold params
|
||||
const foldOfRow = (i: number): number => {
|
||||
const t = rows[i]!.t;
|
||||
return foldTs.findIndex((f) => t >= f.fromT && t < f.toT);
|
||||
};
|
||||
const baseParamsForRow = (p: Pred): ModelParams => baseParams[Math.max(0, foldOfRow(p.i))]!;
|
||||
const eloLambdasFor = (p: Pred, cfg: Cfg): { lh: number; la: number } => {
|
||||
const fast = baseWalk.fast.get(cfg.m)!;
|
||||
const eh = (1 - cfg.beta) * baseWalk.preH[p.i]! + cfg.beta * fast.h[p.i]!;
|
||||
const ea = (1 - cfg.beta) * baseWalk.preA[p.i]! + cfg.beta * fast.a[p.i]!;
|
||||
const params = baseParamsForRow(p);
|
||||
const { lambdaHome, lambdaAway } = lambdasFromElo(eh, ea, params, rows[p.i]!.neutral ? 0 : params.homeAdvElo);
|
||||
return { lh: lambdaHome, la: lambdaAway };
|
||||
};
|
||||
|
||||
const foldProbSets = foldPreds(SHIPPED.xi, SHIPPED.k).map(probsFor);
|
||||
let rawRps = 0, calRps = 0, rawEce = 0, calEce = 0;
|
||||
for (let held = 0; held < foldProbSets.length; held++) {
|
||||
const train = foldProbSets.flatMap((s, i) => (i === held ? [] : s));
|
||||
const maps = {
|
||||
h: fitIsotonic(train.map(({ p, o }) => ({ p: p.h, y: o === 'h' ? 1 : 0 }))),
|
||||
d: fitIsotonic(train.map(({ p, o }) => ({ p: p.d, y: o === 'd' ? 1 : 0 }))),
|
||||
a: fitIsotonic(train.map(({ p, o }) => ({ p: p.a, y: o === 'a' ? 1 : 0 }))),
|
||||
};
|
||||
const heldSet = foldProbSets[held]!;
|
||||
const calSet = heldSet.map(({ p, o }) => ({ p: calibrate(p, maps), o }));
|
||||
rawRps += heldSet.reduce((s, x) => s + rps(x.p, x.o), 0) / heldSet.length;
|
||||
calRps += calSet.reduce((s, x) => s + rps(x.p, x.o), 0) / calSet.length;
|
||||
rawEce += eceOf(heldSet);
|
||||
calEce += eceOf(calSet);
|
||||
}
|
||||
const nF = foldProbSets.length;
|
||||
rawRps /= nF; calRps /= nF; rawEce /= nF; calEce /= nF;
|
||||
const isoOk = calRps <= rawRps + 0.0002 && calEce <= rawEce * 0.9;
|
||||
console.log(` LOFO raw: rps ${rawRps.toFixed(4)} ece ${rawEce.toFixed(4)} | calibrated: rps ${calRps.toFixed(4)} ece ${calEce.toFixed(4)} → ${isoOk ? 'SHIP isotonic layer' : 'KEEP raw (bar not met)'}`);
|
||||
|
||||
// Final maps (fit on ALL folds) — committed via data/calibration-maps.json and
|
||||
// embedded into ratings.json by buildRatings.ts when the experiment ships.
|
||||
let finalMaps: { home: IsoMap; draw: IsoMap; away: IsoMap } | null = null;
|
||||
if (isoOk) {
|
||||
const all = foldProbSets.flat();
|
||||
finalMaps = {
|
||||
home: fitIsotonic(all.map(({ p, o }) => ({ p: p.h, y: o === 'h' ? 1 : 0 }))),
|
||||
draw: fitIsotonic(all.map(({ p, o }) => ({ p: p.d, y: o === 'd' ? 1 : 0 }))),
|
||||
away: fitIsotonic(all.map(({ p, o }) => ({ p: p.a, y: o === 'a' ? 1 : 0 }))),
|
||||
};
|
||||
const round = (m: IsoMap): IsoMap => ({ x: m.x.map((v) => +v.toFixed(5)), y: m.y.map((v) => +v.toFixed(5)) });
|
||||
finalMaps = { home: round(finalMaps.home), draw: round(finalMaps.draw), away: round(finalMaps.away) };
|
||||
writeFileSync(join(ROOT, 'data', 'calibration-maps.json'), JSON.stringify({
|
||||
generatedAt: new Date().toISOString(),
|
||||
fitOn: FOLDS,
|
||||
config: SHIPPED,
|
||||
lofo: { rawRps: +rawRps.toFixed(4), calRps: +calRps.toFixed(4), rawEce: +rawEce.toFixed(4), calEce: +calEce.toFixed(4) },
|
||||
maps: finalMaps,
|
||||
}, null, 2));
|
||||
console.log(' final maps → data/calibration-maps.json');
|
||||
}
|
||||
|
||||
mkdirSync(dirname(OUT), { recursive: true });
|
||||
writeFileSync(OUT, JSON.stringify({
|
||||
generatedAt: new Date().toISOString(),
|
||||
@@ -373,6 +512,11 @@ function main(): void {
|
||||
shipped: { ...SHIPPED, meanFoldRps: +shippedMean.toFixed(4), testRps: +shippedTest.toFixed(4) },
|
||||
champion: { ...best, rps: undefined, meanFoldRps: +best.rps.toFixed(4), testRps: +champTest.toFixed(4) },
|
||||
shipChange: ship,
|
||||
isotonic: {
|
||||
bar: 'LOFO RPS worsens ≤0.0002 AND LOFO ECE improves ≥10%',
|
||||
lofo: { rawRps: +rawRps.toFixed(4), calRps: +calRps.toFixed(4), rawEce: +rawEce.toFixed(4), calEce: +calEce.toFixed(4) },
|
||||
ship: isoOk,
|
||||
},
|
||||
runtimeSec: Math.round((Date.now() - t0) / 1000),
|
||||
}, null, 2));
|
||||
console.log(`report → data/cv-report.json (${Math.round((Date.now() - t0) / 1000)}s total)`);
|
||||
|
||||
@@ -114,3 +114,49 @@ describe('fast-Elo blend (v4 recency)', () => {
|
||||
expect(blendedRating(m, 'Strong')).toBeCloseTo(0.7 * 2050 + 0.3 * 1500, 9);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isotonic calibration (v5)', () => {
|
||||
const base: RatingsModel = {
|
||||
asOf: '2026-06-10',
|
||||
params: PARAMS,
|
||||
ratings: { Strong: 2050, Weak: 1650 },
|
||||
};
|
||||
// identity maps — calibration applied but should change nothing
|
||||
const identity = { x: [0, 1], y: [0, 1] };
|
||||
// a map that inflates home probability
|
||||
const homeBoost = { x: [0, 0.5, 1], y: [0, 0.6, 1] };
|
||||
|
||||
it('absent calibration ⇒ bit-identical (golden regression)', () => {
|
||||
const a = predictMatch('Strong', 'Weak', base);
|
||||
const b = predictMatch('Strong', 'Weak', { ...base });
|
||||
expect(b.probs).toEqual(a.probs);
|
||||
});
|
||||
|
||||
it('identity maps leave probabilities (numerically) unchanged', () => {
|
||||
const cal = { home: identity, draw: identity, away: identity };
|
||||
const a = predictMatch('Strong', 'Weak', base);
|
||||
const b = predictMatch('Strong', 'Weak', { ...base, calibration: cal });
|
||||
expect(b.probs.home).toBeCloseTo(a.probs.home, 9);
|
||||
expect(b.probs.draw).toBeCloseTo(a.probs.draw, 9);
|
||||
});
|
||||
|
||||
it('calibrated probabilities renormalize to 1 and shift as mapped', () => {
|
||||
const cal = { home: homeBoost, draw: identity, away: identity };
|
||||
const a = predictMatch('Strong', 'Weak', base);
|
||||
const b = predictMatch('Strong', 'Weak', { ...base, calibration: cal });
|
||||
expect(b.probs.home + b.probs.draw + b.probs.away).toBeCloseTo(1, 9);
|
||||
expect(b.probs.home).toBeGreaterThan(a.probs.home);
|
||||
});
|
||||
|
||||
it('scoreline shapes survive within each outcome region', () => {
|
||||
const cal = { home: homeBoost, draw: identity, away: identity };
|
||||
const a = predictMatch('Strong', 'Weak', base);
|
||||
const b = predictMatch('Strong', 'Weak', { ...base, calibration: cal });
|
||||
const ratio = (s: { home: number; away: number; p: number }[], h: number, a2: number) =>
|
||||
s.find((x) => x.home === h && x.away === a2)?.p ?? 0;
|
||||
// two home-win scorelines keep their relative odds
|
||||
const rawRatio = ratio(a.scorelines, 2, 0) / ratio(a.scorelines, 1, 0);
|
||||
const calRatio = ratio(b.scorelines, 2, 0) / ratio(b.scorelines, 1, 0);
|
||||
expect(calRatio).toBeCloseTo(rawRatio, 6);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -27,6 +27,47 @@ export interface RatingsModel {
|
||||
fastM?: number;
|
||||
/** Team-DC fit hyper-params, shipped so the server can refit in-tournament. */
|
||||
teamDcFit?: { xi: number; shrinkK: number; windowYears: number };
|
||||
/** Isotonic outcome calibration (fit walk-forward on the CV folds, validated
|
||||
* leave-one-fold-out). Applied by scaling the matrix's outcome regions, so
|
||||
* scorelines, simulations and displayed probabilities stay consistent. */
|
||||
calibration?: Calibration;
|
||||
}
|
||||
|
||||
export interface IsoMap {
|
||||
x: number[];
|
||||
y: number[];
|
||||
}
|
||||
|
||||
export interface Calibration {
|
||||
home: IsoMap;
|
||||
draw: IsoMap;
|
||||
away: IsoMap;
|
||||
}
|
||||
|
||||
function isoApply(map: IsoMap, p: number): number {
|
||||
const { x, y } = map;
|
||||
if (p <= x[0]!) return y[0]!;
|
||||
for (let i = 1; i < x.length; i++) {
|
||||
if (p <= x[i]!) {
|
||||
const f = (p - x[i - 1]!) / (x[i]! - x[i - 1]!);
|
||||
return y[i - 1]! + f * (y[i]! - y[i - 1]!);
|
||||
}
|
||||
}
|
||||
return y[y.length - 1]!;
|
||||
}
|
||||
|
||||
/** Scale the matrix's home/draw/away regions to the calibrated outcome masses.
|
||||
* Scoreline shapes inside each region are preserved. */
|
||||
function calibrateMatrix(m: number[][], cal: Calibration): number[][] {
|
||||
const p = outcomeProbs(m);
|
||||
const h = Math.max(1e-6, isoApply(cal.home, p.home));
|
||||
const d = Math.max(1e-6, isoApply(cal.draw, p.draw));
|
||||
const a = Math.max(1e-6, isoApply(cal.away, p.away));
|
||||
const z = h + d + a;
|
||||
const fh = p.home > 1e-12 ? h / z / p.home : 1;
|
||||
const fd = p.draw > 1e-12 ? d / z / p.draw : 1;
|
||||
const fa = p.away > 1e-12 ? a / z / p.away : 1;
|
||||
return m.map((row, i) => row.map((v, j) => v * (i > j ? fh : i === j ? fd : fa)));
|
||||
}
|
||||
|
||||
export const DEFAULT_ELO = 1500;
|
||||
@@ -72,7 +113,8 @@ export function matchMatrix(
|
||||
blendedRating(model, home), blendedRating(model, away), model.params, homeAdv,
|
||||
);
|
||||
const mElo = scoreMatrix(lambdaHome, lambdaAway, model.params.rho);
|
||||
if (!model.teamDc || model.ensembleW == null || model.ensembleW >= 1) return mElo;
|
||||
const finish = (m: number[][]): number[][] => (model.calibration ? calibrateMatrix(m, model.calibration) : m);
|
||||
if (!model.teamDc || model.ensembleW == null || model.ensembleW >= 1) return finish(mElo);
|
||||
|
||||
// Team-DC orientation: homeAdv > 0 → home side at home; < 0 → away side at
|
||||
// home (compute flipped, then swap); 0 → neutral.
|
||||
@@ -85,7 +127,7 @@ export function matchMatrix(
|
||||
dcHome = f.lambdaHome; dcAway = f.lambdaAway;
|
||||
}
|
||||
const mDc = scoreMatrix(dcHome, dcAway, model.params.rho);
|
||||
return poolMatrices(mElo, mDc, model.ensembleW);
|
||||
return finish(poolMatrices(mElo, mDc, model.ensembleW));
|
||||
}
|
||||
|
||||
/** Expected goals per side under a scoreline matrix. */
|
||||
|
||||
Reference in New Issue
Block a user