Isotonic calibration layer — validated leave-one-fold-out, shipped

The CV harness gained a calibration experiment: per-outcome isotonic
maps (pool-adjacent-violators) fit walk-forward on the five folds and
judged leave-one-fold-out against a pre-registered bar. It cleared it:
LOFO ECE improves 16% (0.0146 → 0.0122) with RPS slightly better too
(0.1724 → 0.1722). The final maps (fit on all folds) are committed as
a research artifact, embedded into ratings.json by buildRatings when
the shipped config matches, and applied in matchMatrix by scaling the
score-matrix outcome regions — scorelines, Monte Carlo and displayed
probabilities all stay mutually consistent. Without the field, output
is bit-identical (golden tests).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

data/cv-report.json

@@ -1,5 +1,5 @@
 {
-  "generatedAt": "2026-06-11T22:41:08.860Z",
+  "generatedAt": "2026-06-11T23:06:22.047Z",
   "protocol": {
     "folds": [
       {
@@ -56,5 +56,15 @@
     "testRps": 0.1647
   },
   "shipChange": false,
-  "runtimeSec": 72
+  "isotonic": {
+    "bar": "LOFO RPS worsens ≤0.0002 AND LOFO ECE improves ≥10%",
+    "lofo": {
+      "rawRps": 0.1724,
+      "calRps": 0.1722,
+      "rawEce": 0.0146,
+      "calEce": 0.0122
+    },
+    "ship": true
+  },
+  "runtimeSec": 73
 }

scripts/buildRatings.ts

@@ -3,7 +3,7 @@
 // World-Football-Elo rating per team. Then calibrates the goals model
 // (goals-per-Elo slope, mean goals) on the modern era and MLE-fits Dixon-Coles
 // rho. The runtime re-uses these ratings + params to predict and simulate.
-import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
 import { dirname, join } from 'node:path';
 import { canonicalTeam, ALL_TEAMS } from '../src/lib/teams';
@@ -160,6 +160,23 @@ function main(): void {
   teamDc.mu = Math.round(teamDc.mu * 1e4) / 1e4;
   teamDc.homeMult = Math.round(teamDc.homeMult * 1e4) / 1e4;
 
+  // Isotonic calibration maps — research artifact from scripts/cvSearch.ts
+  // (committed; only valid while the shipped config matches its fit config).
+  let calibration: unknown = null;
+  const calFile = join(ROOT, 'data', 'calibration-maps.json');
+  if (existsSync(calFile)) {
+    const cal = JSON.parse(readFileSync(calFile, 'utf8')) as {
+      config: { xi: number; k: number; w: number; m: number; beta: number };
+      maps: unknown;
+    };
+    const c = cal.config;
+    if (c.xi === TEAMDC_XI && c.k === TEAMDC_SHRINK_K && c.w === ENSEMBLE_W && c.m === FAST_M && c.beta === FAST_BETA) {
+      calibration = cal.maps;
+    } else {
+      console.warn('calibration-maps.json was fit for a different config — skipping (rerun scripts/cvSearch.ts)');
+    }
+  }
+
   const out = {
     generatedAt: new Date().toISOString(),
     asOf: lastDate,
@@ -178,6 +195,7 @@ function main(): void {
     fastBeta: FAST_BETA,
     fastM: FAST_M,
     teamDcFit: { xi: TEAMDC_XI, shrinkK: TEAMDC_SHRINK_K, windowYears: FIT_WINDOW_YEARS },
+    ...(calibration ? { calibration } : {}),
     ratings,
   };
 

scripts/cvSearch.ts

@@ -257,6 +257,69 @@ function sweepW(
   return W_STEPS.map((w, wi) => ({ w, rps: acc[wi]! / Math.max(1, preds.length) }));
 }
 
+// ---------------------------------------------------------------------------
+// Isotonic regression (PAV) for the calibration experiment: monotone map from
+// predicted probability to observed frequency, fit per outcome class.
+// ---------------------------------------------------------------------------
+interface IsoMap { x: number[]; y: number[] }
+
+function fitIsotonic(pairs: { p: number; y: number }[]): IsoMap {
+  const sorted = [...pairs].sort((a, b) => a.p - b.p);
+  // pool-adjacent-violators on means, weighted by block size
+  const blocks: { sum: number; n: number; minP: number; maxP: number }[] = [];
+  for (const { p, y } of sorted) {
+    blocks.push({ sum: y, n: 1, minP: p, maxP: p });
+    while (blocks.length > 1) {
+      const b = blocks[blocks.length - 1]!;
+      const a = blocks[blocks.length - 2]!;
+      if (a.sum / a.n <= b.sum / b.n) break;
+      blocks.splice(blocks.length - 2, 2, { sum: a.sum + b.sum, n: a.n + b.n, minP: a.minP, maxP: b.maxP });
+    }
+  }
+  const x: number[] = [0];
+  const y: number[] = [0];
+  for (const b of blocks) {
+    x.push((b.minP + b.maxP) / 2);
+    y.push(b.sum / b.n);
+  }
+  x.push(1); y.push(1);
+  // enforce strictly increasing x for interpolation
+  for (let i = 1; i < x.length; i++) if (x[i]! <= x[i - 1]!) x[i] = x[i - 1]! + 1e-9;
+  return { x, y };
+}
+
+function applyIso(map: IsoMap, p: number): number {
+  const { x, y } = map;
+  if (p <= x[0]!) return y[0]!;
+  for (let i = 1; i < x.length; i++) {
+    if (p <= x[i]!) {
+      const f = (p - x[i - 1]!) / (x[i]! - x[i - 1]!);
+      return y[i - 1]! + f * (y[i]! - y[i - 1]!);
+    }
+  }
+  return y[y.length - 1]!;
+}
+
+function calibrate(p: Probs, maps: { h: IsoMap; d: IsoMap; a: IsoMap }): Probs {
+  const h = Math.max(1e-6, applyIso(maps.h, p.h));
+  const d = Math.max(1e-6, applyIso(maps.d, p.d));
+  const a = Math.max(1e-6, applyIso(maps.a, p.a));
+  const z = h + d + a;
+  return { h: h / z, d: d / z, a: a / z };
+}
+
+function eceOf(preds: { p: Probs; o: Outcome }[]): number {
+  const bins = Array.from({ length: 10 }, () => ({ sp: 0, sy: 0, n: 0 }));
+  for (const { p, o } of preds) {
+    for (const [v, hit] of [[p.h, o === 'h'], [p.d, o === 'd'], [p.a, o === 'a']] as [number, boolean][]) {
+      const i = Math.min(9, Math.floor(v * 10));
+      bins[i]!.sp += v; bins[i]!.sy += hit ? 1 : 0; bins[i]!.n++;
+    }
+  }
+  const total = preds.length * 3;
+  return bins.reduce((s, b) => s + (b.n ? (b.n / total) * Math.abs(b.sp / b.n - b.sy / b.n) : 0), 0);
+}
+
 function main(): void {
   const t0 = Date.now();
   const rows = parse();
@@ -366,6 +429,82 @@ function main(): void {
   const shippedTest = testEval(SHIPPED);
   console.log(`TEST 2024–26: champion rps ${champTest.toFixed(4)} | shipped rps ${shippedTest.toFixed(4)}`);
 
+  // ---- isotonic calibration experiment (leave-one-fold-out, shipped config) ----
+  // Bar (pre-registered): mean LOFO RPS must not worsen by >0.0002 AND mean
+  // LOFO ECE must improve by ≥10% relative — otherwise the layer stays out.
+  console.log('isotonic calibration experiment (LOFO)…');
+  const probsFor = (preds: Pred[]): { p: Probs; o: Outcome }[] => {
+    const out: { p: Probs; o: Outcome }[] = [];
+    for (const p of preds) {
+      const r = rows[p.i]!;
+      out.push({ p: pooledFor(p), o: outcomeOf(r.hs, r.as) });
+    }
+    return out;
+  };
+  const pooledFor = (p: Pred): Probs => {
+    const el = eloLambdasFor(p, SHIPPED);
+    const mE = scoreMatrix(el.lh, el.la, baseParamsForRow(p).rho);
+    const mD = scoreMatrix(p.lDcH, p.lDcA, baseParamsForRow(p).rho);
+    return probsOf(SHIPPED.w === 0 ? mD : SHIPPED.w === 1 ? mE : poolMatrices(mE, mD, SHIPPED.w));
+  };
+  // helpers bound to the shipped config + per-fold params
+  const foldOfRow = (i: number): number => {
+    const t = rows[i]!.t;
+    return foldTs.findIndex((f) => t >= f.fromT && t < f.toT);
+  };
+  const baseParamsForRow = (p: Pred): ModelParams => baseParams[Math.max(0, foldOfRow(p.i))]!;
+  const eloLambdasFor = (p: Pred, cfg: Cfg): { lh: number; la: number } => {
+    const fast = baseWalk.fast.get(cfg.m)!;
+    const eh = (1 - cfg.beta) * baseWalk.preH[p.i]! + cfg.beta * fast.h[p.i]!;
+    const ea = (1 - cfg.beta) * baseWalk.preA[p.i]! + cfg.beta * fast.a[p.i]!;
+    const params = baseParamsForRow(p);
+    const { lambdaHome, lambdaAway } = lambdasFromElo(eh, ea, params, rows[p.i]!.neutral ? 0 : params.homeAdvElo);
+    return { lh: lambdaHome, la: lambdaAway };
+  };
+
+  const foldProbSets = foldPreds(SHIPPED.xi, SHIPPED.k).map(probsFor);
+  let rawRps = 0, calRps = 0, rawEce = 0, calEce = 0;
+  for (let held = 0; held < foldProbSets.length; held++) {
+    const train = foldProbSets.flatMap((s, i) => (i === held ? [] : s));
+    const maps = {
+      h: fitIsotonic(train.map(({ p, o }) => ({ p: p.h, y: o === 'h' ? 1 : 0 }))),
+      d: fitIsotonic(train.map(({ p, o }) => ({ p: p.d, y: o === 'd' ? 1 : 0 }))),
+      a: fitIsotonic(train.map(({ p, o }) => ({ p: p.a, y: o === 'a' ? 1 : 0 }))),
+    };
+    const heldSet = foldProbSets[held]!;
+    const calSet = heldSet.map(({ p, o }) => ({ p: calibrate(p, maps), o }));
+    rawRps += heldSet.reduce((s, x) => s + rps(x.p, x.o), 0) / heldSet.length;
+    calRps += calSet.reduce((s, x) => s + rps(x.p, x.o), 0) / calSet.length;
+    rawEce += eceOf(heldSet);
+    calEce += eceOf(calSet);
+  }
+  const nF = foldProbSets.length;
+  rawRps /= nF; calRps /= nF; rawEce /= nF; calEce /= nF;
+  const isoOk = calRps <= rawRps + 0.0002 && calEce <= rawEce * 0.9;
+  console.log(`  LOFO raw: rps ${rawRps.toFixed(4)} ece ${rawEce.toFixed(4)} | calibrated: rps ${calRps.toFixed(4)} ece ${calEce.toFixed(4)} → ${isoOk ? 'SHIP isotonic layer' : 'KEEP raw (bar not met)'}`);
+
+  // Final maps (fit on ALL folds) — committed via data/calibration-maps.json and
+  // embedded into ratings.json by buildRatings.ts when the experiment ships.
+  let finalMaps: { home: IsoMap; draw: IsoMap; away: IsoMap } | null = null;
+  if (isoOk) {
+    const all = foldProbSets.flat();
+    finalMaps = {
+      home: fitIsotonic(all.map(({ p, o }) => ({ p: p.h, y: o === 'h' ? 1 : 0 }))),
+      draw: fitIsotonic(all.map(({ p, o }) => ({ p: p.d, y: o === 'd' ? 1 : 0 }))),
+      away: fitIsotonic(all.map(({ p, o }) => ({ p: p.a, y: o === 'a' ? 1 : 0 }))),
+    };
+    const round = (m: IsoMap): IsoMap => ({ x: m.x.map((v) => +v.toFixed(5)), y: m.y.map((v) => +v.toFixed(5)) });
+    finalMaps = { home: round(finalMaps.home), draw: round(finalMaps.draw), away: round(finalMaps.away) };
+    writeFileSync(join(ROOT, 'data', 'calibration-maps.json'), JSON.stringify({
+      generatedAt: new Date().toISOString(),
+      fitOn: FOLDS,
+      config: SHIPPED,
+      lofo: { rawRps: +rawRps.toFixed(4), calRps: +calRps.toFixed(4), rawEce: +rawEce.toFixed(4), calEce: +calEce.toFixed(4) },
+      maps: finalMaps,
+    }, null, 2));
+    console.log('  final maps → data/calibration-maps.json');
+  }
+
   mkdirSync(dirname(OUT), { recursive: true });
   writeFileSync(OUT, JSON.stringify({
     generatedAt: new Date().toISOString(),
@@ -373,6 +512,11 @@ function main(): void {
     shipped: { ...SHIPPED, meanFoldRps: +shippedMean.toFixed(4), testRps: +shippedTest.toFixed(4) },
     champion: { ...best, rps: undefined, meanFoldRps: +best.rps.toFixed(4), testRps: +champTest.toFixed(4) },
     shipChange: ship,
+    isotonic: {
+      bar: 'LOFO RPS worsens ≤0.0002 AND LOFO ECE improves ≥10%',
+      lofo: { rawRps: +rawRps.toFixed(4), calRps: +calRps.toFixed(4), rawEce: +rawEce.toFixed(4), calEce: +calEce.toFixed(4) },
+      ship: isoOk,
+    },
     runtimeSec: Math.round((Date.now() - t0) / 1000),
   }, null, 2));
   console.log(`report → data/cv-report.json (${Math.round((Date.now() - t0) / 1000)}s total)`);

src/lib/model/model.test.ts

@@ -114,3 +114,49 @@ describe('fast-Elo blend (v4 recency)', () => {
     expect(blendedRating(m, 'Strong')).toBeCloseTo(0.7 * 2050 + 0.3 * 1500, 9);
   });
 });
+
+describe('isotonic calibration (v5)', () => {
+  const base: RatingsModel = {
+    asOf: '2026-06-10',
+    params: PARAMS,
+    ratings: { Strong: 2050, Weak: 1650 },
+  };
+  // identity maps — calibration applied but should change nothing
+  const identity = { x: [0, 1], y: [0, 1] };
+  // a map that inflates home probability
+  const homeBoost = { x: [0, 0.5, 1], y: [0, 0.6, 1] };
+
+  it('absent calibration ⇒ bit-identical (golden regression)', () => {
+    const a = predictMatch('Strong', 'Weak', base);
+    const b = predictMatch('Strong', 'Weak', { ...base });
+    expect(b.probs).toEqual(a.probs);
+  });
+
+  it('identity maps leave probabilities (numerically) unchanged', () => {
+    const cal = { home: identity, draw: identity, away: identity };
+    const a = predictMatch('Strong', 'Weak', base);
+    const b = predictMatch('Strong', 'Weak', { ...base, calibration: cal });
+    expect(b.probs.home).toBeCloseTo(a.probs.home, 9);
+    expect(b.probs.draw).toBeCloseTo(a.probs.draw, 9);
+  });
+
+  it('calibrated probabilities renormalize to 1 and shift as mapped', () => {
+    const cal = { home: homeBoost, draw: identity, away: identity };
+    const a = predictMatch('Strong', 'Weak', base);
+    const b = predictMatch('Strong', 'Weak', { ...base, calibration: cal });
+    expect(b.probs.home + b.probs.draw + b.probs.away).toBeCloseTo(1, 9);
+    expect(b.probs.home).toBeGreaterThan(a.probs.home);
+  });
+
+  it('scoreline shapes survive within each outcome region', () => {
+    const cal = { home: homeBoost, draw: identity, away: identity };
+    const a = predictMatch('Strong', 'Weak', base);
+    const b = predictMatch('Strong', 'Weak', { ...base, calibration: cal });
+    const ratio = (s: { home: number; away: number; p: number }[], h: number, a2: number) =>
+      s.find((x) => x.home === h && x.away === a2)?.p ?? 0;
+    // two home-win scorelines keep their relative odds
+    const rawRatio = ratio(a.scorelines, 2, 0) / ratio(a.scorelines, 1, 0);
+    const calRatio = ratio(b.scorelines, 2, 0) / ratio(b.scorelines, 1, 0);
+    expect(calRatio).toBeCloseTo(rawRatio, 6);
+  });
+});

src/lib/model/predict.ts

@@ -27,6 +27,47 @@ export interface RatingsModel {
   fastM?: number;
   /** Team-DC fit hyper-params, shipped so the server can refit in-tournament. */
   teamDcFit?: { xi: number; shrinkK: number; windowYears: number };
+  /** Isotonic outcome calibration (fit walk-forward on the CV folds, validated
+   *  leave-one-fold-out). Applied by scaling the matrix's outcome regions, so
+   *  scorelines, simulations and displayed probabilities stay consistent. */
+  calibration?: Calibration;
+}
+
+export interface IsoMap {
+  x: number[];
+  y: number[];
+}
+
+export interface Calibration {
+  home: IsoMap;
+  draw: IsoMap;
+  away: IsoMap;
+}
+
+function isoApply(map: IsoMap, p: number): number {
+  const { x, y } = map;
+  if (p <= x[0]!) return y[0]!;
+  for (let i = 1; i < x.length; i++) {
+    if (p <= x[i]!) {
+      const f = (p - x[i - 1]!) / (x[i]! - x[i - 1]!);
+      return y[i - 1]! + f * (y[i]! - y[i - 1]!);
+    }
+  }
+  return y[y.length - 1]!;
+}
+
+/** Scale the matrix's home/draw/away regions to the calibrated outcome masses.
+ *  Scoreline shapes inside each region are preserved. */
+function calibrateMatrix(m: number[][], cal: Calibration): number[][] {
+  const p = outcomeProbs(m);
+  const h = Math.max(1e-6, isoApply(cal.home, p.home));
+  const d = Math.max(1e-6, isoApply(cal.draw, p.draw));
+  const a = Math.max(1e-6, isoApply(cal.away, p.away));
+  const z = h + d + a;
+  const fh = p.home > 1e-12 ? h / z / p.home : 1;
+  const fd = p.draw > 1e-12 ? d / z / p.draw : 1;
+  const fa = p.away > 1e-12 ? a / z / p.away : 1;
+  return m.map((row, i) => row.map((v, j) => v * (i > j ? fh : i === j ? fd : fa)));
 }
 
 export const DEFAULT_ELO = 1500;
@@ -72,7 +113,8 @@ export function matchMatrix(
     blendedRating(model, home), blendedRating(model, away), model.params, homeAdv,
   );
   const mElo = scoreMatrix(lambdaHome, lambdaAway, model.params.rho);
-  if (!model.teamDc || model.ensembleW == null || model.ensembleW >= 1) return mElo;
+  const finish = (m: number[][]): number[][] => (model.calibration ? calibrateMatrix(m, model.calibration) : m);
+  if (!model.teamDc || model.ensembleW == null || model.ensembleW >= 1) return finish(mElo);
 
   // Team-DC orientation: homeAdv > 0 → home side at home; < 0 → away side at
   // home (compute flipped, then swap); 0 → neutral.
@@ -85,7 +127,7 @@ export function matchMatrix(
     dcHome = f.lambdaHome; dcAway = f.lambdaAway;
   }
   const mDc = scoreMatrix(dcHome, dcAway, model.params.rho);
-  return poolMatrices(mElo, mDc, model.ensembleW);
+  return finish(poolMatrices(mElo, mDc, model.ensembleW));
 }
 
 /** Expected goals per side under a scoreline matrix. */