Isotonic calibration layer — validated leave-one-fold-out, shipped
The CV harness gained a calibration experiment: per-outcome isotonic maps (pool-adjacent-violators) fit walk-forward on the five folds and judged leave-one-fold-out against a pre-registered bar. It cleared it: LOFO ECE improves 16% (0.0146 → 0.0122) with RPS slightly better too (0.1724 → 0.1722). The final maps (fit on all folds) are committed as a research artifact, embedded into ratings.json by buildRatings when the shipped config matches, and applied in matchMatrix by scaling the score-matrix outcome regions — scorelines, Monte Carlo and displayed probabilities all stay mutually consistent. Without the field, output is bit-identical (golden tests). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -257,6 +257,69 @@ function sweepW(
|
||||
return W_STEPS.map((w, wi) => ({ w, rps: acc[wi]! / Math.max(1, preds.length) }));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Isotonic regression (PAV) for the calibration experiment: monotone map from
|
||||
// predicted probability to observed frequency, fit per outcome class.
|
||||
// ---------------------------------------------------------------------------
|
||||
interface IsoMap { x: number[]; y: number[] }
|
||||
|
||||
function fitIsotonic(pairs: { p: number; y: number }[]): IsoMap {
|
||||
const sorted = [...pairs].sort((a, b) => a.p - b.p);
|
||||
// pool-adjacent-violators on means, weighted by block size
|
||||
const blocks: { sum: number; n: number; minP: number; maxP: number }[] = [];
|
||||
for (const { p, y } of sorted) {
|
||||
blocks.push({ sum: y, n: 1, minP: p, maxP: p });
|
||||
while (blocks.length > 1) {
|
||||
const b = blocks[blocks.length - 1]!;
|
||||
const a = blocks[blocks.length - 2]!;
|
||||
if (a.sum / a.n <= b.sum / b.n) break;
|
||||
blocks.splice(blocks.length - 2, 2, { sum: a.sum + b.sum, n: a.n + b.n, minP: a.minP, maxP: b.maxP });
|
||||
}
|
||||
}
|
||||
const x: number[] = [0];
|
||||
const y: number[] = [0];
|
||||
for (const b of blocks) {
|
||||
x.push((b.minP + b.maxP) / 2);
|
||||
y.push(b.sum / b.n);
|
||||
}
|
||||
x.push(1); y.push(1);
|
||||
// enforce strictly increasing x for interpolation
|
||||
for (let i = 1; i < x.length; i++) if (x[i]! <= x[i - 1]!) x[i] = x[i - 1]! + 1e-9;
|
||||
return { x, y };
|
||||
}
|
||||
|
||||
function applyIso(map: IsoMap, p: number): number {
|
||||
const { x, y } = map;
|
||||
if (p <= x[0]!) return y[0]!;
|
||||
for (let i = 1; i < x.length; i++) {
|
||||
if (p <= x[i]!) {
|
||||
const f = (p - x[i - 1]!) / (x[i]! - x[i - 1]!);
|
||||
return y[i - 1]! + f * (y[i]! - y[i - 1]!);
|
||||
}
|
||||
}
|
||||
return y[y.length - 1]!;
|
||||
}
|
||||
|
||||
function calibrate(p: Probs, maps: { h: IsoMap; d: IsoMap; a: IsoMap }): Probs {
|
||||
const h = Math.max(1e-6, applyIso(maps.h, p.h));
|
||||
const d = Math.max(1e-6, applyIso(maps.d, p.d));
|
||||
const a = Math.max(1e-6, applyIso(maps.a, p.a));
|
||||
const z = h + d + a;
|
||||
return { h: h / z, d: d / z, a: a / z };
|
||||
}
|
||||
|
||||
function eceOf(preds: { p: Probs; o: Outcome }[]): number {
|
||||
const bins = Array.from({ length: 10 }, () => ({ sp: 0, sy: 0, n: 0 }));
|
||||
for (const { p, o } of preds) {
|
||||
for (const [v, hit] of [[p.h, o === 'h'], [p.d, o === 'd'], [p.a, o === 'a']] as [number, boolean][]) {
|
||||
const i = Math.min(9, Math.floor(v * 10));
|
||||
bins[i]!.sp += v; bins[i]!.sy += hit ? 1 : 0; bins[i]!.n++;
|
||||
}
|
||||
}
|
||||
const total = preds.length * 3;
|
||||
return bins.reduce((s, b) => s + (b.n ? (b.n / total) * Math.abs(b.sp / b.n - b.sy / b.n) : 0), 0);
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const t0 = Date.now();
|
||||
const rows = parse();
|
||||
@@ -366,6 +429,82 @@ function main(): void {
|
||||
const shippedTest = testEval(SHIPPED);
|
||||
console.log(`TEST 2024–26: champion rps ${champTest.toFixed(4)} | shipped rps ${shippedTest.toFixed(4)}`);
|
||||
|
||||
// ---- isotonic calibration experiment (leave-one-fold-out, shipped config) ----
|
||||
// Bar (pre-registered): mean LOFO RPS must not worsen by >0.0002 AND mean
|
||||
// LOFO ECE must improve by ≥10% relative — otherwise the layer stays out.
|
||||
console.log('isotonic calibration experiment (LOFO)…');
|
||||
const probsFor = (preds: Pred[]): { p: Probs; o: Outcome }[] => {
|
||||
const out: { p: Probs; o: Outcome }[] = [];
|
||||
for (const p of preds) {
|
||||
const r = rows[p.i]!;
|
||||
out.push({ p: pooledFor(p), o: outcomeOf(r.hs, r.as) });
|
||||
}
|
||||
return out;
|
||||
};
|
||||
const pooledFor = (p: Pred): Probs => {
|
||||
const el = eloLambdasFor(p, SHIPPED);
|
||||
const mE = scoreMatrix(el.lh, el.la, baseParamsForRow(p).rho);
|
||||
const mD = scoreMatrix(p.lDcH, p.lDcA, baseParamsForRow(p).rho);
|
||||
return probsOf(SHIPPED.w === 0 ? mD : SHIPPED.w === 1 ? mE : poolMatrices(mE, mD, SHIPPED.w));
|
||||
};
|
||||
// helpers bound to the shipped config + per-fold params
|
||||
const foldOfRow = (i: number): number => {
|
||||
const t = rows[i]!.t;
|
||||
return foldTs.findIndex((f) => t >= f.fromT && t < f.toT);
|
||||
};
|
||||
const baseParamsForRow = (p: Pred): ModelParams => baseParams[Math.max(0, foldOfRow(p.i))]!;
|
||||
const eloLambdasFor = (p: Pred, cfg: Cfg): { lh: number; la: number } => {
|
||||
const fast = baseWalk.fast.get(cfg.m)!;
|
||||
const eh = (1 - cfg.beta) * baseWalk.preH[p.i]! + cfg.beta * fast.h[p.i]!;
|
||||
const ea = (1 - cfg.beta) * baseWalk.preA[p.i]! + cfg.beta * fast.a[p.i]!;
|
||||
const params = baseParamsForRow(p);
|
||||
const { lambdaHome, lambdaAway } = lambdasFromElo(eh, ea, params, rows[p.i]!.neutral ? 0 : params.homeAdvElo);
|
||||
return { lh: lambdaHome, la: lambdaAway };
|
||||
};
|
||||
|
||||
const foldProbSets = foldPreds(SHIPPED.xi, SHIPPED.k).map(probsFor);
|
||||
let rawRps = 0, calRps = 0, rawEce = 0, calEce = 0;
|
||||
for (let held = 0; held < foldProbSets.length; held++) {
|
||||
const train = foldProbSets.flatMap((s, i) => (i === held ? [] : s));
|
||||
const maps = {
|
||||
h: fitIsotonic(train.map(({ p, o }) => ({ p: p.h, y: o === 'h' ? 1 : 0 }))),
|
||||
d: fitIsotonic(train.map(({ p, o }) => ({ p: p.d, y: o === 'd' ? 1 : 0 }))),
|
||||
a: fitIsotonic(train.map(({ p, o }) => ({ p: p.a, y: o === 'a' ? 1 : 0 }))),
|
||||
};
|
||||
const heldSet = foldProbSets[held]!;
|
||||
const calSet = heldSet.map(({ p, o }) => ({ p: calibrate(p, maps), o }));
|
||||
rawRps += heldSet.reduce((s, x) => s + rps(x.p, x.o), 0) / heldSet.length;
|
||||
calRps += calSet.reduce((s, x) => s + rps(x.p, x.o), 0) / calSet.length;
|
||||
rawEce += eceOf(heldSet);
|
||||
calEce += eceOf(calSet);
|
||||
}
|
||||
const nF = foldProbSets.length;
|
||||
rawRps /= nF; calRps /= nF; rawEce /= nF; calEce /= nF;
|
||||
const isoOk = calRps <= rawRps + 0.0002 && calEce <= rawEce * 0.9;
|
||||
console.log(` LOFO raw: rps ${rawRps.toFixed(4)} ece ${rawEce.toFixed(4)} | calibrated: rps ${calRps.toFixed(4)} ece ${calEce.toFixed(4)} → ${isoOk ? 'SHIP isotonic layer' : 'KEEP raw (bar not met)'}`);
|
||||
|
||||
// Final maps (fit on ALL folds) — committed via data/calibration-maps.json and
|
||||
// embedded into ratings.json by buildRatings.ts when the experiment ships.
|
||||
let finalMaps: { home: IsoMap; draw: IsoMap; away: IsoMap } | null = null;
|
||||
if (isoOk) {
|
||||
const all = foldProbSets.flat();
|
||||
finalMaps = {
|
||||
home: fitIsotonic(all.map(({ p, o }) => ({ p: p.h, y: o === 'h' ? 1 : 0 }))),
|
||||
draw: fitIsotonic(all.map(({ p, o }) => ({ p: p.d, y: o === 'd' ? 1 : 0 }))),
|
||||
away: fitIsotonic(all.map(({ p, o }) => ({ p: p.a, y: o === 'a' ? 1 : 0 }))),
|
||||
};
|
||||
const round = (m: IsoMap): IsoMap => ({ x: m.x.map((v) => +v.toFixed(5)), y: m.y.map((v) => +v.toFixed(5)) });
|
||||
finalMaps = { home: round(finalMaps.home), draw: round(finalMaps.draw), away: round(finalMaps.away) };
|
||||
writeFileSync(join(ROOT, 'data', 'calibration-maps.json'), JSON.stringify({
|
||||
generatedAt: new Date().toISOString(),
|
||||
fitOn: FOLDS,
|
||||
config: SHIPPED,
|
||||
lofo: { rawRps: +rawRps.toFixed(4), calRps: +calRps.toFixed(4), rawEce: +rawEce.toFixed(4), calEce: +calEce.toFixed(4) },
|
||||
maps: finalMaps,
|
||||
}, null, 2));
|
||||
console.log(' final maps → data/calibration-maps.json');
|
||||
}
|
||||
|
||||
mkdirSync(dirname(OUT), { recursive: true });
|
||||
writeFileSync(OUT, JSON.stringify({
|
||||
generatedAt: new Date().toISOString(),
|
||||
@@ -373,6 +512,11 @@ function main(): void {
|
||||
shipped: { ...SHIPPED, meanFoldRps: +shippedMean.toFixed(4), testRps: +shippedTest.toFixed(4) },
|
||||
champion: { ...best, rps: undefined, meanFoldRps: +best.rps.toFixed(4), testRps: +champTest.toFixed(4) },
|
||||
shipChange: ship,
|
||||
isotonic: {
|
||||
bar: 'LOFO RPS worsens ≤0.0002 AND LOFO ECE improves ≥10%',
|
||||
lofo: { rawRps: +rawRps.toFixed(4), calRps: +calRps.toFixed(4), rawEce: +rawEce.toFixed(4), calEce: +calEce.toFixed(4) },
|
||||
ship: isoOk,
|
||||
},
|
||||
runtimeSec: Math.round((Date.now() - t0) / 1000),
|
||||
}, null, 2));
|
||||
console.log(`report → data/cv-report.json (${Math.round((Date.now() - t0) / 1000)}s total)`);
|
||||
|
||||
Reference in New Issue
Block a user