NilsBriggen/cup26
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Quick backtest mode for image builds (identical published numbers)

Browse Source 
BACKTEST_QUICK=1 skips the hyperparameter grid and scores only the
shipped + previous configs — same test metrics, a fraction of the
build time on the 1-CPU VPS. The full grid search stays a local
activity. Verified byte-level: quick output matches the grid run.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Nils
2026-06-12 00:33:42 +02:00
parent dc3c375081
commit 716dab2f23
2 changed files with 53 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/buildBacktest.ts
+49 -30
View File
@@ -40,6 +40,12 @@ const FORM_WINDOW = 10; // last N non-friendly matches feeding the form signal
const MIN_VAL_GAIN = 0.0005; // candidate must beat the previous config by this much
/** the previously shipped v3 config — the bar every candidate has to clear */
const PREV = { xi: 0.5, k: 5, w: 0.75, m: 2, beta: 0, gamma: 0 };
/** the v4 winner (from the full grid run) — keep in sync with buildRatings.ts */
const SHIPPED = { xi: 0.25, k: 3, w: 0.45, m: 3, beta: 0.3, gamma: 0 };
/** BACKTEST_QUICK=1 (Docker builds): skip the grid search and just score the
* shipped + previous configs — the published test numbers are identical, the
* hyperparam hunt only happens locally. */
const QUICK = process.env.BACKTEST_QUICK === '1';
interface Row { date: string; home: string; away: string; hs: number; as: number; tournament: string; neutral: boolean; t: number }
type Probs = { h: number; d: number; a: number };
@@ -241,42 +247,55 @@ function main(): void {
return W_STEPS.map((w, wi) => ({ w, rps: acc[wi]! / preds.length }));
};
// ---------- validation stage 1: tune ξ, k, w (β=0, γ=0) ----------
console.log('validation stage 1 — ξ/k/w grid (β=0, γ=0)…');
const valWalks = new Map<string, Pred[]>();
let best = { xi: PREV.xi, k: PREV.k, w: PREV.w, m: PREV.m, beta: 0, gamma: 0, rps: Infinity };
let prevValRps = Infinity; // validation RPS of the previously shipped config
for (const xi of XI_GRID) {
for (const k of K_GRID) {
const preds = walkTeamDc(xi, k, tValFrom, tTestFrom);
valWalks.set(`${xi}|${k}`, preds);
for (const { w, rps: r } of sweepW(preds, 0, PREV.m, 0)) {
if (xi === PREV.xi && k === PREV.k && w === PREV.w) prevValRps = r;
if (r < best.rps) best = { xi, k, w, m: PREV.m, beta: 0, gamma: 0, rps: r };
}
console.log(` ξ=${xi} k=${k} done (best so far: ξ=${best.xi} k=${best.k} w=${best.w} rps=${best.rps.toFixed(4)})`);
}
}
// ---------- validation stage 2: fast-Elo blend (m, β) at the stage-1 ξ/k ----------
console.log('validation stage 2 — fast-Elo blend (m, β)…');
const stagePreds = valWalks.get(`${best.xi}|${best.k}`)!;
for (const m of M_GRID) {
for (const beta of BETA_GRID) {
if (beta === 0) continue; // control already covered by stage 1
for (const { w, rps: r } of sweepW(stagePreds, beta, m, 0)) {
if (r < best.rps) best = { ...best, m, beta, w, rps: r };
if (QUICK) {
// ---------- quick mode: score only shipped + previous on validation ----------
console.log('quick mode — scoring shipped + previous configs on validation…');
const predsShipped = walkTeamDc(SHIPPED.xi, SHIPPED.k, tValFrom, tTestFrom);
const predsPrev = SHIPPED.xi === PREV.xi && SHIPPED.k === PREV.k
? predsShipped
: walkTeamDc(PREV.xi, PREV.k, tValFrom, tTestFrom);
const shippedRps = sweepW(predsShipped, SHIPPED.beta, SHIPPED.m, SHIPPED.gamma).find((x) => x.w === SHIPPED.w)!.rps;
prevValRps = sweepW(predsPrev, PREV.beta, PREV.m, PREV.gamma).find((x) => x.w === PREV.w)!.rps;
best = { ...SHIPPED, rps: shippedRps };
} else {
// ---------- validation stage 1: tune ξ, k, w (β=0, γ=0) ----------
console.log('validation stage 1 — ξ/k/w grid (β=0, γ=0)…');
const valWalks = new Map<string, Pred[]>();
for (const xi of XI_GRID) {
for (const k of K_GRID) {
const preds = walkTeamDc(xi, k, tValFrom, tTestFrom);
valWalks.set(`${xi}|${k}`, preds);
for (const { w, rps: r } of sweepW(preds, 0, PREV.m, 0)) {
if (xi === PREV.xi && k === PREV.k && w === PREV.w) prevValRps = r;
if (r < best.rps) best = { xi, k, w, m: PREV.m, beta: 0, gamma: 0, rps: r };
}
console.log(` ξ=${xi} k=${k} done (best so far: ξ=${best.xi} k=${best.k} w=${best.w} rps=${best.rps.toFixed(4)})`);
}
}
console.log(` m=${m} swept (best: β=${best.beta} m=${best.m} w=${best.w} rps=${best.rps.toFixed(4)})`);
}
// ---------- validation stage 3: recent-form multiplier γ at the winners ----------
console.log('validation stage 3 — form multiplier γ…');
for (const gamma of GAMMA_GRID) {
if (gamma === 0) continue; // control covered above
for (const { w, rps: r } of sweepW(stagePreds, best.beta, best.m, gamma)) {
if (r < best.rps) best = { ...best, gamma, w, rps: r };
// ---------- validation stage 2: fast-Elo blend (m, β) at the stage-1 ξ/k ----------
console.log('validation stage 2 — fast-Elo blend (m, β)…');
const stagePreds = valWalks.get(`${best.xi}|${best.k}`)!;
for (const m of M_GRID) {
for (const beta of BETA_GRID) {
if (beta === 0) continue; // control already covered by stage 1
for (const { w, rps: r } of sweepW(stagePreds, beta, m, 0)) {
if (r < best.rps) best = { ...best, m, beta, w, rps: r };
}
}
console.log(` m=${m} swept (best: β=${best.beta} m=${best.m} w=${best.w} rps=${best.rps.toFixed(4)})`);
}
// ---------- validation stage 3: recent-form multiplier γ at the winners ----------
console.log('validation stage 3 — form multiplier γ…');
for (const gamma of GAMMA_GRID) {
if (gamma === 0) continue; // control covered above
for (const { w, rps: r } of sweepW(stagePreds, best.beta, best.m, gamma)) {
if (r < best.rps) best = { ...best, gamma, w, rps: r };
}
}
}
console.log(` champion: ξ=${best.xi} k=${best.k} w=${best.w} β=${best.beta} m=${best.m} γ=${best.gamma} rps=${best.rps.toFixed(4)}`);