Quick backtest mode for image builds (identical published numbers)
BACKTEST_QUICK=1 skips the hyperparameter grid and scores only the shipped + previous configs — same test metrics, a fraction of the build time on the 1-CPU VPS. The full grid search stays a local activity. Verified byte-level: quick output matches the grid run. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+4
-1
@@ -4,7 +4,10 @@ WORKDIR /app
|
||||
COPY package.json bun.lock* bun.lockb* ./
|
||||
RUN bun install
|
||||
COPY . .
|
||||
RUN bun run data:build # icons + fixtures.json + ratings.json → public/
|
||||
# Quick backtest in image builds: score the shipped config only — the published
|
||||
# numbers are identical; the hyperparam grid search is a local-dev activity.
|
||||
ENV BACKTEST_QUICK=1
|
||||
RUN bun run data:build # icons + fixtures.json + ratings.json + backtest → public/
|
||||
RUN bun run build # tsc -b && vite build → /app/dist
|
||||
RUN bun run build:server # esbuild → /app/server/dist/index.js
|
||||
|
||||
|
||||
+49
-30
@@ -40,6 +40,12 @@ const FORM_WINDOW = 10; // last N non-friendly matches feeding the form signal
|
||||
const MIN_VAL_GAIN = 0.0005; // candidate must beat the previous config by this much
|
||||
/** the previously shipped v3 config — the bar every candidate has to clear */
|
||||
const PREV = { xi: 0.5, k: 5, w: 0.75, m: 2, beta: 0, gamma: 0 };
|
||||
/** the v4 winner (from the full grid run) — keep in sync with buildRatings.ts */
|
||||
const SHIPPED = { xi: 0.25, k: 3, w: 0.45, m: 3, beta: 0.3, gamma: 0 };
|
||||
/** BACKTEST_QUICK=1 (Docker builds): skip the grid search and just score the
|
||||
* shipped + previous configs — the published test numbers are identical, the
|
||||
* hyperparam hunt only happens locally. */
|
||||
const QUICK = process.env.BACKTEST_QUICK === '1';
|
||||
|
||||
interface Row { date: string; home: string; away: string; hs: number; as: number; tournament: string; neutral: boolean; t: number }
|
||||
type Probs = { h: number; d: number; a: number };
|
||||
@@ -241,42 +247,55 @@ function main(): void {
|
||||
return W_STEPS.map((w, wi) => ({ w, rps: acc[wi]! / preds.length }));
|
||||
};
|
||||
|
||||
// ---------- validation stage 1: tune ξ, k, w (β=0, γ=0) ----------
|
||||
console.log('validation stage 1 — ξ/k/w grid (β=0, γ=0)…');
|
||||
const valWalks = new Map<string, Pred[]>();
|
||||
let best = { xi: PREV.xi, k: PREV.k, w: PREV.w, m: PREV.m, beta: 0, gamma: 0, rps: Infinity };
|
||||
let prevValRps = Infinity; // validation RPS of the previously shipped config
|
||||
for (const xi of XI_GRID) {
|
||||
for (const k of K_GRID) {
|
||||
const preds = walkTeamDc(xi, k, tValFrom, tTestFrom);
|
||||
valWalks.set(`${xi}|${k}`, preds);
|
||||
for (const { w, rps: r } of sweepW(preds, 0, PREV.m, 0)) {
|
||||
if (xi === PREV.xi && k === PREV.k && w === PREV.w) prevValRps = r;
|
||||
if (r < best.rps) best = { xi, k, w, m: PREV.m, beta: 0, gamma: 0, rps: r };
|
||||
}
|
||||
console.log(` ξ=${xi} k=${k} done (best so far: ξ=${best.xi} k=${best.k} w=${best.w} rps=${best.rps.toFixed(4)})`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- validation stage 2: fast-Elo blend (m, β) at the stage-1 ξ/k ----------
|
||||
console.log('validation stage 2 — fast-Elo blend (m, β)…');
|
||||
const stagePreds = valWalks.get(`${best.xi}|${best.k}`)!;
|
||||
for (const m of M_GRID) {
|
||||
for (const beta of BETA_GRID) {
|
||||
if (beta === 0) continue; // control already covered by stage 1
|
||||
for (const { w, rps: r } of sweepW(stagePreds, beta, m, 0)) {
|
||||
if (r < best.rps) best = { ...best, m, beta, w, rps: r };
|
||||
if (QUICK) {
|
||||
// ---------- quick mode: score only shipped + previous on validation ----------
|
||||
console.log('quick mode — scoring shipped + previous configs on validation…');
|
||||
const predsShipped = walkTeamDc(SHIPPED.xi, SHIPPED.k, tValFrom, tTestFrom);
|
||||
const predsPrev = SHIPPED.xi === PREV.xi && SHIPPED.k === PREV.k
|
||||
? predsShipped
|
||||
: walkTeamDc(PREV.xi, PREV.k, tValFrom, tTestFrom);
|
||||
const shippedRps = sweepW(predsShipped, SHIPPED.beta, SHIPPED.m, SHIPPED.gamma).find((x) => x.w === SHIPPED.w)!.rps;
|
||||
prevValRps = sweepW(predsPrev, PREV.beta, PREV.m, PREV.gamma).find((x) => x.w === PREV.w)!.rps;
|
||||
best = { ...SHIPPED, rps: shippedRps };
|
||||
} else {
|
||||
// ---------- validation stage 1: tune ξ, k, w (β=0, γ=0) ----------
|
||||
console.log('validation stage 1 — ξ/k/w grid (β=0, γ=0)…');
|
||||
const valWalks = new Map<string, Pred[]>();
|
||||
for (const xi of XI_GRID) {
|
||||
for (const k of K_GRID) {
|
||||
const preds = walkTeamDc(xi, k, tValFrom, tTestFrom);
|
||||
valWalks.set(`${xi}|${k}`, preds);
|
||||
for (const { w, rps: r } of sweepW(preds, 0, PREV.m, 0)) {
|
||||
if (xi === PREV.xi && k === PREV.k && w === PREV.w) prevValRps = r;
|
||||
if (r < best.rps) best = { xi, k, w, m: PREV.m, beta: 0, gamma: 0, rps: r };
|
||||
}
|
||||
console.log(` ξ=${xi} k=${k} done (best so far: ξ=${best.xi} k=${best.k} w=${best.w} rps=${best.rps.toFixed(4)})`);
|
||||
}
|
||||
}
|
||||
console.log(` m=${m} swept (best: β=${best.beta} m=${best.m} w=${best.w} rps=${best.rps.toFixed(4)})`);
|
||||
}
|
||||
|
||||
// ---------- validation stage 3: recent-form multiplier γ at the winners ----------
|
||||
console.log('validation stage 3 — form multiplier γ…');
|
||||
for (const gamma of GAMMA_GRID) {
|
||||
if (gamma === 0) continue; // control covered above
|
||||
for (const { w, rps: r } of sweepW(stagePreds, best.beta, best.m, gamma)) {
|
||||
if (r < best.rps) best = { ...best, gamma, w, rps: r };
|
||||
// ---------- validation stage 2: fast-Elo blend (m, β) at the stage-1 ξ/k ----------
|
||||
console.log('validation stage 2 — fast-Elo blend (m, β)…');
|
||||
const stagePreds = valWalks.get(`${best.xi}|${best.k}`)!;
|
||||
for (const m of M_GRID) {
|
||||
for (const beta of BETA_GRID) {
|
||||
if (beta === 0) continue; // control already covered by stage 1
|
||||
for (const { w, rps: r } of sweepW(stagePreds, beta, m, 0)) {
|
||||
if (r < best.rps) best = { ...best, m, beta, w, rps: r };
|
||||
}
|
||||
}
|
||||
console.log(` m=${m} swept (best: β=${best.beta} m=${best.m} w=${best.w} rps=${best.rps.toFixed(4)})`);
|
||||
}
|
||||
|
||||
// ---------- validation stage 3: recent-form multiplier γ at the winners ----------
|
||||
console.log('validation stage 3 — form multiplier γ…');
|
||||
for (const gamma of GAMMA_GRID) {
|
||||
if (gamma === 0) continue; // control covered above
|
||||
for (const { w, rps: r } of sweepW(stagePreds, best.beta, best.m, gamma)) {
|
||||
if (r < best.rps) best = { ...best, gamma, w, rps: r };
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(` champion: ξ=${best.xi} k=${best.k} w=${best.w} β=${best.beta} m=${best.m} γ=${best.gamma} rps=${best.rps.toFixed(4)}`);
|
||||
|
||||
Reference in New Issue
Block a user