Files
cup26/server/src/scoreboard.ts
T
NilsBriggen dc52a908c0 v3: Model-vs-Market scoreboard — frozen forecasts, fair scoring, live page
- prediction_snapshots: model + de-vigged market frozen ≤15min pre-kickoff
  (or at first sight of a live match, flagged 'late' and excluded from totals).
- src/lib/model/scoring.ts: ONE shared RPS/Brier/log-loss implementation for
  the backtest and the live scoreboard.
- /api/scoreboard + /scoreboard page: per-match model-vs-bookmaker bars, RPS
  for both after FT, 'closer' badges, running head-to-head with honest rules
  printed (small-sample caveat included). Nav: 'vs Market'.
- Methodology page now shows the three-way split + ensemble bake-off variants.
- Verified end-to-end with a simulated match: snapshot froze model 80.7% and
  DraftKings 67.0% (de-vig correct), both scored at FT, late-flag honored.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-11 17:31:09 +02:00

114 lines
4.4 KiB
TypeScript

import { allSnapshots, hasSnapshot, saveSnapshot, oddsFor } from './db/db';
import { deVig } from '../../src/lib/odds';
import { brierScore, outcomeOfScore, rpsScore } from '../../src/lib/model/scoring';
import type { TournamentState } from './tournament';
import type { ModelEngine } from './model';
import type { MarketSnapshot, MatchProbs, ScoreboardData, ScoreboardRow } from '../../src/lib/types';
// The Model-vs-Market scoreboard. Fairness rules:
// - the model's forecast is FROZEN at most SNAPSHOT_BEFORE_MS before kickoff
// (whatever the model believed then — no hindsight);
// - the market benchmark is the latest captured pre-kickoff bookmaker line,
// de-vigged to fair probabilities;
// - snapshots taken after kickoff are stored but flagged `late` and excluded
// from the head-to-head totals.
const SNAPSHOT_BEFORE_MS = 15 * 60 * 1000;
function latestPreKickoffMarket(num: number, kickoffT: number): MarketSnapshot | null {
const history = oddsFor(num).filter((o) => o.captured_at <= kickoffT + 60_000);
const last = history[history.length - 1];
if (!last) return null;
const probs = deVig(last.home_ml, last.draw_ml, last.away_ml);
if (!probs) return null;
return {
provider: last.provider,
probs,
homeMl: last.home_ml!,
drawMl: last.draw_ml!,
awayMl: last.away_ml!,
};
}
/** Freeze forecasts for any fixture at/inside the snapshot window. Runs on every
* live tick — cheap (indexed point lookups, no network). */
export function snapshotCheck(state: TournamentState, model: ModelEngine): number {
const now = Date.now();
let taken = 0;
for (const f of state.allFixtures()) {
if (!f.home.team || !f.away.team) continue;
const kickoffT = new Date(f.kickoff).getTime();
// Inside the pre-kickoff window — or the match already started regardless of
// the scheduled clock (reschedules); the `late` flag keeps totals honest.
if (now < kickoffT - SNAPSHOT_BEFORE_MS && f.status === 'scheduled') continue;
if (hasSnapshot(f.num)) continue;
const pred = model.current().matches.find((m) => m.num === f.num);
if (!pred) continue; // finished before we ever saw it — nothing honest to freeze
const late = f.status !== 'scheduled';
saveSnapshot(f.num, late, { probs: pred.probs, lambdaHome: pred.lambdaHome, lambdaAway: pred.lambdaAway }, latestPreKickoffMarket(f.num, kickoffT));
taken++;
}
return taken;
}
export function buildScoreboard(state: TournamentState): ScoreboardData {
const byNum = new Map(state.allFixtures().map((f) => [f.num, f]));
const rows: ScoreboardRow[] = [];
let n = 0, mR = 0, kR = 0, mB = 0, kB = 0, mCloser = 0, kCloser = 0;
for (const s of allSnapshots()) {
const f = byNum.get(s.fixture_num);
if (!f || !f.home.team || !f.away.team) continue;
const modelStored = JSON.parse(s.model_json) as { probs: MatchProbs };
const market = s.market_json ? (JSON.parse(s.market_json) as MarketSnapshot) : null;
let scored: ScoreboardRow['scored'] = null;
if (f.status === 'finished' && f.homeScore != null && f.awayScore != null) {
const o = outcomeOfScore(f.homeScore, f.awayScore);
scored = {
outcome: o,
modelRps: +rpsScore(modelStored.probs, o).toFixed(4),
marketRps: market ? +rpsScore(market.probs, o).toFixed(4) : null,
modelBrier: +brierScore(modelStored.probs, o).toFixed(4),
marketBrier: market ? +brierScore(market.probs, o).toFixed(4) : null,
};
if (!s.late && market && scored.marketRps != null) {
n++;
mR += scored.modelRps; kR += scored.marketRps;
mB += scored.modelBrier; kB += scored.marketBrier!;
if (scored.modelRps < scored.marketRps) mCloser++;
else if (scored.marketRps < scored.modelRps) kCloser++;
}
}
rows.push({
num: f.num,
home: f.home.team,
away: f.away.team,
group: f.group,
kickoff: f.kickoff,
status: f.status,
homeScore: f.homeScore,
awayScore: f.awayScore,
takenAt: s.taken_at,
late: !!s.late,
model: modelStored.probs,
market,
scored,
});
}
rows.sort((a, b) => a.kickoff.localeCompare(b.kickoff));
return {
rows,
totals: n
? {
n,
modelRps: +(mR / n).toFixed(4), marketRps: +(kR / n).toFixed(4),
modelBrier: +(mB / n).toFixed(4), marketBrier: +(kB / n).toFixed(4),
modelCloser: mCloser, marketCloser: kCloser,
}
: null,
};
}