dc52a908c0
- prediction_snapshots: model + de-vigged market frozen ≤15min pre-kickoff (or at first sight of a live match, flagged 'late' and excluded from totals). - src/lib/model/scoring.ts: ONE shared RPS/Brier/log-loss implementation for the backtest and the live scoreboard. - /api/scoreboard + /scoreboard page: per-match model-vs-bookmaker bars, RPS for both after FT, 'closer' badges, running head-to-head with honest rules printed (small-sample caveat included). Nav: 'vs Market'. - Methodology page now shows the three-way split + ensemble bake-off variants. - Verified end-to-end with a simulated match: snapshot froze model 80.7% and DraftKings 67.0% (de-vig correct), both scored at FT, late-flag honored. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
114 lines
4.4 KiB
TypeScript
114 lines
4.4 KiB
TypeScript
import { allSnapshots, hasSnapshot, saveSnapshot, oddsFor } from './db/db';
|
|
import { deVig } from '../../src/lib/odds';
|
|
import { brierScore, outcomeOfScore, rpsScore } from '../../src/lib/model/scoring';
|
|
import type { TournamentState } from './tournament';
|
|
import type { ModelEngine } from './model';
|
|
import type { MarketSnapshot, MatchProbs, ScoreboardData, ScoreboardRow } from '../../src/lib/types';
|
|
|
|
// The Model-vs-Market scoreboard. Fairness rules:
|
|
// - the model's forecast is FROZEN at most SNAPSHOT_BEFORE_MS before kickoff
|
|
// (whatever the model believed then — no hindsight);
|
|
// - the market benchmark is the latest captured pre-kickoff bookmaker line,
|
|
// de-vigged to fair probabilities;
|
|
// - snapshots taken after kickoff are stored but flagged `late` and excluded
|
|
// from the head-to-head totals.
|
|
|
|
const SNAPSHOT_BEFORE_MS = 15 * 60 * 1000;
|
|
|
|
function latestPreKickoffMarket(num: number, kickoffT: number): MarketSnapshot | null {
|
|
const history = oddsFor(num).filter((o) => o.captured_at <= kickoffT + 60_000);
|
|
const last = history[history.length - 1];
|
|
if (!last) return null;
|
|
const probs = deVig(last.home_ml, last.draw_ml, last.away_ml);
|
|
if (!probs) return null;
|
|
return {
|
|
provider: last.provider,
|
|
probs,
|
|
homeMl: last.home_ml!,
|
|
drawMl: last.draw_ml!,
|
|
awayMl: last.away_ml!,
|
|
};
|
|
}
|
|
|
|
/** Freeze forecasts for any fixture at/inside the snapshot window. Runs on every
|
|
* live tick — cheap (indexed point lookups, no network). */
|
|
export function snapshotCheck(state: TournamentState, model: ModelEngine): number {
|
|
const now = Date.now();
|
|
let taken = 0;
|
|
for (const f of state.allFixtures()) {
|
|
if (!f.home.team || !f.away.team) continue;
|
|
const kickoffT = new Date(f.kickoff).getTime();
|
|
// Inside the pre-kickoff window — or the match already started regardless of
|
|
// the scheduled clock (reschedules); the `late` flag keeps totals honest.
|
|
if (now < kickoffT - SNAPSHOT_BEFORE_MS && f.status === 'scheduled') continue;
|
|
if (hasSnapshot(f.num)) continue;
|
|
const pred = model.current().matches.find((m) => m.num === f.num);
|
|
if (!pred) continue; // finished before we ever saw it — nothing honest to freeze
|
|
const late = f.status !== 'scheduled';
|
|
saveSnapshot(f.num, late, { probs: pred.probs, lambdaHome: pred.lambdaHome, lambdaAway: pred.lambdaAway }, latestPreKickoffMarket(f.num, kickoffT));
|
|
taken++;
|
|
}
|
|
return taken;
|
|
}
|
|
|
|
export function buildScoreboard(state: TournamentState): ScoreboardData {
|
|
const byNum = new Map(state.allFixtures().map((f) => [f.num, f]));
|
|
const rows: ScoreboardRow[] = [];
|
|
let n = 0, mR = 0, kR = 0, mB = 0, kB = 0, mCloser = 0, kCloser = 0;
|
|
|
|
for (const s of allSnapshots()) {
|
|
const f = byNum.get(s.fixture_num);
|
|
if (!f || !f.home.team || !f.away.team) continue;
|
|
const modelStored = JSON.parse(s.model_json) as { probs: MatchProbs };
|
|
const market = s.market_json ? (JSON.parse(s.market_json) as MarketSnapshot) : null;
|
|
|
|
let scored: ScoreboardRow['scored'] = null;
|
|
if (f.status === 'finished' && f.homeScore != null && f.awayScore != null) {
|
|
const o = outcomeOfScore(f.homeScore, f.awayScore);
|
|
scored = {
|
|
outcome: o,
|
|
modelRps: +rpsScore(modelStored.probs, o).toFixed(4),
|
|
marketRps: market ? +rpsScore(market.probs, o).toFixed(4) : null,
|
|
modelBrier: +brierScore(modelStored.probs, o).toFixed(4),
|
|
marketBrier: market ? +brierScore(market.probs, o).toFixed(4) : null,
|
|
};
|
|
if (!s.late && market && scored.marketRps != null) {
|
|
n++;
|
|
mR += scored.modelRps; kR += scored.marketRps;
|
|
mB += scored.modelBrier; kB += scored.marketBrier!;
|
|
if (scored.modelRps < scored.marketRps) mCloser++;
|
|
else if (scored.marketRps < scored.modelRps) kCloser++;
|
|
}
|
|
}
|
|
|
|
rows.push({
|
|
num: f.num,
|
|
home: f.home.team,
|
|
away: f.away.team,
|
|
group: f.group,
|
|
kickoff: f.kickoff,
|
|
status: f.status,
|
|
homeScore: f.homeScore,
|
|
awayScore: f.awayScore,
|
|
takenAt: s.taken_at,
|
|
late: !!s.late,
|
|
model: modelStored.probs,
|
|
market,
|
|
scored,
|
|
});
|
|
}
|
|
|
|
rows.sort((a, b) => a.kickoff.localeCompare(b.kickoff));
|
|
return {
|
|
rows,
|
|
totals: n
|
|
? {
|
|
n,
|
|
modelRps: +(mR / n).toFixed(4), marketRps: +(kR / n).toFixed(4),
|
|
modelBrier: +(mB / n).toFixed(4), marketBrier: +(kB / n).toFixed(4),
|
|
modelCloser: mCloser, marketCloser: kCloser,
|
|
}
|
|
: null,
|
|
};
|
|
}
|