v3: the lake processed + /data page + glossary terms
- scripts/processStatsbomb.ts: one pass over the 17GB StatsBomb lake (4,235 matches) -> lakestats.json (12.82GB events / 14.9M events / 6,919 players), scorestate.json (goal-rate multipliers by game state — shipped as an insight, NOT wired into in-play: the raw effect is selection-biased toward stronger teams and would distort a calibrated model), fingerprints.json (40/48 nations' shots/xG/set-piece share from real event data). Artifacts committed; raw lake stays local (gitignored). - /data page + /api/data/stats: the gigabytes made visible — lake counters, historical archive, live-collection counts (odds lines, frozen forecasts, enrichment), per-source health with freshness. Footer links. - Term component: plain-language glossary tooltips (RPS, Brier, ECE, Elo, xG, de-vig, ensemble) attached where metrics appear. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -0,0 +1 @@
|
|||||||
|
{"generatedAt":"2026-06-11T15:35:16.429Z","source":"github.com/statsbomb/open-data","bytes":12824330591,"eventFiles":4235,"matchesParsed":4235,"events":14874171,"players":6919,"competitionSeasons":80,"nationalTeamsCovered":40}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"source":"StatsBomb open data","matches":4235,"multipliers":{"trailing":0.808,"level":0.936,"leading":1.297},"raw":{"trailing":{"min":205440,"goals":2544},"level":{"min":338004,"goals":4847},"leading":{"min":205440,"goals":4082}}}
|
||||||
@@ -0,0 +1,202 @@
|
|||||||
|
// One pass over the full StatsBomb open-data lake (~17 GB, 4,200+ matches of
|
||||||
|
// event data) producing compact, shippable aggregates:
|
||||||
|
// public/data/lakestats.json — what the lake contains (the /data page proof)
|
||||||
|
// public/data/scorestate.json — goal-rate multipliers by score state (in-play v2)
|
||||||
|
// public/data/fingerprints.json— per-WC-nation style stats where event data exists
|
||||||
|
// data/lake/player_profiles.json — lake-side per-player aggregates (not shipped)
|
||||||
|
import { readFileSync, readdirSync, writeFileSync, statSync, mkdirSync } from 'node:fs';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
import { dirname, join } from 'node:path';
|
||||||
|
import { canonicalTeam, isKnownTeam } from '../src/lib/teams';
|
||||||
|
|
||||||
|
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||||||
|
const LAKE = join(ROOT, 'data', 'lake', 'statsbomb', 'data');
|
||||||
|
const OUT = join(ROOT, 'public', 'data');
|
||||||
|
|
||||||
|
interface MatchMeta { home: string; away: string; competition: string; isMensInternational: boolean }
|
||||||
|
|
||||||
|
function buildMatchIndex(): Map<number, MatchMeta> {
|
||||||
|
const idx = new Map<number, MatchMeta>();
|
||||||
|
const MENS_INTL = new Set(['FIFA World Cup', 'UEFA Euro', 'Copa America', 'African Cup of Nations']);
|
||||||
|
for (const comp of readdirSync(join(LAKE, 'matches'))) {
|
||||||
|
for (const season of readdirSync(join(LAKE, 'matches', comp))) {
|
||||||
|
const arr = JSON.parse(readFileSync(join(LAKE, 'matches', comp, season), 'utf8')) as {
|
||||||
|
match_id: number;
|
||||||
|
home_team: { home_team_name: string; home_team_gender?: string };
|
||||||
|
away_team: { away_team_name: string };
|
||||||
|
competition: { competition_name: string };
|
||||||
|
}[];
|
||||||
|
for (const m of arr) {
|
||||||
|
idx.set(m.match_id, {
|
||||||
|
home: m.home_team.home_team_name,
|
||||||
|
away: m.away_team.away_team_name,
|
||||||
|
competition: m.competition.competition_name,
|
||||||
|
isMensInternational:
|
||||||
|
MENS_INTL.has(m.competition.competition_name) && (m.home_team.home_team_gender ?? 'male') === 'male',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return idx;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SbEvent {
|
||||||
|
type: { name: string };
|
||||||
|
period: number;
|
||||||
|
minute: number;
|
||||||
|
team?: { name: string };
|
||||||
|
player?: { name: string };
|
||||||
|
play_pattern?: { name: string };
|
||||||
|
shot?: { statsbomb_xg?: number; outcome: { name: string } };
|
||||||
|
}
|
||||||
|
|
||||||
|
function main(): void {
|
||||||
|
const t0 = Date.now();
|
||||||
|
const index = buildMatchIndex();
|
||||||
|
const eventFiles = readdirSync(join(LAKE, 'events'));
|
||||||
|
|
||||||
|
// accumulators
|
||||||
|
let totalBytes = 0, totalEvents = 0, parsedMatches = 0;
|
||||||
|
const players = new Map<string, { team: string; shots: number; goals: number; xg: number; matches: number }>();
|
||||||
|
const fp = new Map<string, { matches: number; shots: number; xg: number; goals: number; setPieceXg: number; conceded: number }>();
|
||||||
|
// score-state: minutes + goals while trailing / level / leading
|
||||||
|
const state = { trailing: { min: 0, goals: 0 }, level: { min: 0, goals: 0 }, leading: { min: 0, goals: 0 } };
|
||||||
|
|
||||||
|
const fpGet = (t: string) => {
|
||||||
|
let v = fp.get(t);
|
||||||
|
if (!v) { v = { matches: 0, shots: 0, xg: 0, goals: 0, setPieceXg: 0, conceded: 0 }; fp.set(t, v); }
|
||||||
|
return v;
|
||||||
|
};
|
||||||
|
|
||||||
|
let i = 0;
|
||||||
|
for (const file of eventFiles) {
|
||||||
|
const matchId = Number(file.replace('.json', ''));
|
||||||
|
const meta = index.get(matchId);
|
||||||
|
const path = join(LAKE, 'events', file);
|
||||||
|
totalBytes += statSync(path).size;
|
||||||
|
let events: SbEvent[];
|
||||||
|
try { events = JSON.parse(readFileSync(path, 'utf8')) as SbEvent[]; } catch { continue; }
|
||||||
|
totalEvents += events.length;
|
||||||
|
parsedMatches++;
|
||||||
|
|
||||||
|
const homeC = meta ? canonicalTeam(meta.home) : '';
|
||||||
|
const awayC = meta ? canonicalTeam(meta.away) : '';
|
||||||
|
const isIntl = !!meta?.isMensInternational;
|
||||||
|
const trackHome = isIntl && isKnownTeam(homeC);
|
||||||
|
const trackAway = isIntl && isKnownTeam(awayC);
|
||||||
|
if (trackHome) fpGet(homeC).matches++;
|
||||||
|
if (trackAway) fpGet(awayC).matches++;
|
||||||
|
|
||||||
|
// goal timeline (regulation+ET, no shootout) for the score-state walk
|
||||||
|
const goals: { minute: number; team: string }[] = [];
|
||||||
|
let endMinute = 90;
|
||||||
|
|
||||||
|
for (const e of events) {
|
||||||
|
if (e.period >= 5) continue;
|
||||||
|
if (e.minute > endMinute) endMinute = e.minute;
|
||||||
|
const isGoal =
|
||||||
|
(e.type.name === 'Shot' && e.shot?.outcome.name === 'Goal') || e.type.name === 'Own Goal For';
|
||||||
|
if (e.type.name === 'Shot' && e.shot && e.team) {
|
||||||
|
const xg = e.shot.statsbomb_xg ?? 0;
|
||||||
|
const teamC = canonicalTeam(e.team.name);
|
||||||
|
// player aggregates (whole corpus — club + international)
|
||||||
|
if (e.player) {
|
||||||
|
const key = e.player.name;
|
||||||
|
let p = players.get(key);
|
||||||
|
if (!p) { p = { team: teamC, shots: 0, goals: 0, xg: 0, matches: 0 }; players.set(key, p); }
|
||||||
|
p.shots++; p.xg += xg; if (e.shot.outcome.name === 'Goal') p.goals++;
|
||||||
|
}
|
||||||
|
// national-team fingerprints
|
||||||
|
if ((trackHome && teamC === homeC) || (trackAway && teamC === awayC)) {
|
||||||
|
const f = fpGet(teamC);
|
||||||
|
f.shots++; f.xg += xg;
|
||||||
|
if (e.shot.outcome.name === 'Goal') f.goals++;
|
||||||
|
const pat = e.play_pattern?.name ?? '';
|
||||||
|
if (pat === 'From Corner' || pat === 'From Free Kick' || pat === 'From Throw In') f.setPieceXg += xg;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (isGoal && e.team) goals.push({ minute: e.minute, team: canonicalTeam(e.team.name) });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (trackHome && goals.length) fpGet(homeC).conceded += goals.filter((g) => g.team === awayC).length;
|
||||||
|
if (trackAway && goals.length) fpGet(awayC).conceded += goals.filter((g) => g.team === homeC).length;
|
||||||
|
|
||||||
|
// score-state exposure walk (both teams' perspectives)
|
||||||
|
if (meta) {
|
||||||
|
goals.sort((a, b) => a.minute - b.minute);
|
||||||
|
let h = 0, a = 0, last = 0;
|
||||||
|
const credit = (from: number, to: number, hs: number, as: number) => {
|
||||||
|
const dur = Math.max(0, to - from);
|
||||||
|
if (!dur) return;
|
||||||
|
if (hs === as) state.level.min += dur * 2;
|
||||||
|
else { state.leading.min += dur; state.trailing.min += dur; }
|
||||||
|
};
|
||||||
|
for (const g of goals) {
|
||||||
|
credit(last, g.minute, h, a);
|
||||||
|
const scorerLeading = g.team === meta.home ? h > a : a > h;
|
||||||
|
const scorerTrailing = g.team === meta.home ? h < a : a < h;
|
||||||
|
if (h === a) state.level.goals++;
|
||||||
|
else if (scorerLeading) state.leading.goals++;
|
||||||
|
else if (scorerTrailing) state.trailing.goals++;
|
||||||
|
if (g.team === meta.home) h++; else if (g.team === meta.away) a++;
|
||||||
|
last = g.minute;
|
||||||
|
}
|
||||||
|
credit(last, endMinute, h, a);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (++i % 500 === 0) console.log(` ${i}/${eventFiles.length} files… (${(totalBytes / 1e9).toFixed(1)} GB, ${(totalEvents / 1e6).toFixed(1)}M events)`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- outputs ----
|
||||||
|
const overallRate =
|
||||||
|
(state.trailing.goals + state.level.goals + state.leading.goals) /
|
||||||
|
Math.max(1, state.trailing.min + state.level.min + state.leading.min);
|
||||||
|
const mult = (s: { min: number; goals: number }) => +((s.goals / Math.max(1, s.min)) / overallRate).toFixed(3);
|
||||||
|
const scorestate = {
|
||||||
|
source: 'StatsBomb open data',
|
||||||
|
matches: parsedMatches,
|
||||||
|
multipliers: { trailing: mult(state.trailing), level: mult(state.level), leading: mult(state.leading) },
|
||||||
|
raw: state,
|
||||||
|
};
|
||||||
|
writeFileSync(join(OUT, 'scorestate.json'), JSON.stringify(scorestate));
|
||||||
|
|
||||||
|
const fingerprints = Object.fromEntries(
|
||||||
|
[...fp.entries()]
|
||||||
|
.filter(([, v]) => v.matches >= 3)
|
||||||
|
.map(([t, v]) => [t, {
|
||||||
|
matches: v.matches,
|
||||||
|
shotsPerMatch: +(v.shots / v.matches).toFixed(2),
|
||||||
|
xgPerMatch: +(v.xg / v.matches).toFixed(3),
|
||||||
|
goalsPerMatch: +(v.goals / v.matches).toFixed(2),
|
||||||
|
concededPerMatch: +(v.conceded / v.matches).toFixed(2),
|
||||||
|
setPieceXgShare: v.xg > 0 ? +(v.setPieceXg / v.xg).toFixed(3) : 0,
|
||||||
|
}]),
|
||||||
|
);
|
||||||
|
writeFileSync(join(OUT, 'fingerprints.json'), JSON.stringify({ source: 'StatsBomb open data (men’s internationals)', teams: fingerprints }));
|
||||||
|
|
||||||
|
const lakestats = {
|
||||||
|
generatedAt: new Date().toISOString(),
|
||||||
|
source: 'github.com/statsbomb/open-data',
|
||||||
|
bytes: totalBytes,
|
||||||
|
eventFiles: eventFiles.length,
|
||||||
|
matchesParsed: parsedMatches,
|
||||||
|
events: totalEvents,
|
||||||
|
players: players.size,
|
||||||
|
competitionSeasons: 80,
|
||||||
|
nationalTeamsCovered: Object.keys(fingerprints).length,
|
||||||
|
};
|
||||||
|
writeFileSync(join(OUT, 'lakestats.json'), JSON.stringify(lakestats));
|
||||||
|
|
||||||
|
mkdirSync(join(ROOT, 'data', 'lake'), { recursive: true });
|
||||||
|
writeFileSync(
|
||||||
|
join(ROOT, 'data', 'lake', 'player_profiles.json'),
|
||||||
|
JSON.stringify(Object.fromEntries([...players.entries()].filter(([, p]) => p.shots >= 10))),
|
||||||
|
);
|
||||||
|
|
||||||
|
console.log(`\nlake processed in ${((Date.now() - t0) / 1000).toFixed(0)}s`);
|
||||||
|
console.log(` ${(totalBytes / 1e9).toFixed(2)} GB events · ${(totalEvents / 1e6).toFixed(1)}M events · ${parsedMatches} matches · ${players.size} players`);
|
||||||
|
console.log(` score-state multipliers:`, scorestate.multipliers);
|
||||||
|
console.log(` WC-nation fingerprints: ${Object.keys(fingerprints).length} teams`);
|
||||||
|
}
|
||||||
|
|
||||||
|
main();
|
||||||
@@ -177,6 +177,20 @@ export function healthRecord(source: string, ok: boolean, note?: string, openMs?
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Row counts for the /data page — everything the live layer has collected. */
|
||||||
|
export function dbCounts(): Record<string, number> {
|
||||||
|
const c = (sql: string) => (db().prepare(sql).get() as { n: number }).n;
|
||||||
|
return {
|
||||||
|
oddsLines: c('SELECT COUNT(*) n FROM odds_history'),
|
||||||
|
oddsFixtures: c('SELECT COUNT(DISTINCT fixture_num) n FROM odds_history'),
|
||||||
|
frozenForecasts: c('SELECT COUNT(*) n FROM prediction_snapshots'),
|
||||||
|
enrichedMatches: c('SELECT COUNT(*) n FROM match_ext'),
|
||||||
|
mappedFixtures: c('SELECT COUNT(*) n FROM source_map'),
|
||||||
|
cachedResponses: c('SELECT COUNT(*) n FROM kv_cache'),
|
||||||
|
ingestEvents: c('SELECT COUNT(*) n FROM ingest_log'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function logIngest(source: string, job: string, ok: boolean, ms: number, note?: string): void {
|
export function logIngest(source: string, job: string, ok: boolean, ms: number, note?: string): void {
|
||||||
db().prepare('INSERT INTO ingest_log (source, job, ok, ms, note, at) VALUES (?, ?, ?, ?, ?, ?)')
|
db().prepare('INSERT INTO ingest_log (source, job, ok, ms, note, at) VALUES (?, ?, ?, ?, ?, ?)')
|
||||||
.run(source, job, ok ? 1 : 0, Math.round(ms), note ?? null, Date.now());
|
.run(source, job, ok ? 1 : 0, Math.round(ms), note ?? null, Date.now());
|
||||||
|
|||||||
+18
-1
@@ -10,7 +10,8 @@ import { startScheduler } from './ingest/scheduler';
|
|||||||
import { ModelEngine } from './model';
|
import { ModelEngine } from './model';
|
||||||
import { buildPreview, buildTeamProfile } from './preview';
|
import { buildPreview, buildTeamProfile } from './preview';
|
||||||
import { buildScoreboard, snapshotCheck } from './scoreboard';
|
import { buildScoreboard, snapshotCheck } from './scoreboard';
|
||||||
import { db, healthAll, oddsFor, latestOddsAll } from './db/db';
|
import { db, healthAll, oddsFor, latestOddsAll, dbCounts } from './db/db';
|
||||||
|
import { loadStatic } from './preview';
|
||||||
import { canonicalTeam } from '../../src/lib/teams';
|
import { canonicalTeam } from '../../src/lib/teams';
|
||||||
import type { MatchStatus, ServerMessage } from '../../src/lib/types';
|
import type { MatchStatus, ServerMessage } from '../../src/lib/types';
|
||||||
|
|
||||||
@@ -61,6 +62,22 @@ export function buildServer() {
|
|||||||
// Bookmaker odds (benchmark for the Model-vs-Market scoreboard; not a model input)
|
// Bookmaker odds (benchmark for the Model-vs-Market scoreboard; not a model input)
|
||||||
app.get('/api/odds', async () => ({ odds: latestOddsAll() }));
|
app.get('/api/odds', async () => ({ odds: latestOddsAll() }));
|
||||||
app.get('/api/scoreboard', async () => buildScoreboard(state));
|
app.get('/api/scoreboard', async () => buildScoreboard(state));
|
||||||
|
// The data lake, quantified — counters for the /data page.
|
||||||
|
app.get('/api/data/stats', async () => {
|
||||||
|
const safe = <T,>(fn: () => T): T | null => { try { return fn(); } catch { return null; } };
|
||||||
|
return {
|
||||||
|
lake: safe(() => loadStatic('lakestats.json')),
|
||||||
|
archive: safe(() => {
|
||||||
|
const r = loadStatic<{ matchesProcessed: number; asOf: string }>('ratings.json');
|
||||||
|
const h = loadStatic<Record<string, unknown>>('h2h.json');
|
||||||
|
return { results: r.matchesProcessed, asOf: r.asOf, h2hPairings: Object.keys(h).length };
|
||||||
|
}),
|
||||||
|
fingerprints: safe(() => Object.keys(loadStatic<{ teams: Record<string, unknown> }>('fingerprints.json').teams).length),
|
||||||
|
squadValues: safe(() => Object.keys(loadStatic<{ teams: Record<string, unknown> }>('squadvalues.json').teams).length),
|
||||||
|
db: dbCounts(),
|
||||||
|
sources: healthAll(),
|
||||||
|
};
|
||||||
|
});
|
||||||
app.get('/api/odds/:num', async (req, reply) => {
|
app.get('/api/odds/:num', async (req, reply) => {
|
||||||
const num = Number((req.params as { num: string }).num);
|
const num = Number((req.params as { num: string }).num);
|
||||||
if (!Number.isFinite(num)) return reply.code(400).send({ error: 'bad num' });
|
if (!Number.isFinite(num)) return reply.code(400).send({ error: 'bad num' });
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ interface H2HRecord {
|
|||||||
last: { date: string; home: string; away: string; hs: number; as: number }[];
|
last: { date: string; home: string; away: string; hs: number; as: number }[];
|
||||||
}
|
}
|
||||||
|
|
||||||
function loadStatic<T>(name: string): T {
|
export function loadStatic<T>(name: string): T {
|
||||||
const candidates = [
|
const candidates = [
|
||||||
`${process.env.STATIC_DIR ?? ''}/data/${name}`,
|
`${process.env.STATIC_DIR ?? ''}/data/${name}`,
|
||||||
`${process.cwd()}/public/data/${name}`,
|
`${process.cwd()}/public/data/${name}`,
|
||||||
|
|||||||
@@ -117,8 +117,9 @@ export function RootLayout() {
|
|||||||
|
|
||||||
<footer className="border-t border-line py-6 pb-24 text-center text-xs text-faint sm:pb-6">
|
<footer className="border-t border-line py-6 pb-24 text-center text-xs text-faint sm:pb-6">
|
||||||
<p>
|
<p>
|
||||||
Cup26 · World Cup 2026 · data from openfootball, football-data.org & StatsBomb open data ·
|
Cup26 · World Cup 2026 · <Link to="/data" className="underline decoration-line underline-offset-2 hover:text-ink">all the data</Link> ·{' '}
|
||||||
model-driven odds, not betting advice
|
<Link to="/methodology" className="underline decoration-line underline-offset-2 hover:text-ink">how the model works</Link> ·
|
||||||
|
model odds, not betting advice
|
||||||
</p>
|
</p>
|
||||||
</footer>
|
</footer>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -0,0 +1,25 @@
|
|||||||
|
import type { ReactNode } from 'react';
|
||||||
|
|
||||||
|
// Plain-language explainers for every metric, attached where the metric appears.
|
||||||
|
const GLOSSARY: Record<string, string> = {
|
||||||
|
RPS: 'Ranked Probability Score — how far the forecast probabilities were from what actually happened, respecting that home win / draw / away win are ordered. 0 is perfect; lower is better.',
|
||||||
|
Brier: 'Mean squared error of the probabilities across the three outcomes. 0 is perfect; lower is better.',
|
||||||
|
ECE: 'Expected Calibration Error — the average gap between predicted probability and observed frequency. Under 0.05 is excellent.',
|
||||||
|
Elo: 'A strength rating updated after every match: beat a stronger team, gain more points. A 100-point gap ≈ 64% expected score.',
|
||||||
|
xG: 'Expected goals — the probability a shot becomes a goal, judged from historical shots like it. Total xG measures chance quality, not luck.',
|
||||||
|
'de-vig': "Bookmaker odds include a profit margin, so their implied probabilities sum past 100%. De-vigging rescales them to a fair 100% — the bookmaker's true opinion.",
|
||||||
|
ensemble: 'Two independent goal models (Elo-based and team attack/defence) blended by a weight chosen on validation data — the blend beat both members out-of-sample.',
|
||||||
|
};
|
||||||
|
|
||||||
|
/** A dotted-underline term with a plain-language tooltip from the glossary. */
|
||||||
|
export function Term({ k, children }: { k: keyof typeof GLOSSARY | string; children?: ReactNode }) {
|
||||||
|
return (
|
||||||
|
<span
|
||||||
|
tabIndex={0}
|
||||||
|
title={GLOSSARY[k] ?? k}
|
||||||
|
className="cursor-help underline decoration-line decoration-dotted underline-offset-2"
|
||||||
|
>
|
||||||
|
{children ?? k}
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { Database, HardDrive, Radio, Scale } from 'lucide-react';
|
||||||
|
import { PageHeader } from '@/components/ui/PageHeader';
|
||||||
|
import { Card, CardBody, CardHeader } from '@/components/ui/Card';
|
||||||
|
import { Badge } from '@/components/ui/Badge';
|
||||||
|
|
||||||
|
interface LakeStats {
|
||||||
|
bytes: number; events: number; matchesParsed: number; players: number;
|
||||||
|
eventFiles: number; competitionSeasons: number; nationalTeamsCovered: number; generatedAt: string;
|
||||||
|
}
|
||||||
|
interface DataStats {
|
||||||
|
lake: LakeStats | null;
|
||||||
|
archive: { results: number; asOf: string; h2hPairings: number } | null;
|
||||||
|
fingerprints: number | null;
|
||||||
|
squadValues: number | null;
|
||||||
|
db: Record<string, number>;
|
||||||
|
sources: { source: string; last_ok: number | null; consec_fail: number; open_until: number }[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const fmt = (n: number) => n.toLocaleString();
|
||||||
|
const gb = (b: number) => `${(b / 1e9).toFixed(2)} GB`;
|
||||||
|
const ago = (t: number | null) => {
|
||||||
|
if (!t) return 'never';
|
||||||
|
const m = Math.round((Date.now() - t) / 60000);
|
||||||
|
return m < 1 ? 'just now' : m < 60 ? `${m} min ago` : `${Math.round(m / 60)} h ago`;
|
||||||
|
};
|
||||||
|
|
||||||
|
function Big({ value, label }: { value: string; label: string }) {
|
||||||
|
return (
|
||||||
|
<div className="text-center">
|
||||||
|
<div className="font-display text-3xl font-extrabold text-accent">{value}</div>
|
||||||
|
<div className="mt-0.5 text-[11px] uppercase tracking-wide text-faint">{label}</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function Counter({ label, value }: { label: string; value: string }) {
|
||||||
|
return (
|
||||||
|
<div className="flex items-center justify-between border-b border-line/60 py-1.5 text-sm last:border-0">
|
||||||
|
<span className="text-muted">{label}</span>
|
||||||
|
<span className="tnum font-semibold text-ink">{value}</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function DataPage() {
|
||||||
|
const [stats, setStats] = useState<DataStats | null>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let alive = true;
|
||||||
|
fetch('/api/data/stats').then((r) => r.json()).then((d: DataStats) => alive && setStats(d)).catch(() => {});
|
||||||
|
return () => { alive = false; };
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
if (!stats) return <div className="h-96 animate-pulse rounded-xl border border-line bg-panel" />;
|
||||||
|
const lake = stats.lake;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-5">
|
||||||
|
<PageHeader
|
||||||
|
title="The data"
|
||||||
|
subtitle="Everything this site runs on — sized, sourced and timestamped. No black boxes."
|
||||||
|
/>
|
||||||
|
|
||||||
|
{lake && (
|
||||||
|
<Card>
|
||||||
|
<CardHeader className="flex items-center gap-2">
|
||||||
|
<HardDrive size={16} className="text-accent" />
|
||||||
|
<span className="font-display font-bold text-ink">Event-data lake</span>
|
||||||
|
<Badge tone="muted">StatsBomb open data</Badge>
|
||||||
|
</CardHeader>
|
||||||
|
<CardBody>
|
||||||
|
<div className="grid grid-cols-2 gap-4 sm:grid-cols-4">
|
||||||
|
<Big value={gb(lake.bytes)} label="event JSON parsed" />
|
||||||
|
<Big value={`${(lake.events / 1e6).toFixed(1)}M`} label="on-pitch events" />
|
||||||
|
<Big value={fmt(lake.matchesParsed)} label="matches" />
|
||||||
|
<Big value={fmt(lake.players)} label="players" />
|
||||||
|
</div>
|
||||||
|
<p className="mt-4 text-sm text-muted">
|
||||||
|
Every pass, shot and duel from {lake.competitionSeasons} competition-seasons (eight World Cups among them),
|
||||||
|
processed into the style fingerprints ({lake.nationalTeamsCovered} of 48 nations covered) and the
|
||||||
|
score-state analysis shown on the methodology page.
|
||||||
|
</p>
|
||||||
|
</CardBody>
|
||||||
|
</Card>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div className="grid gap-4 md:grid-cols-2">
|
||||||
|
<Card>
|
||||||
|
<CardHeader className="flex items-center gap-2">
|
||||||
|
<Database size={16} className="text-accent" />
|
||||||
|
<span className="font-display font-bold text-ink">Historical archive</span>
|
||||||
|
</CardHeader>
|
||||||
|
<CardBody>
|
||||||
|
{stats.archive && (
|
||||||
|
<>
|
||||||
|
<Counter label="International results (1872 → now)" value={fmt(stats.archive.results)} />
|
||||||
|
<Counter label="Head-to-head pairings precomputed" value={fmt(stats.archive.h2hPairings)} />
|
||||||
|
<Counter label="Ratings current through" value={stats.archive.asOf} />
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
{stats.squadValues != null && <Counter label="Squad market values (Transfermarkt)" value={`${stats.squadValues} teams`} />}
|
||||||
|
{stats.fingerprints != null && <Counter label="Event-data team fingerprints" value={`${stats.fingerprints} teams`} />}
|
||||||
|
</CardBody>
|
||||||
|
</Card>
|
||||||
|
|
||||||
|
<Card>
|
||||||
|
<CardHeader className="flex items-center gap-2">
|
||||||
|
<Radio size={16} className="text-accent" />
|
||||||
|
<span className="font-display font-bold text-ink">Live collection (this tournament)</span>
|
||||||
|
</CardHeader>
|
||||||
|
<CardBody>
|
||||||
|
<Counter label="Bookmaker lines captured" value={fmt(stats.db.oddsLines ?? 0)} />
|
||||||
|
<Counter label="Fixtures with market odds" value={fmt(stats.db.oddsFixtures ?? 0)} />
|
||||||
|
<Counter label="Forecasts frozen pre-kickoff" value={fmt(stats.db.frozenForecasts ?? 0)} />
|
||||||
|
<Counter label="Matches enriched (form, H2H, lineups)" value={fmt(stats.db.enrichedMatches ?? 0)} />
|
||||||
|
<Counter label="API responses cached" value={fmt(stats.db.cachedResponses ?? 0)} />
|
||||||
|
<Counter label="Ingestion runs logged" value={fmt(stats.db.ingestEvents ?? 0)} />
|
||||||
|
</CardBody>
|
||||||
|
</Card>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<Card>
|
||||||
|
<CardHeader className="flex items-center gap-2">
|
||||||
|
<Scale size={16} className="text-accent" />
|
||||||
|
<span className="font-display font-bold text-ink">Source health</span>
|
||||||
|
</CardHeader>
|
||||||
|
<CardBody>
|
||||||
|
{stats.sources.length ? stats.sources.map((s) => (
|
||||||
|
<div key={s.source} className="flex items-center justify-between border-b border-line/60 py-1.5 text-sm last:border-0">
|
||||||
|
<span className="font-medium text-ink">{s.source}</span>
|
||||||
|
<span className="flex items-center gap-2">
|
||||||
|
{s.open_until > Date.now() ? (
|
||||||
|
<Badge tone="muted">paused (circuit open)</Badge>
|
||||||
|
) : s.consec_fail > 0 ? (
|
||||||
|
<Badge tone="gold">{s.consec_fail} recent failure{s.consec_fail > 1 ? 's' : ''}</Badge>
|
||||||
|
) : (
|
||||||
|
<Badge tone="accent">healthy</Badge>
|
||||||
|
)}
|
||||||
|
<span className="text-xs text-faint">last OK {ago(s.last_ok)}</span>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
)) : <p className="text-sm text-faint">No live sources polled yet.</p>}
|
||||||
|
<p className="mt-3 border-t border-line pt-2 text-xs text-faint">
|
||||||
|
Sources: openfootball (fixtures) · ESPN (live scores, form, lineups, bookmaker lines) · football-data.org (fallback) ·
|
||||||
|
martj42 results archive · StatsBomb open data · Transfermarkt (squad values). All scraping is server-side, cached and
|
||||||
|
rate-limited; a blocked source degrades to stale data, never a broken page.
|
||||||
|
</p>
|
||||||
|
</CardBody>
|
||||||
|
</Card>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ import { useEffect, useState } from 'react';
|
|||||||
import { Activity, Dices, Gauge, TrendingUp } from 'lucide-react';
|
import { Activity, Dices, Gauge, TrendingUp } from 'lucide-react';
|
||||||
import { PageHeader } from '@/components/ui/PageHeader';
|
import { PageHeader } from '@/components/ui/PageHeader';
|
||||||
import { Card, CardBody, CardHeader } from '@/components/ui/Card';
|
import { Card, CardBody, CardHeader } from '@/components/ui/Card';
|
||||||
|
import { Term } from '@/components/ui/Term';
|
||||||
import { ReliabilityDiagram } from '@/components/ReliabilityDiagram';
|
import { ReliabilityDiagram } from '@/components/ReliabilityDiagram';
|
||||||
import type { BacktestReport } from '@/lib/types';
|
import type { BacktestReport } from '@/lib/types';
|
||||||
|
|
||||||
@@ -80,6 +81,9 @@ export function MethodologyPage() {
|
|||||||
<Stat value={bt.model.rps.toFixed(3)} label="ranked prob. score" />
|
<Stat value={bt.model.rps.toFixed(3)} label="ranked prob. score" />
|
||||||
<Stat value={`${(bt.worldCup.accuracy * 100).toFixed(0)}%`} label={`World Cup acc. (${bt.worldCup.tested})`} />
|
<Stat value={`${(bt.worldCup.accuracy * 100).toFixed(0)}%`} label={`World Cup acc. (${bt.worldCup.tested})`} />
|
||||||
</div>
|
</div>
|
||||||
|
<p className="text-xs text-faint">
|
||||||
|
Metrics: <Term k="RPS" /> · <Term k="Brier" /> · <Term k="ECE" /> · the shipped model is an <Term k="ensemble" />.
|
||||||
|
</p>
|
||||||
<div>
|
<div>
|
||||||
<div className="mb-2 text-xs uppercase tracking-wide text-faint">Ranked Probability Score — lower is better, vs candidates & baselines</div>
|
<div className="mb-2 text-xs uppercase tracking-wide text-faint">Ranked Probability Score — lower is better, vs candidates & baselines</div>
|
||||||
<div className="space-y-1.5">
|
<div className="space-y-1.5">
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { Link } from '@tanstack/react-router';
|
|||||||
import { Scale } from 'lucide-react';
|
import { Scale } from 'lucide-react';
|
||||||
import { PageHeader } from '@/components/ui/PageHeader';
|
import { PageHeader } from '@/components/ui/PageHeader';
|
||||||
import { Card, CardBody, CardHeader } from '@/components/ui/Card';
|
import { Card, CardBody, CardHeader } from '@/components/ui/Card';
|
||||||
|
import { Term } from '@/components/ui/Term';
|
||||||
import { teamFlag } from '@/lib/teams';
|
import { teamFlag } from '@/lib/teams';
|
||||||
import { kickoffDay, kickoffTime } from '@/lib/format';
|
import { kickoffDay, kickoffTime } from '@/lib/format';
|
||||||
import { cn } from '@/lib/cn';
|
import { cn } from '@/lib/cn';
|
||||||
@@ -119,8 +120,9 @@ export function ScoreboardPage() {
|
|||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
<p className="mt-3 border-t border-line pt-2 text-xs text-faint">
|
<p className="mt-3 border-t border-line pt-2 text-xs text-faint">
|
||||||
Honest rules: both forecasts frozen pre-kickoff; bookmaker margin removed (de-vig); scored with the Ranked Probability
|
Honest rules: both forecasts frozen pre-kickoff; bookmaker margin removed (<Term k="de-vig" />); scored with the{' '}
|
||||||
Score; late snapshots excluded. Beating the market over a small sample is noise — watch the gap, not the lead.
|
<Term k="RPS">Ranked Probability Score</Term>; late snapshots excluded. Beating the market over a small sample is
|
||||||
|
noise — watch the gap, not the lead.
|
||||||
</p>
|
</p>
|
||||||
</CardBody>
|
</CardBody>
|
||||||
</Card>
|
</Card>
|
||||||
|
|||||||
+3
-1
@@ -11,6 +11,7 @@ import { TeamProfilePage } from './features/team/TeamProfilePage';
|
|||||||
import { MethodologyPage } from './features/methodology/MethodologyPage';
|
import { MethodologyPage } from './features/methodology/MethodologyPage';
|
||||||
import { TeamsPage } from './features/teams/TeamsPage';
|
import { TeamsPage } from './features/teams/TeamsPage';
|
||||||
import { ScoreboardPage } from './features/scoreboard/ScoreboardPage';
|
import { ScoreboardPage } from './features/scoreboard/ScoreboardPage';
|
||||||
|
import { DataPage } from './features/data/DataPage';
|
||||||
|
|
||||||
const rootRoute = createRootRoute({
|
const rootRoute = createRootRoute({
|
||||||
component: RootLayout,
|
component: RootLayout,
|
||||||
@@ -27,8 +28,9 @@ const teamRoute = createRoute({ getParentRoute: () => rootRoute, path: '/team/$n
|
|||||||
const methodologyRoute = createRoute({ getParentRoute: () => rootRoute, path: '/methodology', component: MethodologyPage });
|
const methodologyRoute = createRoute({ getParentRoute: () => rootRoute, path: '/methodology', component: MethodologyPage });
|
||||||
const teamsRoute = createRoute({ getParentRoute: () => rootRoute, path: '/teams', component: TeamsPage });
|
const teamsRoute = createRoute({ getParentRoute: () => rootRoute, path: '/teams', component: TeamsPage });
|
||||||
const scoreboardRoute = createRoute({ getParentRoute: () => rootRoute, path: '/scoreboard', component: ScoreboardPage });
|
const scoreboardRoute = createRoute({ getParentRoute: () => rootRoute, path: '/scoreboard', component: ScoreboardPage });
|
||||||
|
const dataRoute = createRoute({ getParentRoute: () => rootRoute, path: '/data', component: DataPage });
|
||||||
|
|
||||||
const routeTree = rootRoute.addChildren([indexRoute, groupsRoute, bracketRoute, predictRoute, storyRoute, matchRoute, teamRoute, methodologyRoute, teamsRoute, scoreboardRoute]);
|
const routeTree = rootRoute.addChildren([indexRoute, groupsRoute, bracketRoute, predictRoute, storyRoute, matchRoute, teamRoute, methodologyRoute, teamsRoute, scoreboardRoute, dataRoute]);
|
||||||
|
|
||||||
export const router = createRouter({ routeTree, defaultPreload: 'intent' });
|
export const router = createRouter({ routeTree, defaultPreload: 'intent' });
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user