v3: the lake processed + /data page + glossary terms
- scripts/processStatsbomb.ts: one pass over the 17GB StatsBomb lake (4,235 matches) -> lakestats.json (12.82GB events / 14.9M events / 6,919 players), scorestate.json (goal-rate multipliers by game state — shipped as an insight, NOT wired into in-play: the raw effect is selection-biased toward stronger teams and would distort a calibrated model), fingerprints.json (40/48 nations' shots/xG/set-piece share from real event data). Artifacts committed; raw lake stays local (gitignored). - /data page + /api/data/stats: the gigabytes made visible — lake counters, historical archive, live-collection counts (odds lines, frozen forecasts, enrichment), per-source health with freshness. Footer links. - Term component: plain-language glossary tooltips (RPS, Brier, ECE, Elo, xG, de-vig, ensemble) attached where metrics appear. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,202 @@
|
||||
// One pass over the full StatsBomb open-data lake (~17 GB, 4,200+ matches of
|
||||
// event data) producing compact, shippable aggregates:
|
||||
// public/data/lakestats.json — what the lake contains (the /data page proof)
|
||||
// public/data/scorestate.json — goal-rate multipliers by score state (in-play v2)
|
||||
// public/data/fingerprints.json— per-WC-nation style stats where event data exists
|
||||
// data/lake/player_profiles.json — lake-side per-player aggregates (not shipped)
|
||||
import { readFileSync, readdirSync, writeFileSync, statSync, mkdirSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { canonicalTeam, isKnownTeam } from '../src/lib/teams';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const LAKE = join(ROOT, 'data', 'lake', 'statsbomb', 'data');
|
||||
const OUT = join(ROOT, 'public', 'data');
|
||||
|
||||
interface MatchMeta { home: string; away: string; competition: string; isMensInternational: boolean }
|
||||
|
||||
function buildMatchIndex(): Map<number, MatchMeta> {
|
||||
const idx = new Map<number, MatchMeta>();
|
||||
const MENS_INTL = new Set(['FIFA World Cup', 'UEFA Euro', 'Copa America', 'African Cup of Nations']);
|
||||
for (const comp of readdirSync(join(LAKE, 'matches'))) {
|
||||
for (const season of readdirSync(join(LAKE, 'matches', comp))) {
|
||||
const arr = JSON.parse(readFileSync(join(LAKE, 'matches', comp, season), 'utf8')) as {
|
||||
match_id: number;
|
||||
home_team: { home_team_name: string; home_team_gender?: string };
|
||||
away_team: { away_team_name: string };
|
||||
competition: { competition_name: string };
|
||||
}[];
|
||||
for (const m of arr) {
|
||||
idx.set(m.match_id, {
|
||||
home: m.home_team.home_team_name,
|
||||
away: m.away_team.away_team_name,
|
||||
competition: m.competition.competition_name,
|
||||
isMensInternational:
|
||||
MENS_INTL.has(m.competition.competition_name) && (m.home_team.home_team_gender ?? 'male') === 'male',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return idx;
|
||||
}
|
||||
|
||||
interface SbEvent {
|
||||
type: { name: string };
|
||||
period: number;
|
||||
minute: number;
|
||||
team?: { name: string };
|
||||
player?: { name: string };
|
||||
play_pattern?: { name: string };
|
||||
shot?: { statsbomb_xg?: number; outcome: { name: string } };
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const t0 = Date.now();
|
||||
const index = buildMatchIndex();
|
||||
const eventFiles = readdirSync(join(LAKE, 'events'));
|
||||
|
||||
// accumulators
|
||||
let totalBytes = 0, totalEvents = 0, parsedMatches = 0;
|
||||
const players = new Map<string, { team: string; shots: number; goals: number; xg: number; matches: number }>();
|
||||
const fp = new Map<string, { matches: number; shots: number; xg: number; goals: number; setPieceXg: number; conceded: number }>();
|
||||
// score-state: minutes + goals while trailing / level / leading
|
||||
const state = { trailing: { min: 0, goals: 0 }, level: { min: 0, goals: 0 }, leading: { min: 0, goals: 0 } };
|
||||
|
||||
const fpGet = (t: string) => {
|
||||
let v = fp.get(t);
|
||||
if (!v) { v = { matches: 0, shots: 0, xg: 0, goals: 0, setPieceXg: 0, conceded: 0 }; fp.set(t, v); }
|
||||
return v;
|
||||
};
|
||||
|
||||
let i = 0;
|
||||
for (const file of eventFiles) {
|
||||
const matchId = Number(file.replace('.json', ''));
|
||||
const meta = index.get(matchId);
|
||||
const path = join(LAKE, 'events', file);
|
||||
totalBytes += statSync(path).size;
|
||||
let events: SbEvent[];
|
||||
try { events = JSON.parse(readFileSync(path, 'utf8')) as SbEvent[]; } catch { continue; }
|
||||
totalEvents += events.length;
|
||||
parsedMatches++;
|
||||
|
||||
const homeC = meta ? canonicalTeam(meta.home) : '';
|
||||
const awayC = meta ? canonicalTeam(meta.away) : '';
|
||||
const isIntl = !!meta?.isMensInternational;
|
||||
const trackHome = isIntl && isKnownTeam(homeC);
|
||||
const trackAway = isIntl && isKnownTeam(awayC);
|
||||
if (trackHome) fpGet(homeC).matches++;
|
||||
if (trackAway) fpGet(awayC).matches++;
|
||||
|
||||
// goal timeline (regulation+ET, no shootout) for the score-state walk
|
||||
const goals: { minute: number; team: string }[] = [];
|
||||
let endMinute = 90;
|
||||
|
||||
for (const e of events) {
|
||||
if (e.period >= 5) continue;
|
||||
if (e.minute > endMinute) endMinute = e.minute;
|
||||
const isGoal =
|
||||
(e.type.name === 'Shot' && e.shot?.outcome.name === 'Goal') || e.type.name === 'Own Goal For';
|
||||
if (e.type.name === 'Shot' && e.shot && e.team) {
|
||||
const xg = e.shot.statsbomb_xg ?? 0;
|
||||
const teamC = canonicalTeam(e.team.name);
|
||||
// player aggregates (whole corpus — club + international)
|
||||
if (e.player) {
|
||||
const key = e.player.name;
|
||||
let p = players.get(key);
|
||||
if (!p) { p = { team: teamC, shots: 0, goals: 0, xg: 0, matches: 0 }; players.set(key, p); }
|
||||
p.shots++; p.xg += xg; if (e.shot.outcome.name === 'Goal') p.goals++;
|
||||
}
|
||||
// national-team fingerprints
|
||||
if ((trackHome && teamC === homeC) || (trackAway && teamC === awayC)) {
|
||||
const f = fpGet(teamC);
|
||||
f.shots++; f.xg += xg;
|
||||
if (e.shot.outcome.name === 'Goal') f.goals++;
|
||||
const pat = e.play_pattern?.name ?? '';
|
||||
if (pat === 'From Corner' || pat === 'From Free Kick' || pat === 'From Throw In') f.setPieceXg += xg;
|
||||
}
|
||||
}
|
||||
if (isGoal && e.team) goals.push({ minute: e.minute, team: canonicalTeam(e.team.name) });
|
||||
}
|
||||
|
||||
if (trackHome && goals.length) fpGet(homeC).conceded += goals.filter((g) => g.team === awayC).length;
|
||||
if (trackAway && goals.length) fpGet(awayC).conceded += goals.filter((g) => g.team === homeC).length;
|
||||
|
||||
// score-state exposure walk (both teams' perspectives)
|
||||
if (meta) {
|
||||
goals.sort((a, b) => a.minute - b.minute);
|
||||
let h = 0, a = 0, last = 0;
|
||||
const credit = (from: number, to: number, hs: number, as: number) => {
|
||||
const dur = Math.max(0, to - from);
|
||||
if (!dur) return;
|
||||
if (hs === as) state.level.min += dur * 2;
|
||||
else { state.leading.min += dur; state.trailing.min += dur; }
|
||||
};
|
||||
for (const g of goals) {
|
||||
credit(last, g.minute, h, a);
|
||||
const scorerLeading = g.team === meta.home ? h > a : a > h;
|
||||
const scorerTrailing = g.team === meta.home ? h < a : a < h;
|
||||
if (h === a) state.level.goals++;
|
||||
else if (scorerLeading) state.leading.goals++;
|
||||
else if (scorerTrailing) state.trailing.goals++;
|
||||
if (g.team === meta.home) h++; else if (g.team === meta.away) a++;
|
||||
last = g.minute;
|
||||
}
|
||||
credit(last, endMinute, h, a);
|
||||
}
|
||||
|
||||
if (++i % 500 === 0) console.log(` ${i}/${eventFiles.length} files… (${(totalBytes / 1e9).toFixed(1)} GB, ${(totalEvents / 1e6).toFixed(1)}M events)`);
|
||||
}
|
||||
|
||||
// ---- outputs ----
|
||||
const overallRate =
|
||||
(state.trailing.goals + state.level.goals + state.leading.goals) /
|
||||
Math.max(1, state.trailing.min + state.level.min + state.leading.min);
|
||||
const mult = (s: { min: number; goals: number }) => +((s.goals / Math.max(1, s.min)) / overallRate).toFixed(3);
|
||||
const scorestate = {
|
||||
source: 'StatsBomb open data',
|
||||
matches: parsedMatches,
|
||||
multipliers: { trailing: mult(state.trailing), level: mult(state.level), leading: mult(state.leading) },
|
||||
raw: state,
|
||||
};
|
||||
writeFileSync(join(OUT, 'scorestate.json'), JSON.stringify(scorestate));
|
||||
|
||||
const fingerprints = Object.fromEntries(
|
||||
[...fp.entries()]
|
||||
.filter(([, v]) => v.matches >= 3)
|
||||
.map(([t, v]) => [t, {
|
||||
matches: v.matches,
|
||||
shotsPerMatch: +(v.shots / v.matches).toFixed(2),
|
||||
xgPerMatch: +(v.xg / v.matches).toFixed(3),
|
||||
goalsPerMatch: +(v.goals / v.matches).toFixed(2),
|
||||
concededPerMatch: +(v.conceded / v.matches).toFixed(2),
|
||||
setPieceXgShare: v.xg > 0 ? +(v.setPieceXg / v.xg).toFixed(3) : 0,
|
||||
}]),
|
||||
);
|
||||
writeFileSync(join(OUT, 'fingerprints.json'), JSON.stringify({ source: 'StatsBomb open data (men’s internationals)', teams: fingerprints }));
|
||||
|
||||
const lakestats = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
source: 'github.com/statsbomb/open-data',
|
||||
bytes: totalBytes,
|
||||
eventFiles: eventFiles.length,
|
||||
matchesParsed: parsedMatches,
|
||||
events: totalEvents,
|
||||
players: players.size,
|
||||
competitionSeasons: 80,
|
||||
nationalTeamsCovered: Object.keys(fingerprints).length,
|
||||
};
|
||||
writeFileSync(join(OUT, 'lakestats.json'), JSON.stringify(lakestats));
|
||||
|
||||
mkdirSync(join(ROOT, 'data', 'lake'), { recursive: true });
|
||||
writeFileSync(
|
||||
join(ROOT, 'data', 'lake', 'player_profiles.json'),
|
||||
JSON.stringify(Object.fromEntries([...players.entries()].filter(([, p]) => p.shots >= 10))),
|
||||
);
|
||||
|
||||
console.log(`\nlake processed in ${((Date.now() - t0) / 1000).toFixed(0)}s`);
|
||||
console.log(` ${(totalBytes / 1e9).toFixed(2)} GB events · ${(totalEvents / 1e6).toFixed(1)}M events · ${parsedMatches} matches · ${players.size} players`);
|
||||
console.log(` score-state multipliers:`, scorestate.multipliers);
|
||||
console.log(` WC-nation fingerprints: ${Object.keys(fingerprints).length} teams`);
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in New Issue
Block a user