// One pass over the full StatsBomb open-data lake (~17 GB, 4,200+ matches of // event data) producing compact, shippable aggregates: // public/data/lakestats.json — what the lake contains (the /data page proof) // public/data/scorestate.json — goal-rate multipliers by score state (in-play v2) // public/data/fingerprints.json— per-WC-nation style stats where event data exists // data/lake/player_profiles.json — lake-side per-player aggregates (not shipped) import { readFileSync, readdirSync, writeFileSync, statSync, mkdirSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { dirname, join } from 'node:path'; import { canonicalTeam, isKnownTeam } from '../src/lib/teams'; const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); const LAKE = join(ROOT, 'data', 'lake', 'statsbomb', 'data'); const OUT = join(ROOT, 'public', 'data'); interface MatchMeta { home: string; away: string; competition: string; isMensInternational: boolean } function buildMatchIndex(): Map { const idx = new Map(); const MENS_INTL = new Set(['FIFA World Cup', 'UEFA Euro', 'Copa America', 'African Cup of Nations']); for (const comp of readdirSync(join(LAKE, 'matches'))) { for (const season of readdirSync(join(LAKE, 'matches', comp))) { const arr = JSON.parse(readFileSync(join(LAKE, 'matches', comp, season), 'utf8')) as { match_id: number; home_team: { home_team_name: string; home_team_gender?: string }; away_team: { away_team_name: string }; competition: { competition_name: string }; }[]; for (const m of arr) { idx.set(m.match_id, { home: m.home_team.home_team_name, away: m.away_team.away_team_name, competition: m.competition.competition_name, isMensInternational: MENS_INTL.has(m.competition.competition_name) && (m.home_team.home_team_gender ?? 'male') === 'male', }); } } } return idx; } interface SbEvent { type: { name: string }; period: number; minute: number; team?: { name: string }; player?: { name: string }; play_pattern?: { name: string }; shot?: { statsbomb_xg?: number; outcome: { name: string } }; } function main(): void { const t0 = Date.now(); const index = buildMatchIndex(); const eventFiles = readdirSync(join(LAKE, 'events')); // accumulators let totalBytes = 0, totalEvents = 0, parsedMatches = 0; const players = new Map(); const fp = new Map(); // score-state: minutes + goals while trailing / level / leading const state = { trailing: { min: 0, goals: 0 }, level: { min: 0, goals: 0 }, leading: { min: 0, goals: 0 } }; const fpGet = (t: string) => { let v = fp.get(t); if (!v) { v = { matches: 0, shots: 0, xg: 0, goals: 0, setPieceXg: 0, conceded: 0 }; fp.set(t, v); } return v; }; let i = 0; for (const file of eventFiles) { const matchId = Number(file.replace('.json', '')); const meta = index.get(matchId); const path = join(LAKE, 'events', file); totalBytes += statSync(path).size; let events: SbEvent[]; try { events = JSON.parse(readFileSync(path, 'utf8')) as SbEvent[]; } catch { continue; } totalEvents += events.length; parsedMatches++; const homeC = meta ? canonicalTeam(meta.home) : ''; const awayC = meta ? canonicalTeam(meta.away) : ''; const isIntl = !!meta?.isMensInternational; const trackHome = isIntl && isKnownTeam(homeC); const trackAway = isIntl && isKnownTeam(awayC); if (trackHome) fpGet(homeC).matches++; if (trackAway) fpGet(awayC).matches++; // goal timeline (regulation+ET, no shootout) for the score-state walk const goals: { minute: number; team: string }[] = []; let endMinute = 90; for (const e of events) { if (e.period >= 5) continue; if (e.minute > endMinute) endMinute = e.minute; const isGoal = (e.type.name === 'Shot' && e.shot?.outcome.name === 'Goal') || e.type.name === 'Own Goal For'; if (e.type.name === 'Shot' && e.shot && e.team) { const xg = e.shot.statsbomb_xg ?? 0; const teamC = canonicalTeam(e.team.name); // player aggregates (whole corpus — club + international) if (e.player) { const key = e.player.name; let p = players.get(key); if (!p) { p = { team: teamC, shots: 0, goals: 0, xg: 0, matches: 0 }; players.set(key, p); } p.shots++; p.xg += xg; if (e.shot.outcome.name === 'Goal') p.goals++; } // national-team fingerprints if ((trackHome && teamC === homeC) || (trackAway && teamC === awayC)) { const f = fpGet(teamC); f.shots++; f.xg += xg; if (e.shot.outcome.name === 'Goal') f.goals++; const pat = e.play_pattern?.name ?? ''; if (pat === 'From Corner' || pat === 'From Free Kick' || pat === 'From Throw In') f.setPieceXg += xg; } } if (isGoal && e.team) goals.push({ minute: e.minute, team: canonicalTeam(e.team.name) }); } if (trackHome && goals.length) fpGet(homeC).conceded += goals.filter((g) => g.team === awayC).length; if (trackAway && goals.length) fpGet(awayC).conceded += goals.filter((g) => g.team === homeC).length; // score-state exposure walk (both teams' perspectives) if (meta) { goals.sort((a, b) => a.minute - b.minute); let h = 0, a = 0, last = 0; const credit = (from: number, to: number, hs: number, as: number) => { const dur = Math.max(0, to - from); if (!dur) return; if (hs === as) state.level.min += dur * 2; else { state.leading.min += dur; state.trailing.min += dur; } }; for (const g of goals) { credit(last, g.minute, h, a); const scorerLeading = g.team === meta.home ? h > a : a > h; const scorerTrailing = g.team === meta.home ? h < a : a < h; if (h === a) state.level.goals++; else if (scorerLeading) state.leading.goals++; else if (scorerTrailing) state.trailing.goals++; if (g.team === meta.home) h++; else if (g.team === meta.away) a++; last = g.minute; } credit(last, endMinute, h, a); } if (++i % 500 === 0) console.log(` ${i}/${eventFiles.length} files… (${(totalBytes / 1e9).toFixed(1)} GB, ${(totalEvents / 1e6).toFixed(1)}M events)`); } // ---- outputs ---- const overallRate = (state.trailing.goals + state.level.goals + state.leading.goals) / Math.max(1, state.trailing.min + state.level.min + state.leading.min); const mult = (s: { min: number; goals: number }) => +((s.goals / Math.max(1, s.min)) / overallRate).toFixed(3); const scorestate = { source: 'StatsBomb open data', matches: parsedMatches, multipliers: { trailing: mult(state.trailing), level: mult(state.level), leading: mult(state.leading) }, raw: state, }; writeFileSync(join(OUT, 'scorestate.json'), JSON.stringify(scorestate)); const fingerprints = Object.fromEntries( [...fp.entries()] .filter(([, v]) => v.matches >= 3) .map(([t, v]) => [t, { matches: v.matches, shotsPerMatch: +(v.shots / v.matches).toFixed(2), xgPerMatch: +(v.xg / v.matches).toFixed(3), goalsPerMatch: +(v.goals / v.matches).toFixed(2), concededPerMatch: +(v.conceded / v.matches).toFixed(2), setPieceXgShare: v.xg > 0 ? +(v.setPieceXg / v.xg).toFixed(3) : 0, }]), ); writeFileSync(join(OUT, 'fingerprints.json'), JSON.stringify({ source: 'StatsBomb open data (men’s internationals)', teams: fingerprints })); const lakestats = { generatedAt: new Date().toISOString(), source: 'github.com/statsbomb/open-data', bytes: totalBytes, eventFiles: eventFiles.length, matchesParsed: parsedMatches, events: totalEvents, players: players.size, competitionSeasons: 80, nationalTeamsCovered: Object.keys(fingerprints).length, }; writeFileSync(join(OUT, 'lakestats.json'), JSON.stringify(lakestats)); mkdirSync(join(ROOT, 'data', 'lake'), { recursive: true }); writeFileSync( join(ROOT, 'data', 'lake', 'player_profiles.json'), JSON.stringify(Object.fromEntries([...players.entries()].filter(([, p]) => p.shots >= 10))), ); console.log(`\nlake processed in ${((Date.now() - t0) / 1000).toFixed(0)}s`); console.log(` ${(totalBytes / 1e9).toFixed(2)} GB events · ${(totalEvents / 1e6).toFixed(1)}M events · ${parsedMatches} matches · ${players.size} players`); console.log(` score-state multipliers:`, scorestate.multipliers); console.log(` WC-nation fingerprints: ${Object.keys(fingerprints).length} teams`); } main();