cup26/scripts/processStatsbomb.ts

// One pass over the full StatsBomb open-data lake (~17 GB, 4,200+ matches of
// event data) producing compact, shippable aggregates:
//   public/data/lakestats.json   — what the lake contains (the /data page proof)
//   public/data/scorestate.json  — goal-rate multipliers by score state (in-play v2)
//   public/data/fingerprints.json— per-WC-nation style stats where event data exists
//   data/lake/player_profiles.json — lake-side per-player aggregates (not shipped)
import { readFileSync, readdirSync, writeFileSync, statSync, mkdirSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
import { canonicalTeam, isKnownTeam } from '../src/lib/teams';

const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const LAKE = join(ROOT, 'data', 'lake', 'statsbomb', 'data');
const OUT = join(ROOT, 'public', 'data');

interface MatchMeta { home: string; away: string; competition: string; isMensInternational: boolean }

function buildMatchIndex(): Map<number, MatchMeta> {
  const idx = new Map<number, MatchMeta>();
  const MENS_INTL = new Set(['FIFA World Cup', 'UEFA Euro', 'Copa America', 'African Cup of Nations']);
  for (const comp of readdirSync(join(LAKE, 'matches'))) {
    for (const season of readdirSync(join(LAKE, 'matches', comp))) {
      const arr = JSON.parse(readFileSync(join(LAKE, 'matches', comp, season), 'utf8')) as {
        match_id: number;
        home_team: { home_team_name: string; home_team_gender?: string };
        away_team: { away_team_name: string };
        competition: { competition_name: string };
      }[];
      for (const m of arr) {
        idx.set(m.match_id, {
          home: m.home_team.home_team_name,
          away: m.away_team.away_team_name,
          competition: m.competition.competition_name,
          isMensInternational:
            MENS_INTL.has(m.competition.competition_name) && (m.home_team.home_team_gender ?? 'male') === 'male',
        });
      }
    }
  }
  return idx;
}

interface SbEvent {
  type: { name: string };
  period: number;
  minute: number;
  team?: { name: string };
  player?: { name: string };
  play_pattern?: { name: string };
  shot?: { statsbomb_xg?: number; outcome: { name: string } };
}

function main(): void {
  const t0 = Date.now();
  const index = buildMatchIndex();
  const eventFiles = readdirSync(join(LAKE, 'events'));

  // accumulators
  let totalBytes = 0, totalEvents = 0, parsedMatches = 0;
  const players = new Map<string, { team: string; shots: number; goals: number; xg: number; matches: number }>();
  const fp = new Map<string, { matches: number; shots: number; xg: number; goals: number; setPieceXg: number; conceded: number }>();
  // score-state: minutes + goals while trailing / level / leading
  const state = { trailing: { min: 0, goals: 0 }, level: { min: 0, goals: 0 }, leading: { min: 0, goals: 0 } };

  const fpGet = (t: string) => {
    let v = fp.get(t);
    if (!v) { v = { matches: 0, shots: 0, xg: 0, goals: 0, setPieceXg: 0, conceded: 0 }; fp.set(t, v); }
    return v;
  };

  let i = 0;
  for (const file of eventFiles) {
    const matchId = Number(file.replace('.json', ''));
    const meta = index.get(matchId);
    const path = join(LAKE, 'events', file);
    totalBytes += statSync(path).size;
    let events: SbEvent[];
    try { events = JSON.parse(readFileSync(path, 'utf8')) as SbEvent[]; } catch { continue; }
    totalEvents += events.length;
    parsedMatches++;

    const homeC = meta ? canonicalTeam(meta.home) : '';
    const awayC = meta ? canonicalTeam(meta.away) : '';
    const isIntl = !!meta?.isMensInternational;
    const trackHome = isIntl && isKnownTeam(homeC);
    const trackAway = isIntl && isKnownTeam(awayC);
    if (trackHome) fpGet(homeC).matches++;
    if (trackAway) fpGet(awayC).matches++;

    // goal timeline (regulation+ET, no shootout) for the score-state walk
    const goals: { minute: number; team: string }[] = [];
    let endMinute = 90;

    for (const e of events) {
      if (e.period >= 5) continue;
      if (e.minute > endMinute) endMinute = e.minute;
      const isGoal =
        (e.type.name === 'Shot' && e.shot?.outcome.name === 'Goal') || e.type.name === 'Own Goal For';
      if (e.type.name === 'Shot' && e.shot && e.team) {
        const xg = e.shot.statsbomb_xg ?? 0;
        const teamC = canonicalTeam(e.team.name);
        // player aggregates (whole corpus — club + international)
        if (e.player) {
          const key = e.player.name;
          let p = players.get(key);
          if (!p) { p = { team: teamC, shots: 0, goals: 0, xg: 0, matches: 0 }; players.set(key, p); }
          p.shots++; p.xg += xg; if (e.shot.outcome.name === 'Goal') p.goals++;
        }
        // national-team fingerprints
        if ((trackHome && teamC === homeC) || (trackAway && teamC === awayC)) {
          const f = fpGet(teamC);
          f.shots++; f.xg += xg;
          if (e.shot.outcome.name === 'Goal') f.goals++;
          const pat = e.play_pattern?.name ?? '';
          if (pat === 'From Corner' || pat === 'From Free Kick' || pat === 'From Throw In') f.setPieceXg += xg;
        }
      }
      if (isGoal && e.team) goals.push({ minute: e.minute, team: canonicalTeam(e.team.name) });
    }

    if (trackHome && goals.length) fpGet(homeC).conceded += goals.filter((g) => g.team === awayC).length;
    if (trackAway && goals.length) fpGet(awayC).conceded += goals.filter((g) => g.team === homeC).length;

    // score-state exposure walk (both teams' perspectives)
    if (meta) {
      goals.sort((a, b) => a.minute - b.minute);
      let h = 0, a = 0, last = 0;
      const credit = (from: number, to: number, hs: number, as: number) => {
        const dur = Math.max(0, to - from);
        if (!dur) return;
        if (hs === as) state.level.min += dur * 2;
        else { state.leading.min += dur; state.trailing.min += dur; }
      };
      for (const g of goals) {
        credit(last, g.minute, h, a);
        const scorerLeading = g.team === meta.home ? h > a : a > h;
        const scorerTrailing = g.team === meta.home ? h < a : a < h;
        if (h === a) state.level.goals++;
        else if (scorerLeading) state.leading.goals++;
        else if (scorerTrailing) state.trailing.goals++;
        if (g.team === meta.home) h++; else if (g.team === meta.away) a++;
        last = g.minute;
      }
      credit(last, endMinute, h, a);
    }

    if (++i % 500 === 0) console.log(`  ${i}/${eventFiles.length} files… (${(totalBytes / 1e9).toFixed(1)} GB, ${(totalEvents / 1e6).toFixed(1)}M events)`);
  }

  // ---- outputs ----
  const overallRate =
    (state.trailing.goals + state.level.goals + state.leading.goals) /
    Math.max(1, state.trailing.min + state.level.min + state.leading.min);
  const mult = (s: { min: number; goals: number }) => +((s.goals / Math.max(1, s.min)) / overallRate).toFixed(3);
  const scorestate = {
    source: 'StatsBomb open data',
    matches: parsedMatches,
    multipliers: { trailing: mult(state.trailing), level: mult(state.level), leading: mult(state.leading) },
    raw: state,
  };
  writeFileSync(join(OUT, 'scorestate.json'), JSON.stringify(scorestate));

  const fingerprints = Object.fromEntries(
    [...fp.entries()]
      .filter(([, v]) => v.matches >= 3)
      .map(([t, v]) => [t, {
        matches: v.matches,
        shotsPerMatch: +(v.shots / v.matches).toFixed(2),
        xgPerMatch: +(v.xg / v.matches).toFixed(3),
        goalsPerMatch: +(v.goals / v.matches).toFixed(2),
        concededPerMatch: +(v.conceded / v.matches).toFixed(2),
        setPieceXgShare: v.xg > 0 ? +(v.setPieceXg / v.xg).toFixed(3) : 0,
      }]),
  );
  writeFileSync(join(OUT, 'fingerprints.json'), JSON.stringify({ source: 'StatsBomb open data (men’s internationals)', teams: fingerprints }));

  const lakestats = {
    generatedAt: new Date().toISOString(),
    source: 'github.com/statsbomb/open-data',
    bytes: totalBytes,
    eventFiles: eventFiles.length,
    matchesParsed: parsedMatches,
    events: totalEvents,
    players: players.size,
    competitionSeasons: 80,
    nationalTeamsCovered: Object.keys(fingerprints).length,
  };
  writeFileSync(join(OUT, 'lakestats.json'), JSON.stringify(lakestats));

  mkdirSync(join(ROOT, 'data', 'lake'), { recursive: true });
  writeFileSync(
    join(ROOT, 'data', 'lake', 'player_profiles.json'),
    JSON.stringify(Object.fromEntries([...players.entries()].filter(([, p]) => p.shots >= 10))),
  );

  console.log(`\nlake processed in ${((Date.now() - t0) / 1000).toFixed(0)}s`);
  console.log(`  ${(totalBytes / 1e9).toFixed(2)} GB events · ${(totalEvents / 1e6).toFixed(1)}M events · ${parsedMatches} matches · ${players.size} players`);
  console.log(`  score-state multipliers:`, scorestate.multipliers);
  console.log(`  WC-nation fingerprints: ${Object.keys(fingerprints).length} teams`);
}

main();