cup26/scripts/backfillFotmob.ts

// Harvest FotMob matchDetails (team + per-shot xG) for recent international
// matches into the local lake — the training data for the xG-informed Team-DC
// experiment (v2 M1). Runs on the residential machine, politely (≥2.5s/req),
// resumable by date. Standalone on purpose: plain fetch, no app imports
// beyond the team canonicalizer.
//   bun scripts/backfillFotmob.ts [fromYYYY-MM-DD] [toYYYY-MM-DD]
import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';
import { canonicalTeam } from '../src/lib/teams';
import ratings from '../public/data/ratings.json';

const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const LAKE = join(ROOT, 'data', 'lake', 'fotmob');
const OUT = join(LAKE, 'internationals.jsonl');
const PROGRESS = join(LAKE, 'progress.json');
const UA = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36' };
const DELAY_MS = 2_600;

const FROM = process.argv[2] ?? '2022-07-01';
const TO = process.argv[3] ?? '2026-06-10';

const KNOWN = new Set(Object.keys((ratings as { ratings: Record<string, number> }).ratings));
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

interface DayMatches {
  leagues?: { name?: string; ccode?: string; matches?: { id?: number | string; home?: { name?: string }; away?: { name?: string }; status?: { finished?: boolean } }[] }[];
}

async function getJson<T>(url: string): Promise<T | null> {
  for (let attempt = 0; attempt < 3; attempt++) {
    try {
      const res = await fetch(url, { headers: UA });
      if (res.status === 429 || res.status >= 500) {
        await sleep(30_000 * (attempt + 1));
        continue;
      }
      if (!res.ok) return null;
      return (await res.json()) as T;
    } catch {
      await sleep(10_000);
    }
  }
  return null;
}

function* days(from: string, to: string): Generator<string> {
  const d = new Date(`${from}T00:00:00Z`);
  const end = new Date(`${to}T00:00:00Z`).getTime();
  while (d.getTime() <= end) {
    yield d.toISOString().slice(0, 10);
    d.setUTCDate(d.getUTCDate() + 1);
  }
}

async function main(): Promise<void> {
  mkdirSync(LAKE, { recursive: true });
  const done: Record<string, number> = existsSync(PROGRESS) ? JSON.parse(readFileSync(PROGRESS, 'utf8')) : {};
  let harvested = 0;

  for (const day of days(FROM, TO)) {
    if (done[day] != null) continue;
    const ymd = day.replaceAll('-', '');
    const dayData = await getJson<DayMatches>(`https://www.fotmob.com/api/data/matches?date=${ymd}`);
    await sleep(DELAY_MS);
    let dayCount = 0;
    for (const lg of dayData?.leagues ?? []) {
      for (const m of lg.matches ?? []) {
        if (!m.id || !m.status?.finished || !m.home?.name || !m.away?.name) continue;
        const home = canonicalTeam(m.home.name);
        const away = canonicalTeam(m.away.name);
        // internationals only: both sides must be known national teams
        if (!KNOWN.has(home) || !KNOWN.has(away)) continue;
        const det = await getJson<{
          general?: { matchTimeUTC?: string };
          content?: {
            shotmap?: { shots?: { teamId?: number; min?: number; x?: number; y?: number; expectedGoals?: number; expectedGoalsOnTarget?: number; eventType?: string; situation?: string }[] };
            stats?: unknown;
          };
          header?: { teams?: { name?: string; score?: number }[] };
        }>(`https://www.fotmob.com/api/data/matchDetails?matchId=${m.id}`);
        await sleep(DELAY_MS);
        if (!det) continue;
        const shots = det.content?.shotmap?.shots ?? [];
        const teams = det.header?.teams ?? [];
        const xg = (idx: number): number | null => {
          // sum shot xG per side via teamId order in header is unreliable across
          // payloads — derive from shots when both team scores exist
          void idx;
          return null;
        };
        void xg;
        appendFileSync(
          OUT,
          JSON.stringify({
            date: day,
            league: lg.name ?? '',
            matchId: String(m.id),
            home,
            away,
            homeScore: teams[0]?.score ?? null,
            awayScore: teams[1]?.score ?? null,
            shots: shots.map((s) => ({
              teamId: s.teamId ?? null, min: s.min ?? null,
              xg: s.expectedGoals ?? null, xgot: s.expectedGoalsOnTarget ?? null,
              type: s.eventType ?? null, situation: s.situation ?? null,
            })),
          }) + '\n',
        );
        dayCount++;
        harvested++;
      }
    }
    done[day] = dayCount;
    writeFileSync(PROGRESS, JSON.stringify(done));
    if (dayCount) console.log(`${day}: ${dayCount} internationals (total ${harvested})`);
  }
  console.log(`backfill complete → ${OUT} (${harvested} new matches this run)`);
}

void main();