diff --git a/public/data/venues.json b/public/data/venues.json new file mode 100644 index 0000000..99f78ff --- /dev/null +++ b/public/data/venues.json @@ -0,0 +1,101 @@ +{ + "generatedAt": "2026-06-12T07:16:35.623Z", + "venues": { + "Atlanta": { + "lat": 33.749, + "lon": -84.388, + "elevation": 320, + "tz": "America/New_York" + }, + "Boston (Foxborough)": { + "lat": 42.0654, + "lon": -71.2478, + "elevation": 88, + "tz": "America/New_York" + }, + "Dallas (Arlington)": { + "lat": 32.7357, + "lon": -97.1081, + "elevation": 184, + "tz": "America/Chicago" + }, + "Guadalajara (Zapopan)": { + "lat": 20.7211, + "lon": -103.3874, + "elevation": 1565, + "tz": "America/Mexico_City" + }, + "Houston": { + "lat": 29.7633, + "lon": -95.3633, + "elevation": 12, + "tz": "America/Chicago" + }, + "Kansas City": { + "lat": 39.0997, + "lon": -94.5786, + "elevation": 274, + "tz": "America/Chicago" + }, + "Los Angeles (Inglewood)": { + "lat": 33.9617, + "lon": -118.3531, + "elevation": 40, + "tz": "America/Los_Angeles" + }, + "Mexico City": { + "lat": 19.4285, + "lon": -99.1277, + "elevation": 2240, + "tz": "America/Mexico_City" + }, + "Miami (Miami Gardens)": { + "lat": 25.942, + "lon": -80.2456, + "elevation": 2, + "tz": "America/New_York" + }, + "Monterrey (Guadalupe)": { + "lat": 25.6768, + "lon": -100.2565, + "elevation": 497, + "tz": "America/Monterrey" + }, + "New York/New Jersey (East Rutherford)": { + "lat": 40.834, + "lon": -74.0971, + "elevation": 24, + "tz": "America/New_York" + }, + "Philadelphia": { + "lat": 39.9524, + "lon": -75.1636, + "elevation": 12, + "tz": "America/New_York" + }, + "San Francisco Bay Area (Santa Clara)": { + "lat": 37.3541, + "lon": -121.9552, + "elevation": 23, + "tz": "America/Los_Angeles" + }, + "Seattle": { + "lat": 47.6062, + "lon": -122.3321, + "elevation": 56, + "tz": "America/Los_Angeles" + }, + "Toronto": { + "lat": 43.7064, + "lon": -79.3986, + "elevation": 161, + "tz": "America/Toronto" + }, + "Vancouver": { + "lat": 49.2827, + "lon": -123.1207, + "elevation": 70, + "tz": "America/Vancouver" + } + } +} \ No newline at end of file diff --git a/scripts/buildVenues.ts b/scripts/buildVenues.ts new file mode 100644 index 0000000..6fc31da --- /dev/null +++ b/scripts/buildVenues.ts @@ -0,0 +1,69 @@ +// Geocode the 16 World Cup 2026 venues once (Open-Meteo geocoding, keyless) +// → public/data/venues.json with lat/lon/elevation/timezone per fixture-venue +// string. Elevation matters here: Mexico City sits at ~2,240m. +import { writeFileSync, mkdirSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const OUT_DIR = join(dirname(fileURLToPath(import.meta.url)), '..', 'public', 'data'); + +/** fixture venue string → geocoding query + expected country (disambiguator) */ +const VENUES: Record = { + 'Atlanta': { q: 'Atlanta', cc: 'US' }, + 'Boston (Foxborough)': { q: 'Foxborough', cc: 'US' }, + 'Dallas (Arlington)': { q: 'Arlington', cc: 'US' }, + 'Guadalajara (Zapopan)': { q: 'Zapopan', cc: 'MX' }, + 'Houston': { q: 'Houston', cc: 'US' }, + 'Kansas City': { q: 'Kansas City', cc: 'US' }, + 'Los Angeles (Inglewood)': { q: 'Inglewood', cc: 'US' }, + 'Mexico City': { q: 'Mexico City', cc: 'MX' }, + 'Miami (Miami Gardens)': { q: 'Miami Gardens', cc: 'US' }, + 'Monterrey (Guadalupe)': { q: 'Guadalupe', cc: 'MX' }, + 'New York/New Jersey (East Rutherford)': { q: 'East Rutherford', cc: 'US' }, + 'Philadelphia': { q: 'Philadelphia', cc: 'US' }, + 'San Francisco Bay Area (Santa Clara)': { q: 'Santa Clara', cc: 'US' }, + 'Seattle': { q: 'Seattle', cc: 'US' }, + 'Toronto': { q: 'Toronto', cc: 'CA' }, + 'Vancouver': { q: 'Vancouver', cc: 'CA' }, +}; + +/** Geocoder misses (e.g. "Vancouver" resolving to a Vancouver-Island point): + * pin the known city coordinates explicitly. */ +const OVERRIDES: Record = { + Vancouver: { lat: 49.2827, lon: -123.1207, elevation: 70, tz: 'America/Vancouver' }, +}; + +interface GeoResult { + results?: { latitude: number; longitude: number; elevation?: number; timezone?: string; name: string; country_code?: string }[]; +} + +async function main(): Promise { + const out: Record = {}; + for (const [venue, { q, cc }] of Object.entries(VENUES)) { + const override = OVERRIDES[venue]; + if (override) { + out[venue] = override; + console.log(venue, '→ (override)', override); + continue; + } + const url = `https://geocoding-api.open-meteo.com/v1/search?name=${encodeURIComponent(q)}&count=10&language=en&format=json`; + const res = await fetch(url); + const d = (await res.json()) as GeoResult & { results?: { population?: number }[] }; + const candidates = (d.results ?? []).filter((x) => x.country_code === cc); + const r = candidates.sort((a, b) => ((b as { population?: number }).population ?? 0) - ((a as { population?: number }).population ?? 0))[0]; + if (!r) throw new Error(`no geocoding result for ${venue} (${q}, ${cc})`); + out[venue] = { + lat: +r.latitude.toFixed(4), + lon: +r.longitude.toFixed(4), + elevation: Math.round(r.elevation ?? 0), + tz: r.timezone ?? 'UTC', + }; + console.log(venue, '→', out[venue]); + await new Promise((resolve) => setTimeout(resolve, 250)); + } + mkdirSync(OUT_DIR, { recursive: true }); + writeFileSync(join(OUT_DIR, 'venues.json'), JSON.stringify({ generatedAt: new Date().toISOString(), venues: out }, null, 1)); + console.log(`venues → public/data/venues.json (${Object.keys(out).length})`); +} + +void main(); diff --git a/server/src/db/db.ts b/server/src/db/db.ts index 274eb21..731f25b 100644 --- a/server/src/db/db.ts +++ b/server/src/db/db.ts @@ -93,6 +93,33 @@ CREATE TABLE IF NOT EXISTS prediction_snapshots ( market_json TEXT ); +CREATE TABLE IF NOT EXISTS fixture_results ( + fixture_num INTEGER PRIMARY KEY, + home_score INTEGER, + away_score INTEGER, + status TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS match_provider ( + fixture_num INTEGER NOT NULL, + provider TEXT NOT NULL, + json TEXT NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (fixture_num, provider) +); + +CREATE TABLE IF NOT EXISTS scorer_props ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + fixture_num INTEGER NOT NULL, + provider TEXT NOT NULL, + market TEXT NOT NULL, + athlete TEXT NOT NULL, + odds REAL, + captured_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_scorer_props ON scorer_props(fixture_num, market, athlete, captured_at); + CREATE TABLE IF NOT EXISTS push_subs ( endpoint TEXT PRIMARY KEY, json TEXT NOT NULL, @@ -154,6 +181,53 @@ export function cacheSet(key: string, value: string, ttl: number): void { .run(key, value, Date.now(), ttl); } +// ---- fixture results survive restarts (the live snapshot is in-memory and +// ESPN's scoreboard window moves on — without this, yesterday's results +// would vanish from standings/model after a redeploy) ---- +export function saveFixtureResult(num: number, homeScore: number | null, awayScore: number | null, status: string): void { + db() + .prepare('INSERT OR REPLACE INTO fixture_results (fixture_num, home_score, away_score, status, updated_at) VALUES (?, ?, ?, ?, ?)') + .run(num, homeScore, awayScore, status, Date.now()); +} + +export function allFixtureResults(): { fixture_num: number; home_score: number | null; away_score: number | null; status: string }[] { + return db() + .prepare('SELECT fixture_num, home_score, away_score, status FROM fixture_results') + .all() as unknown as { fixture_num: number; home_score: number | null; away_score: number | null; status: string }[]; +} + +// ---- per-provider rich match payloads (fotmob / fifa / weather) ---- +export function setMatchProvider(fixtureNum: number, provider: string, json: unknown): void { + db() + .prepare('INSERT OR REPLACE INTO match_provider (fixture_num, provider, json, updated_at) VALUES (?, ?, ?, ?)') + .run(fixtureNum, provider, JSON.stringify(json), Date.now()); +} + +export function getMatchProvider(fixtureNum: number, provider: string): { data: T; updatedAt: number } | null { + const row = db() + .prepare('SELECT json, updated_at FROM match_provider WHERE fixture_num = ? AND provider = ?') + .get(fixtureNum, provider) as { json: string; updated_at: number } | undefined; + return row ? { data: JSON.parse(row.json) as T, updatedAt: row.updated_at } : null; +} + +/** Insert a scorer prop only when the line moved (mirrors recordOdds). */ +export function recordScorerProp(p: { fixture_num: number; provider: string; market: string; athlete: string; odds: number | null }): boolean { + const last = db() + .prepare('SELECT odds FROM scorer_props WHERE fixture_num = ? AND provider = ? AND market = ? AND athlete = ? ORDER BY captured_at DESC LIMIT 1') + .get(p.fixture_num, p.provider, p.market, p.athlete) as { odds: number | null } | undefined; + if (last && last.odds === p.odds) return false; + db() + .prepare('INSERT INTO scorer_props (fixture_num, provider, market, athlete, odds, captured_at) VALUES (?, ?, ?, ?, ?, ?)') + .run(p.fixture_num, p.provider, p.market, p.athlete, p.odds, Date.now()); + return true; +} + +export function scorerPropsFor(fixtureNum: number): { provider: string; market: string; athlete: string; odds: number | null; captured_at: number }[] { + return db() + .prepare('SELECT provider, market, athlete, odds, captured_at FROM scorer_props WHERE fixture_num = ? ORDER BY captured_at') + .all(fixtureNum) as unknown as { provider: string; market: string; athlete: string; odds: number | null; captured_at: number }[]; +} + // ---- Web Push subscriptions (goal/kickoff notifications, opt-in per team) ---- export interface PushSub { endpoint: string; diff --git a/server/src/index.ts b/server/src/index.ts index 7d45122..8db86b1 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -10,7 +10,7 @@ import { startScheduler } from './ingest/scheduler'; import { ModelEngine } from './model'; import { buildPreview, buildTeamProfile } from './preview'; import { buildScoreboard, snapshotCheck } from './scoreboard'; -import { db, healthAll, oddsFor, latestOddsAll, dbCounts, inplayHistory, recordInplay, getMatchExt } from './db/db'; +import { db, healthAll, oddsFor, latestOddsAll, dbCounts, inplayHistory, recordInplay, getMatchExt, allFixtureResults, saveFixtureResult } from './db/db'; import { inPlayProbs } from '../../src/lib/model/inplay'; import { PushNotifier, pushEnabled, subscribe as pushSubscribe, unsubscribe as pushUnsubscribe, vapidPublicKey } from './push'; import type { EspnSummary } from './ingest/espnNormalize'; @@ -37,6 +37,14 @@ export function buildServer() { db(); // open + migrate SQLite before anything reads it const state = new TournamentState(STATIC_DIR); + // Restore persisted results over the seed — the in-memory snapshot forgets + // them on restart once ESPN's scoreboard window has moved past a match day. + let restored = 0; + for (const r of allFixtureResults()) { + if (r.status === 'scheduled') continue; + if (state.applyScore(r.fixture_num, r.home_score, r.away_score, r.status as MatchStatus, null, 'seed')) restored++; + } + if (restored) console.log(`[state] restored ${restored} fixture result(s) from the DB`); const model = new ModelEngine(STATIC_DIR); model.refitTeamDc(state.allFixtures()); // current Team-DC from day one (covers restarts mid-tournament) model.recompute(state.allFixtures()); @@ -184,6 +192,7 @@ export function buildServer() { 'football-data', ); if (!f) return reply.code(404).send({ error: 'no such fixture' }); + saveFixtureResult(f.num, f.homeScore, f.awayScore, f.status); snapshotCheck(state, model); recordInplayTick(); broadcast(); diff --git a/server/src/ingest/espnProps.ts b/server/src/ingest/espnProps.ts new file mode 100644 index 0000000..8b0556a --- /dev/null +++ b/server/src/ingest/espnProps.ts @@ -0,0 +1,78 @@ +// ESPN core API prop bets: per-athlete goalscorer odds (DraftKings, provider +// 100) — First Goalscorer / Last Goalscorer / To Score 2+ Goals, plus anytime +// variants. Line history is unrecoverable, so capture beats everything; the +// athlete $refs resolve to names lazily through a long-TTL cache. +import { cachedFetch } from './fetcher'; + +const CORE = 'https://sports.core.api.espn.com/v2/sports/soccer/leagues/fifa.world'; +const SOURCE = 'espn-odds'; +const PROVIDER = 100; // DraftKings on the core API + +export interface ScorerProp { + market: string; + athleteName: string; + american: number | null; +} + +interface PropPage { + pageCount?: number; + items?: { + type?: { name?: string; text?: string }; + athlete?: { $ref?: string }; + current?: { odds?: { american?: string | number } | string; american?: string }; + open?: unknown; + }[]; +} + +const WANTED = /goalscorer|to score/i; + +function americanOf(item: PropPage['items'] extends (infer T)[] | undefined ? T : never): number | null { + const cur = item.current as { american?: string | number; odds?: { american?: string | number } } | undefined; + const raw = cur?.american ?? cur?.odds?.american; + if (raw == null) return null; + const n = Number(String(raw).replace('+', '')); + return Number.isFinite(n) ? n : null; +} + +/** Resolve an athlete $ref to a display name (cached ~30 days). */ +async function athleteName(ref: string): Promise { + const m = /athletes\/(\d+)/.exec(ref); + if (!m) return null; + try { + const a = await cachedFetch<{ displayName?: string; fullName?: string }>( + SOURCE, + `espn:athlete:${m[1]}`, + ref.replace(/^http:/, 'https:'), + { ttl: 30 * 24 * 60 * 60_000, minInterval: 1_200 }, + ); + return a.displayName ?? a.fullName ?? null; + } catch { + return null; + } +} + +/** All scorer props for an event. Name resolution is deadline-bounded per + * call: cold caches fill across successive sweeps (each athlete lookup is + * cached ~30 days), warm caches resolve everything instantly. */ +export async function fetchScorerProps(eventId: string, budgetMs = 45_000): Promise { + const out: ScorerProp[] = []; + const deadline = Date.now() + budgetMs; + for (let page = 1; page <= 10; page++) { + const url = `${CORE}/events/${eventId}/competitions/${eventId}/odds/${PROVIDER}/propBets?limit=100&page=${page}`; + const d = await cachedFetch(SOURCE, `espn:props:${eventId}:${page}`, url, { + ttl: 60 * 60_000, + minInterval: 1_500, + }); + for (const item of d.items ?? []) { + const market = item.type?.text ?? item.type?.name ?? ''; + if (!WANTED.test(market)) continue; + const ref = item.athlete?.$ref; + if (!ref || Date.now() > deadline) continue; + const name = await athleteName(ref); + if (!name) continue; + out.push({ market, athleteName: name, american: americanOf(item) }); + } + if ((d.pageCount ?? 1) <= page) break; + } + return out; +} diff --git a/server/src/ingest/fifa.ts b/server/src/ingest/fifa.ts new file mode 100644 index 0000000..cb5874c --- /dev/null +++ b/server/src/ingest/fifa.ts @@ -0,0 +1,83 @@ +// FIFA's official v3 API (api.fifa.com) — open, no auth, works from the VPS. +// Carries what no other source has officially: confirmed lineups with stable +// player IDs + headshots, typed XY event timelines, attendance, tactics, +// officials. World Cup 2026: IdCompetition=17, IdSeason=285023. +import { cachedFetch } from './fetcher'; +import { canonicalTeam } from '../../../src/lib/teams'; + +const BASE = 'https://api.fifa.com/api/v3'; +const SOURCE = 'fifa'; +export const FIFA_COMPETITION = '17'; +export const FIFA_SEASON = '285023'; + +const locDesc = (v: unknown): string => { + const a = v as { Description?: string }[] | undefined; + return a?.[0]?.Description ?? ''; +}; + +export interface FifaMatchRef { + idMatch: string; + idStage: string; + home: string; + away: string; + date: string; + attendance: number | null; + officials: unknown[]; + stadium: string; +} + +interface RawCalendar { + Results?: { + IdMatch?: string; + IdStage?: string; + Date?: string; + Attendance?: string | number | null; + Officials?: unknown[]; + Stadium?: { Name?: unknown }; + Home?: { TeamName?: unknown }; + Away?: { TeamName?: unknown }; + }[]; +} + +/** Full tournament calendar window (id map + attendance/officials source). */ +export async function fetchFifaCalendar(fromIso: string, toIso: string): Promise { + const url = `${BASE}/calendar/matches?idCompetition=${FIFA_COMPETITION}&idSeason=${FIFA_SEASON}&from=${encodeURIComponent(fromIso)}&to=${encodeURIComponent(toIso)}&count=120&language=en`; + const raw = await cachedFetch(SOURCE, `fifa:cal:${fromIso.slice(0, 10)}:${toIso.slice(0, 10)}`, url, { + ttl: 10 * 60_000, + minInterval: 2_000, + }); + const out: FifaMatchRef[] = []; + for (const m of raw.Results ?? []) { + const home = canonicalTeam(locDesc(m.Home?.TeamName)); + const away = canonicalTeam(locDesc(m.Away?.TeamName)); + if (!m.IdMatch || !m.IdStage || !home || !away) continue; + out.push({ + idMatch: m.IdMatch, + idStage: m.IdStage, + home, + away, + date: m.Date ?? '', + attendance: m.Attendance != null && m.Attendance !== '' ? Number(m.Attendance) : null, + officials: m.Officials ?? [], + stadium: locDesc(m.Stadium?.Name), + }); + } + return out; +} + +/** Rich live payload: confirmed lineups (IdPlayer, shirt numbers, positions), + * tactics, match clock. Stored whole — it's the official record. */ +export async function fetchFifaLive(idStage: string, idMatch: string, live = false): Promise { + return cachedFetch(SOURCE, `fifa:live:${idMatch}`, `${BASE}/live/football/${FIFA_COMPETITION}/${FIFA_SEASON}/${idStage}/${idMatch}?language=en`, { + ttl: live ? 60_000 : 60 * 60_000, + minInterval: 2_000, + }); +} + +/** Typed event-by-event timeline with pitch coordinates. */ +export async function fetchFifaTimeline(idStage: string, idMatch: string, live = false): Promise { + return cachedFetch(SOURCE, `fifa:tl:${idMatch}`, `${BASE}/timelines/${FIFA_COMPETITION}/${FIFA_SEASON}/${idStage}/${idMatch}?language=en`, { + ttl: live ? 60_000 : 60 * 60_000, + minInterval: 2_000, + }); +} diff --git a/server/src/ingest/fotmob.ts b/server/src/ingest/fotmob.ts new file mode 100644 index 0000000..fad0bb4 --- /dev/null +++ b/server/src/ingest/fotmob.ts @@ -0,0 +1,107 @@ +// FotMob (unofficial, www.fotmob.com/api/data) — as of June 2026 it serves +// full match details from the VPS with no signed header: per-shot xG/xGOT +// with pitch coordinates, team xG splits, player ratings, player of the +// match, momentum, attacking zones, lineups. Unofficial means it can lock +// again any day — so capture early, store generously, degrade gracefully. +import { cachedFetch } from './fetcher'; +import { canonicalTeam } from '../../../src/lib/teams'; + +const BASE = 'https://www.fotmob.com/api/data'; +const SOURCE = 'fotmob'; +const WC_LEAGUE_ID = 77; + +export interface FotmobMatchRef { + matchId: string; + home: string; + away: string; + utcTime: string; +} + +interface RawMatches { + leagues?: { + primaryId?: number; + matches?: { + id?: number | string; + home?: { name?: string }; + away?: { name?: string }; + status?: { utcTime?: string }; + }[]; + }[]; +} + +/** World Cup matches on a given UTC date (id map source). */ +export async function fetchFotmobMatches(dateYYYYMMDD: string): Promise { + const raw = await cachedFetch(SOURCE, `fotmob:matches:${dateYYYYMMDD}`, `${BASE}/matches?date=${dateYYYYMMDD}`, { + ttl: 10 * 60_000, + minInterval: 2_500, + }); + const out: FotmobMatchRef[] = []; + for (const lg of raw.leagues ?? []) { + if (lg.primaryId !== WC_LEAGUE_ID) continue; + for (const m of lg.matches ?? []) { + if (!m.id || !m.home?.name || !m.away?.name) continue; + out.push({ + matchId: String(m.id), + home: canonicalTeam(m.home.name), + away: canonicalTeam(m.away.name), + utcTime: m.status?.utcTime ?? '', + }); + } + } + return out; +} + +/** Trimmed matchDetails: everything analytic, minus the bulky redundant tabs + * (liveticker/buzz/h2h/table/seo) we already cover via other sources. */ +export interface FotmobDetails { + v: 1; + fetchedAt: number; + matchId: string; + finished: boolean; + shots: unknown[]; + stats: unknown; + matchFacts: unknown; + lineup: unknown; + momentum: unknown; + attackingZones: unknown; + weather: unknown; +} + +interface RawDetails { + general?: { matchId?: string; finished?: boolean }; + content?: { + shotmap?: { shots?: unknown[] }; + stats?: unknown; + matchFacts?: Record; + lineup?: unknown; + momentum?: unknown; + attackingZones?: unknown; + weather?: unknown; + }; +} + +export async function fetchFotmobDetails(matchId: string, live = false): Promise { + const raw = await cachedFetch(SOURCE, `fotmob:details:${matchId}`, `${BASE}/matchDetails?matchId=${matchId}`, { + ttl: live ? 60_000 : 60 * 60_000, + minInterval: 2_500, + }); + const c = raw.content ?? {}; + const mf = { ...(c.matchFacts ?? {}) } as Record; + delete mf.QAData; // SEO filler + delete mf.postReview; + delete mf.preReview; + delete mf.poll; + return { + v: 1, + fetchedAt: Date.now(), + matchId: raw.general?.matchId ?? matchId, + finished: raw.general?.finished ?? false, + shots: c.shotmap?.shots ?? [], + stats: c.stats ?? null, + matchFacts: mf, + lineup: c.lineup ?? null, + momentum: c.momentum ?? null, + attackingZones: c.attackingZones ?? null, + weather: c.weather ?? null, + }; +} diff --git a/server/src/ingest/scheduler.ts b/server/src/ingest/scheduler.ts index 974bd1e..fbd644a 100644 --- a/server/src/ingest/scheduler.ts +++ b/server/src/ingest/scheduler.ts @@ -4,7 +4,11 @@ import { fetchEspnScoreboard, fetchEspnSummary } from './espn'; import { fetchEspnOdds } from './espnOdds'; import { fetchFootballData } from '../footballData'; import { fetchSofascore } from '../sofascore'; -import { setSourceMap, getSourceMap, getMatchExt, setMatchExt, recordOdds, logIngest, prune, backupDb } from '../db/db'; +import { setSourceMap, getSourceMap, getMatchExt, setMatchExt, setMatchProvider, getMatchProvider, recordOdds, recordScorerProp, logIngest, prune, backupDb, saveFixtureResult } from '../db/db'; +import { fetchFotmobDetails, fetchFotmobMatches } from './fotmob'; +import { fetchFifaCalendar, fetchFifaLive, fetchFifaTimeline } from './fifa'; +import { fetchMatchWeather } from './weather'; +import { fetchScorerProps } from './espnProps'; import type { EspnSummary } from './espnNormalize'; import type { Fixture } from '../../../src/lib/types'; @@ -92,7 +96,11 @@ export function startScheduler( } if (matches.length) { const changed = state.mergeLive(matches, source); - if (changed.length) { console.log(`[ingest] ${changed.length} fixture(s) updated via ${source}`); onChange(); } + if (changed.length) { + for (const f of changed) saveFixtureResult(f.num, f.homeScore, f.awayScore, f.status); + console.log(`[ingest] ${changed.length} fixture(s) updated via ${source}`); + onChange(); + } } onTick?.(); // e.g. freeze pre-kickoff forecast snapshots @@ -141,9 +149,121 @@ export function startScheduler( } catch (e) { logIngest('espn', 'summary', false, Date.now() - t0, e instanceof Error ? e.message : String(e)); } + // kickoff-hour weather for fixtures inside the forecast horizon + if (f.status === 'scheduled' && new Date(f.kickoff).getTime() - now < 2 * 24 * 60 * 60 * 1000) { + try { + const w = await fetchMatchWeather(f.venue, f.kickoff); + if (w) setMatchProvider(f.num, 'weather', w); + } catch { /* health-tracked */ } + } } }; + // ---- FotMob: xG / shotmaps / ratings — capture-first (unofficial source, + // could lock again any day). Own gentle cadence: 90s while live, 30m idle. + const FOTMOB_SETTLE_MS = 3 * 60 * 60 * 1000; // refetch finished until FT+3h (settled ratings) + const fotmobTick = async (): Promise => { + const now = Date.now(); + // map: sweep dates (now ±3d) that still contain unmapped fixtures + const dates = new Set(); + for (const f of state.allFixtures()) { + const k = new Date(f.kickoff).getTime(); + if (Math.abs(k - now) > 3 * 24 * 60 * 60 * 1000) continue; + if (!getSourceMap(f.num, 'fotmob')) dates.add(dateUTC(f.kickoff)); + } + for (const date of dates) { + if (stopped) return; + const t0 = Date.now(); + try { + const refs = await fetchFotmobMatches(date); + for (const r of refs) { + const num = state.matchFixtureNum(r.home, r.away, 'group', r.utcTime || `${date.slice(0, 4)}-${date.slice(4, 6)}-${date.slice(6, 8)}T12:00:00Z`); + if (num && !getSourceMap(num, 'fotmob')) setSourceMap(num, 'fotmob', r.matchId); + } + logIngest('fotmob', 'map', true, Date.now() - t0, `${date}: ${refs.length} matches`); + } catch (e) { + logIngest('fotmob', 'map', false, Date.now() - t0, e instanceof Error ? e.message : String(e)); + } + } + // details: live fixtures at the live TTL; finished ones until settled + for (const f of state.allFixtures()) { + if (stopped) return; + const id = getSourceMap(f.num, 'fotmob'); + if (!id) continue; + const live = f.status === 'live'; + if (!live && f.status !== 'finished') continue; + if (f.status === 'finished') { + const row = getMatchProvider(f.num, 'fotmob'); + const settled = row && row.updatedAt > new Date(f.kickoff).getTime() + FOTMOB_SETTLE_MS; + if (settled) continue; + } + const t0 = Date.now(); + try { + setMatchProvider(f.num, 'fotmob', await fetchFotmobDetails(id, live)); + logIngest('fotmob', live ? 'live' : 'final', true, Date.now() - t0, `M${f.num}`); + } catch (e) { + logIngest('fotmob', 'details', false, Date.now() - t0, e instanceof Error ? e.message : String(e)); + } + } + }; + + let fotmobTimer: ReturnType | undefined; + const scheduleFotmob = (): void => { + if (stopped) return; + fotmobTimer = setTimeout(async () => { await fotmobTick().catch(() => {}); scheduleFotmob(); }, state.hasLive() ? 90_000 : 30 * 60_000); + }; + + // ---- FIFA official: confirmed lineups, XY timelines, attendance/officials. + const fifaTick = async (): Promise => { + const now = Date.now(); + const DAY_MS = 24 * 60 * 60 * 1000; + // calendar sweep (one cached call covers ±3d): id map + official match info + const t0 = Date.now(); + try { + const from = new Date(now - 3 * DAY_MS).toISOString().slice(0, 10); + const to = new Date(now + 3 * DAY_MS).toISOString().slice(0, 10); + const refs = await fetchFifaCalendar(`${from}T00:00:00Z`, `${to}T00:00:00Z`); + for (const r of refs) { + const num = state.matchFixtureNum(r.home, r.away, 'group', r.date); + if (!num) continue; + if (!getSourceMap(num, 'fifa')) setSourceMap(num, 'fifa', `${r.idStage}|${r.idMatch}`); + setMatchProvider(num, 'fifa-info', { attendance: r.attendance, officials: r.officials, stadium: r.stadium, date: r.date }); + } + logIngest('fifa', 'calendar', true, Date.now() - t0, `${refs.length} matches`); + } catch (e) { + logIngest('fifa', 'calendar', false, Date.now() - t0, e instanceof Error ? e.message : String(e)); + } + // lineups + timeline: live fixtures hot, finished until settled + for (const f of state.allFixtures()) { + if (stopped) return; + const mapId = getSourceMap(f.num, 'fifa'); + if (!mapId) continue; + const [idStage, idMatch] = mapId.split('|'); + if (!idStage || !idMatch) continue; + const live = f.status === 'live'; + const imminent = f.status === 'scheduled' && new Date(f.kickoff).getTime() - now < 90 * 60_000 && new Date(f.kickoff).getTime() > now; + if (!live && !imminent && f.status !== 'finished') continue; + if (f.status === 'finished') { + const row = getMatchProvider(f.num, 'fifa'); + if (row && row.updatedAt > new Date(f.kickoff).getTime() + FOTMOB_SETTLE_MS) continue; + } + const t1 = Date.now(); + try { + const [liveData, timeline] = [await fetchFifaLive(idStage, idMatch, live), await fetchFifaTimeline(idStage, idMatch, live)]; + setMatchProvider(f.num, 'fifa', { v: 1, fetchedAt: Date.now(), live: liveData, timeline }); + logIngest('fifa', live ? 'live' : imminent ? 'lineups' : 'final', true, Date.now() - t1, `M${f.num}`); + } catch (e) { + logIngest('fifa', 'match', false, Date.now() - t1, e instanceof Error ? e.message : String(e)); + } + } + }; + + let fifaTimer: ReturnType | undefined; + const scheduleFifa = (): void => { + if (stopped) return; + fifaTimer = setTimeout(async () => { await fifaTick().catch(() => {}); scheduleFifa(); }, state.hasLive() ? 2 * 60_000 : 30 * 60_000); + }; + // ---- odds capture: every upcoming fixture inside the horizon, plus imminent // ones near kickoff. Insert-if-changed keeps a clean line-movement history. const oddsTick = async (): Promise => { @@ -168,6 +288,17 @@ export function startScheduler( })) captured++; } } catch { /* breaker/health track it */ } + // scorer props (anytime/first/last goalscorer lines) — only near kickoff, + // where the lines are most meaningful and the budget stays sane + if (new Date(f.kickoff).getTime() - now < 2 * 24 * 60 * 60 * 1000) { + try { + let propMoves = 0; + for (const p of await fetchScorerProps(eid)) { + if (recordScorerProp({ fixture_num: f.num, provider: 'DraftKings', market: p.market, athlete: p.athleteName, odds: p.american })) propMoves++; + } + if (propMoves) logIngest('espn-odds', 'props', true, 0, `M${f.num}: ${propMoves} prop line(s)`); + } catch { /* breaker/health track it */ } + } } logIngest('espn-odds', 'sweep', true, Date.now() - t0, `${relevant.length} fixtures, ${captured} new lines`); if (captured) console.log(`[ingest] odds: ${captured} new line(s) captured`); @@ -208,6 +339,15 @@ export function startScheduler( await mapAllFixtures().catch(() => {}); await liveTick().catch(() => {}); scheduleLive(); // live cadence starts now; slow one-time sweeps follow + // persist whatever the current scoreboard window knows — results learned + // before this table existed would otherwise never be written + for (const f of state.allFixtures()) { + if (f.status !== 'scheduled') saveFixtureResult(f.num, f.homeScore, f.awayScore, f.status); + } + await fotmobTick().catch(() => {}); // capture-first: unofficial source, harvest while it lasts + scheduleFotmob(); + await fifaTick().catch(() => {}); + scheduleFifa(); await oddsTick().catch(() => {}); await backfillExt().catch(() => {}); await enrichTick().catch(() => {}); @@ -221,6 +361,8 @@ export function startScheduler( if (liveTimer) clearTimeout(liveTimer); if (enrichTimer) clearTimeout(enrichTimer); if (oddsTimer) clearTimeout(oddsTimer); + if (fotmobTimer) clearTimeout(fotmobTimer); + if (fifaTimer) clearTimeout(fifaTimer); clearInterval(pruneTimer); }; } diff --git a/server/src/ingest/weather.ts b/server/src/ingest/weather.ts new file mode 100644 index 0000000..d769c7d --- /dev/null +++ b/server/src/ingest/weather.ts @@ -0,0 +1,51 @@ +// Open-Meteo (keyless, works from the VPS): kickoff-hour weather per venue. +// Venues are pre-geocoded at build time into public/data/venues.json — +// includes elevation (Mexico City 2,240m), which is its own story. +import { cachedFetch } from './fetcher'; +import { loadStatic } from '../preview'; + +const SOURCE = 'open-meteo'; + +interface VenuesFile { + venues: Record; +} + +let _venues: VenuesFile | null = null; +const venues = (): VenuesFile => (_venues ??= loadStatic('venues.json')); + +export interface MatchWeather { + v: 1; + fetchedAt: number; + tempC: number | null; + precipProbPct: number | null; + windKmh: number | null; + elevationM: number; + tz: string; +} + +interface Forecast { + hourly?: { time?: string[]; temperature_2m?: number[]; precipitation_probability?: number[]; wind_speed_10m?: number[] }; +} + +/** Kickoff-hour forecast for a fixture's venue (forecast horizon ≈16 days). */ +export async function fetchMatchWeather(venue: string, kickoffIso: string): Promise { + const v = venues().venues[venue]; + if (!v) return null; + const url = + `https://api.open-meteo.com/v1/forecast?latitude=${v.lat}&longitude=${v.lon}` + + `&hourly=temperature_2m,precipitation_probability,wind_speed_10m&forecast_days=16&timezone=UTC`; + const f = await cachedFetch(SOURCE, `weather:${venue}`, url, { ttl: 3 * 60 * 60_000, minInterval: 1_000 }); + const hours = f.hourly?.time ?? []; + const wantHour = kickoffIso.slice(0, 13); // YYYY-MM-DDTHH + const idx = hours.findIndex((t) => t.startsWith(wantHour)); + if (idx < 0) return null; + return { + v: 1, + fetchedAt: Date.now(), + tempC: f.hourly?.temperature_2m?.[idx] ?? null, + precipProbPct: f.hourly?.precipitation_probability?.[idx] ?? null, + windKmh: f.hourly?.wind_speed_10m?.[idx] ?? null, + elevationM: v.elevation, + tz: v.tz, + }; +}