v2 Phase D: capture-first ingestion — FotMob, FIFA, weather, scorer props
Four new feeds, all verified against real tournament matches: - FotMob (unofficial, works from the VPS again): per-shot xG/xGOT with pitch coords, team xG, player ratings, POTM, momentum, attacking zones — trimmed payloads (~87KB/match) on a gentle 90s-live/30m-idle cadence. Capture-first: this source could lock again any day. - FIFA official v3: confirmed lineups with player IDs, typed XY event timelines (80 events for the opener), attendance + named referees, tactics — id-mapped via the calendar, hot while live. - Open-Meteo: kickoff-hour weather per venue from a build-time geocoded venues.json (with elevation — Azteca 2,240m); fetched in the enrich loop inside the 48h horizon. - DraftKings scorer props (first/last/anytime goalscorer per athlete) via ESPN's core propBets, insert-if-changed line history, athlete names resolved through a 30-day cache. Also fixes a real resilience gap found along the way: fixture results now persist in SQLite and restore at boot — previously a restart after ESPN's scoreboard window moved on would silently forget a match day (standings, model and scoreboard would all regress). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -93,6 +93,33 @@ CREATE TABLE IF NOT EXISTS prediction_snapshots (
|
||||
market_json TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fixture_results (
|
||||
fixture_num INTEGER PRIMARY KEY,
|
||||
home_score INTEGER,
|
||||
away_score INTEGER,
|
||||
status TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS match_provider (
|
||||
fixture_num INTEGER NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
json TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (fixture_num, provider)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS scorer_props (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
fixture_num INTEGER NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
market TEXT NOT NULL,
|
||||
athlete TEXT NOT NULL,
|
||||
odds REAL,
|
||||
captured_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_scorer_props ON scorer_props(fixture_num, market, athlete, captured_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS push_subs (
|
||||
endpoint TEXT PRIMARY KEY,
|
||||
json TEXT NOT NULL,
|
||||
@@ -154,6 +181,53 @@ export function cacheSet(key: string, value: string, ttl: number): void {
|
||||
.run(key, value, Date.now(), ttl);
|
||||
}
|
||||
|
||||
// ---- fixture results survive restarts (the live snapshot is in-memory and
|
||||
// ESPN's scoreboard window moves on — without this, yesterday's results
|
||||
// would vanish from standings/model after a redeploy) ----
|
||||
export function saveFixtureResult(num: number, homeScore: number | null, awayScore: number | null, status: string): void {
|
||||
db()
|
||||
.prepare('INSERT OR REPLACE INTO fixture_results (fixture_num, home_score, away_score, status, updated_at) VALUES (?, ?, ?, ?, ?)')
|
||||
.run(num, homeScore, awayScore, status, Date.now());
|
||||
}
|
||||
|
||||
export function allFixtureResults(): { fixture_num: number; home_score: number | null; away_score: number | null; status: string }[] {
|
||||
return db()
|
||||
.prepare('SELECT fixture_num, home_score, away_score, status FROM fixture_results')
|
||||
.all() as unknown as { fixture_num: number; home_score: number | null; away_score: number | null; status: string }[];
|
||||
}
|
||||
|
||||
// ---- per-provider rich match payloads (fotmob / fifa / weather) ----
|
||||
export function setMatchProvider(fixtureNum: number, provider: string, json: unknown): void {
|
||||
db()
|
||||
.prepare('INSERT OR REPLACE INTO match_provider (fixture_num, provider, json, updated_at) VALUES (?, ?, ?, ?)')
|
||||
.run(fixtureNum, provider, JSON.stringify(json), Date.now());
|
||||
}
|
||||
|
||||
export function getMatchProvider<T = unknown>(fixtureNum: number, provider: string): { data: T; updatedAt: number } | null {
|
||||
const row = db()
|
||||
.prepare('SELECT json, updated_at FROM match_provider WHERE fixture_num = ? AND provider = ?')
|
||||
.get(fixtureNum, provider) as { json: string; updated_at: number } | undefined;
|
||||
return row ? { data: JSON.parse(row.json) as T, updatedAt: row.updated_at } : null;
|
||||
}
|
||||
|
||||
/** Insert a scorer prop only when the line moved (mirrors recordOdds). */
|
||||
export function recordScorerProp(p: { fixture_num: number; provider: string; market: string; athlete: string; odds: number | null }): boolean {
|
||||
const last = db()
|
||||
.prepare('SELECT odds FROM scorer_props WHERE fixture_num = ? AND provider = ? AND market = ? AND athlete = ? ORDER BY captured_at DESC LIMIT 1')
|
||||
.get(p.fixture_num, p.provider, p.market, p.athlete) as { odds: number | null } | undefined;
|
||||
if (last && last.odds === p.odds) return false;
|
||||
db()
|
||||
.prepare('INSERT INTO scorer_props (fixture_num, provider, market, athlete, odds, captured_at) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(p.fixture_num, p.provider, p.market, p.athlete, p.odds, Date.now());
|
||||
return true;
|
||||
}
|
||||
|
||||
export function scorerPropsFor(fixtureNum: number): { provider: string; market: string; athlete: string; odds: number | null; captured_at: number }[] {
|
||||
return db()
|
||||
.prepare('SELECT provider, market, athlete, odds, captured_at FROM scorer_props WHERE fixture_num = ? ORDER BY captured_at')
|
||||
.all(fixtureNum) as unknown as { provider: string; market: string; athlete: string; odds: number | null; captured_at: number }[];
|
||||
}
|
||||
|
||||
// ---- Web Push subscriptions (goal/kickoff notifications, opt-in per team) ----
|
||||
export interface PushSub {
|
||||
endpoint: string;
|
||||
|
||||
+10
-1
@@ -10,7 +10,7 @@ import { startScheduler } from './ingest/scheduler';
|
||||
import { ModelEngine } from './model';
|
||||
import { buildPreview, buildTeamProfile } from './preview';
|
||||
import { buildScoreboard, snapshotCheck } from './scoreboard';
|
||||
import { db, healthAll, oddsFor, latestOddsAll, dbCounts, inplayHistory, recordInplay, getMatchExt } from './db/db';
|
||||
import { db, healthAll, oddsFor, latestOddsAll, dbCounts, inplayHistory, recordInplay, getMatchExt, allFixtureResults, saveFixtureResult } from './db/db';
|
||||
import { inPlayProbs } from '../../src/lib/model/inplay';
|
||||
import { PushNotifier, pushEnabled, subscribe as pushSubscribe, unsubscribe as pushUnsubscribe, vapidPublicKey } from './push';
|
||||
import type { EspnSummary } from './ingest/espnNormalize';
|
||||
@@ -37,6 +37,14 @@ export function buildServer() {
|
||||
|
||||
db(); // open + migrate SQLite before anything reads it
|
||||
const state = new TournamentState(STATIC_DIR);
|
||||
// Restore persisted results over the seed — the in-memory snapshot forgets
|
||||
// them on restart once ESPN's scoreboard window has moved past a match day.
|
||||
let restored = 0;
|
||||
for (const r of allFixtureResults()) {
|
||||
if (r.status === 'scheduled') continue;
|
||||
if (state.applyScore(r.fixture_num, r.home_score, r.away_score, r.status as MatchStatus, null, 'seed')) restored++;
|
||||
}
|
||||
if (restored) console.log(`[state] restored ${restored} fixture result(s) from the DB`);
|
||||
const model = new ModelEngine(STATIC_DIR);
|
||||
model.refitTeamDc(state.allFixtures()); // current Team-DC from day one (covers restarts mid-tournament)
|
||||
model.recompute(state.allFixtures());
|
||||
@@ -184,6 +192,7 @@ export function buildServer() {
|
||||
'football-data',
|
||||
);
|
||||
if (!f) return reply.code(404).send({ error: 'no such fixture' });
|
||||
saveFixtureResult(f.num, f.homeScore, f.awayScore, f.status);
|
||||
snapshotCheck(state, model);
|
||||
recordInplayTick();
|
||||
broadcast();
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
// ESPN core API prop bets: per-athlete goalscorer odds (DraftKings, provider
|
||||
// 100) — First Goalscorer / Last Goalscorer / To Score 2+ Goals, plus anytime
|
||||
// variants. Line history is unrecoverable, so capture beats everything; the
|
||||
// athlete $refs resolve to names lazily through a long-TTL cache.
|
||||
import { cachedFetch } from './fetcher';
|
||||
|
||||
const CORE = 'https://sports.core.api.espn.com/v2/sports/soccer/leagues/fifa.world';
|
||||
const SOURCE = 'espn-odds';
|
||||
const PROVIDER = 100; // DraftKings on the core API
|
||||
|
||||
export interface ScorerProp {
|
||||
market: string;
|
||||
athleteName: string;
|
||||
american: number | null;
|
||||
}
|
||||
|
||||
interface PropPage {
|
||||
pageCount?: number;
|
||||
items?: {
|
||||
type?: { name?: string; text?: string };
|
||||
athlete?: { $ref?: string };
|
||||
current?: { odds?: { american?: string | number } | string; american?: string };
|
||||
open?: unknown;
|
||||
}[];
|
||||
}
|
||||
|
||||
const WANTED = /goalscorer|to score/i;
|
||||
|
||||
function americanOf(item: PropPage['items'] extends (infer T)[] | undefined ? T : never): number | null {
|
||||
const cur = item.current as { american?: string | number; odds?: { american?: string | number } } | undefined;
|
||||
const raw = cur?.american ?? cur?.odds?.american;
|
||||
if (raw == null) return null;
|
||||
const n = Number(String(raw).replace('+', ''));
|
||||
return Number.isFinite(n) ? n : null;
|
||||
}
|
||||
|
||||
/** Resolve an athlete $ref to a display name (cached ~30 days). */
|
||||
async function athleteName(ref: string): Promise<string | null> {
|
||||
const m = /athletes\/(\d+)/.exec(ref);
|
||||
if (!m) return null;
|
||||
try {
|
||||
const a = await cachedFetch<{ displayName?: string; fullName?: string }>(
|
||||
SOURCE,
|
||||
`espn:athlete:${m[1]}`,
|
||||
ref.replace(/^http:/, 'https:'),
|
||||
{ ttl: 30 * 24 * 60 * 60_000, minInterval: 1_200 },
|
||||
);
|
||||
return a.displayName ?? a.fullName ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** All scorer props for an event. Name resolution is deadline-bounded per
|
||||
* call: cold caches fill across successive sweeps (each athlete lookup is
|
||||
* cached ~30 days), warm caches resolve everything instantly. */
|
||||
export async function fetchScorerProps(eventId: string, budgetMs = 45_000): Promise<ScorerProp[]> {
|
||||
const out: ScorerProp[] = [];
|
||||
const deadline = Date.now() + budgetMs;
|
||||
for (let page = 1; page <= 10; page++) {
|
||||
const url = `${CORE}/events/${eventId}/competitions/${eventId}/odds/${PROVIDER}/propBets?limit=100&page=${page}`;
|
||||
const d = await cachedFetch<PropPage>(SOURCE, `espn:props:${eventId}:${page}`, url, {
|
||||
ttl: 60 * 60_000,
|
||||
minInterval: 1_500,
|
||||
});
|
||||
for (const item of d.items ?? []) {
|
||||
const market = item.type?.text ?? item.type?.name ?? '';
|
||||
if (!WANTED.test(market)) continue;
|
||||
const ref = item.athlete?.$ref;
|
||||
if (!ref || Date.now() > deadline) continue;
|
||||
const name = await athleteName(ref);
|
||||
if (!name) continue;
|
||||
out.push({ market, athleteName: name, american: americanOf(item) });
|
||||
}
|
||||
if ((d.pageCount ?? 1) <= page) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// FIFA's official v3 API (api.fifa.com) — open, no auth, works from the VPS.
|
||||
// Carries what no other source has officially: confirmed lineups with stable
|
||||
// player IDs + headshots, typed XY event timelines, attendance, tactics,
|
||||
// officials. World Cup 2026: IdCompetition=17, IdSeason=285023.
|
||||
import { cachedFetch } from './fetcher';
|
||||
import { canonicalTeam } from '../../../src/lib/teams';
|
||||
|
||||
const BASE = 'https://api.fifa.com/api/v3';
|
||||
const SOURCE = 'fifa';
|
||||
export const FIFA_COMPETITION = '17';
|
||||
export const FIFA_SEASON = '285023';
|
||||
|
||||
const locDesc = (v: unknown): string => {
|
||||
const a = v as { Description?: string }[] | undefined;
|
||||
return a?.[0]?.Description ?? '';
|
||||
};
|
||||
|
||||
export interface FifaMatchRef {
|
||||
idMatch: string;
|
||||
idStage: string;
|
||||
home: string;
|
||||
away: string;
|
||||
date: string;
|
||||
attendance: number | null;
|
||||
officials: unknown[];
|
||||
stadium: string;
|
||||
}
|
||||
|
||||
interface RawCalendar {
|
||||
Results?: {
|
||||
IdMatch?: string;
|
||||
IdStage?: string;
|
||||
Date?: string;
|
||||
Attendance?: string | number | null;
|
||||
Officials?: unknown[];
|
||||
Stadium?: { Name?: unknown };
|
||||
Home?: { TeamName?: unknown };
|
||||
Away?: { TeamName?: unknown };
|
||||
}[];
|
||||
}
|
||||
|
||||
/** Full tournament calendar window (id map + attendance/officials source). */
|
||||
export async function fetchFifaCalendar(fromIso: string, toIso: string): Promise<FifaMatchRef[]> {
|
||||
const url = `${BASE}/calendar/matches?idCompetition=${FIFA_COMPETITION}&idSeason=${FIFA_SEASON}&from=${encodeURIComponent(fromIso)}&to=${encodeURIComponent(toIso)}&count=120&language=en`;
|
||||
const raw = await cachedFetch<RawCalendar>(SOURCE, `fifa:cal:${fromIso.slice(0, 10)}:${toIso.slice(0, 10)}`, url, {
|
||||
ttl: 10 * 60_000,
|
||||
minInterval: 2_000,
|
||||
});
|
||||
const out: FifaMatchRef[] = [];
|
||||
for (const m of raw.Results ?? []) {
|
||||
const home = canonicalTeam(locDesc(m.Home?.TeamName));
|
||||
const away = canonicalTeam(locDesc(m.Away?.TeamName));
|
||||
if (!m.IdMatch || !m.IdStage || !home || !away) continue;
|
||||
out.push({
|
||||
idMatch: m.IdMatch,
|
||||
idStage: m.IdStage,
|
||||
home,
|
||||
away,
|
||||
date: m.Date ?? '',
|
||||
attendance: m.Attendance != null && m.Attendance !== '' ? Number(m.Attendance) : null,
|
||||
officials: m.Officials ?? [],
|
||||
stadium: locDesc(m.Stadium?.Name),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Rich live payload: confirmed lineups (IdPlayer, shirt numbers, positions),
|
||||
* tactics, match clock. Stored whole — it's the official record. */
|
||||
export async function fetchFifaLive(idStage: string, idMatch: string, live = false): Promise<unknown> {
|
||||
return cachedFetch<unknown>(SOURCE, `fifa:live:${idMatch}`, `${BASE}/live/football/${FIFA_COMPETITION}/${FIFA_SEASON}/${idStage}/${idMatch}?language=en`, {
|
||||
ttl: live ? 60_000 : 60 * 60_000,
|
||||
minInterval: 2_000,
|
||||
});
|
||||
}
|
||||
|
||||
/** Typed event-by-event timeline with pitch coordinates. */
|
||||
export async function fetchFifaTimeline(idStage: string, idMatch: string, live = false): Promise<unknown> {
|
||||
return cachedFetch<unknown>(SOURCE, `fifa:tl:${idMatch}`, `${BASE}/timelines/${FIFA_COMPETITION}/${FIFA_SEASON}/${idStage}/${idMatch}?language=en`, {
|
||||
ttl: live ? 60_000 : 60 * 60_000,
|
||||
minInterval: 2_000,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
// FotMob (unofficial, www.fotmob.com/api/data) — as of June 2026 it serves
|
||||
// full match details from the VPS with no signed header: per-shot xG/xGOT
|
||||
// with pitch coordinates, team xG splits, player ratings, player of the
|
||||
// match, momentum, attacking zones, lineups. Unofficial means it can lock
|
||||
// again any day — so capture early, store generously, degrade gracefully.
|
||||
import { cachedFetch } from './fetcher';
|
||||
import { canonicalTeam } from '../../../src/lib/teams';
|
||||
|
||||
const BASE = 'https://www.fotmob.com/api/data';
|
||||
const SOURCE = 'fotmob';
|
||||
const WC_LEAGUE_ID = 77;
|
||||
|
||||
export interface FotmobMatchRef {
|
||||
matchId: string;
|
||||
home: string;
|
||||
away: string;
|
||||
utcTime: string;
|
||||
}
|
||||
|
||||
interface RawMatches {
|
||||
leagues?: {
|
||||
primaryId?: number;
|
||||
matches?: {
|
||||
id?: number | string;
|
||||
home?: { name?: string };
|
||||
away?: { name?: string };
|
||||
status?: { utcTime?: string };
|
||||
}[];
|
||||
}[];
|
||||
}
|
||||
|
||||
/** World Cup matches on a given UTC date (id map source). */
|
||||
export async function fetchFotmobMatches(dateYYYYMMDD: string): Promise<FotmobMatchRef[]> {
|
||||
const raw = await cachedFetch<RawMatches>(SOURCE, `fotmob:matches:${dateYYYYMMDD}`, `${BASE}/matches?date=${dateYYYYMMDD}`, {
|
||||
ttl: 10 * 60_000,
|
||||
minInterval: 2_500,
|
||||
});
|
||||
const out: FotmobMatchRef[] = [];
|
||||
for (const lg of raw.leagues ?? []) {
|
||||
if (lg.primaryId !== WC_LEAGUE_ID) continue;
|
||||
for (const m of lg.matches ?? []) {
|
||||
if (!m.id || !m.home?.name || !m.away?.name) continue;
|
||||
out.push({
|
||||
matchId: String(m.id),
|
||||
home: canonicalTeam(m.home.name),
|
||||
away: canonicalTeam(m.away.name),
|
||||
utcTime: m.status?.utcTime ?? '',
|
||||
});
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Trimmed matchDetails: everything analytic, minus the bulky redundant tabs
|
||||
* (liveticker/buzz/h2h/table/seo) we already cover via other sources. */
|
||||
export interface FotmobDetails {
|
||||
v: 1;
|
||||
fetchedAt: number;
|
||||
matchId: string;
|
||||
finished: boolean;
|
||||
shots: unknown[];
|
||||
stats: unknown;
|
||||
matchFacts: unknown;
|
||||
lineup: unknown;
|
||||
momentum: unknown;
|
||||
attackingZones: unknown;
|
||||
weather: unknown;
|
||||
}
|
||||
|
||||
interface RawDetails {
|
||||
general?: { matchId?: string; finished?: boolean };
|
||||
content?: {
|
||||
shotmap?: { shots?: unknown[] };
|
||||
stats?: unknown;
|
||||
matchFacts?: Record<string, unknown>;
|
||||
lineup?: unknown;
|
||||
momentum?: unknown;
|
||||
attackingZones?: unknown;
|
||||
weather?: unknown;
|
||||
};
|
||||
}
|
||||
|
||||
export async function fetchFotmobDetails(matchId: string, live = false): Promise<FotmobDetails> {
|
||||
const raw = await cachedFetch<RawDetails>(SOURCE, `fotmob:details:${matchId}`, `${BASE}/matchDetails?matchId=${matchId}`, {
|
||||
ttl: live ? 60_000 : 60 * 60_000,
|
||||
minInterval: 2_500,
|
||||
});
|
||||
const c = raw.content ?? {};
|
||||
const mf = { ...(c.matchFacts ?? {}) } as Record<string, unknown>;
|
||||
delete mf.QAData; // SEO filler
|
||||
delete mf.postReview;
|
||||
delete mf.preReview;
|
||||
delete mf.poll;
|
||||
return {
|
||||
v: 1,
|
||||
fetchedAt: Date.now(),
|
||||
matchId: raw.general?.matchId ?? matchId,
|
||||
finished: raw.general?.finished ?? false,
|
||||
shots: c.shotmap?.shots ?? [],
|
||||
stats: c.stats ?? null,
|
||||
matchFacts: mf,
|
||||
lineup: c.lineup ?? null,
|
||||
momentum: c.momentum ?? null,
|
||||
attackingZones: c.attackingZones ?? null,
|
||||
weather: c.weather ?? null,
|
||||
};
|
||||
}
|
||||
@@ -4,7 +4,11 @@ import { fetchEspnScoreboard, fetchEspnSummary } from './espn';
|
||||
import { fetchEspnOdds } from './espnOdds';
|
||||
import { fetchFootballData } from '../footballData';
|
||||
import { fetchSofascore } from '../sofascore';
|
||||
import { setSourceMap, getSourceMap, getMatchExt, setMatchExt, recordOdds, logIngest, prune, backupDb } from '../db/db';
|
||||
import { setSourceMap, getSourceMap, getMatchExt, setMatchExt, setMatchProvider, getMatchProvider, recordOdds, recordScorerProp, logIngest, prune, backupDb, saveFixtureResult } from '../db/db';
|
||||
import { fetchFotmobDetails, fetchFotmobMatches } from './fotmob';
|
||||
import { fetchFifaCalendar, fetchFifaLive, fetchFifaTimeline } from './fifa';
|
||||
import { fetchMatchWeather } from './weather';
|
||||
import { fetchScorerProps } from './espnProps';
|
||||
import type { EspnSummary } from './espnNormalize';
|
||||
import type { Fixture } from '../../../src/lib/types';
|
||||
|
||||
@@ -92,7 +96,11 @@ export function startScheduler(
|
||||
}
|
||||
if (matches.length) {
|
||||
const changed = state.mergeLive(matches, source);
|
||||
if (changed.length) { console.log(`[ingest] ${changed.length} fixture(s) updated via ${source}`); onChange(); }
|
||||
if (changed.length) {
|
||||
for (const f of changed) saveFixtureResult(f.num, f.homeScore, f.awayScore, f.status);
|
||||
console.log(`[ingest] ${changed.length} fixture(s) updated via ${source}`);
|
||||
onChange();
|
||||
}
|
||||
}
|
||||
onTick?.(); // e.g. freeze pre-kickoff forecast snapshots
|
||||
|
||||
@@ -141,9 +149,121 @@ export function startScheduler(
|
||||
} catch (e) {
|
||||
logIngest('espn', 'summary', false, Date.now() - t0, e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
// kickoff-hour weather for fixtures inside the forecast horizon
|
||||
if (f.status === 'scheduled' && new Date(f.kickoff).getTime() - now < 2 * 24 * 60 * 60 * 1000) {
|
||||
try {
|
||||
const w = await fetchMatchWeather(f.venue, f.kickoff);
|
||||
if (w) setMatchProvider(f.num, 'weather', w);
|
||||
} catch { /* health-tracked */ }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// ---- FotMob: xG / shotmaps / ratings — capture-first (unofficial source,
|
||||
// could lock again any day). Own gentle cadence: 90s while live, 30m idle.
|
||||
const FOTMOB_SETTLE_MS = 3 * 60 * 60 * 1000; // refetch finished until FT+3h (settled ratings)
|
||||
const fotmobTick = async (): Promise<void> => {
|
||||
const now = Date.now();
|
||||
// map: sweep dates (now ±3d) that still contain unmapped fixtures
|
||||
const dates = new Set<string>();
|
||||
for (const f of state.allFixtures()) {
|
||||
const k = new Date(f.kickoff).getTime();
|
||||
if (Math.abs(k - now) > 3 * 24 * 60 * 60 * 1000) continue;
|
||||
if (!getSourceMap(f.num, 'fotmob')) dates.add(dateUTC(f.kickoff));
|
||||
}
|
||||
for (const date of dates) {
|
||||
if (stopped) return;
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
const refs = await fetchFotmobMatches(date);
|
||||
for (const r of refs) {
|
||||
const num = state.matchFixtureNum(r.home, r.away, 'group', r.utcTime || `${date.slice(0, 4)}-${date.slice(4, 6)}-${date.slice(6, 8)}T12:00:00Z`);
|
||||
if (num && !getSourceMap(num, 'fotmob')) setSourceMap(num, 'fotmob', r.matchId);
|
||||
}
|
||||
logIngest('fotmob', 'map', true, Date.now() - t0, `${date}: ${refs.length} matches`);
|
||||
} catch (e) {
|
||||
logIngest('fotmob', 'map', false, Date.now() - t0, e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
}
|
||||
// details: live fixtures at the live TTL; finished ones until settled
|
||||
for (const f of state.allFixtures()) {
|
||||
if (stopped) return;
|
||||
const id = getSourceMap(f.num, 'fotmob');
|
||||
if (!id) continue;
|
||||
const live = f.status === 'live';
|
||||
if (!live && f.status !== 'finished') continue;
|
||||
if (f.status === 'finished') {
|
||||
const row = getMatchProvider(f.num, 'fotmob');
|
||||
const settled = row && row.updatedAt > new Date(f.kickoff).getTime() + FOTMOB_SETTLE_MS;
|
||||
if (settled) continue;
|
||||
}
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
setMatchProvider(f.num, 'fotmob', await fetchFotmobDetails(id, live));
|
||||
logIngest('fotmob', live ? 'live' : 'final', true, Date.now() - t0, `M${f.num}`);
|
||||
} catch (e) {
|
||||
logIngest('fotmob', 'details', false, Date.now() - t0, e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let fotmobTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
const scheduleFotmob = (): void => {
|
||||
if (stopped) return;
|
||||
fotmobTimer = setTimeout(async () => { await fotmobTick().catch(() => {}); scheduleFotmob(); }, state.hasLive() ? 90_000 : 30 * 60_000);
|
||||
};
|
||||
|
||||
// ---- FIFA official: confirmed lineups, XY timelines, attendance/officials.
|
||||
const fifaTick = async (): Promise<void> => {
|
||||
const now = Date.now();
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
// calendar sweep (one cached call covers ±3d): id map + official match info
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
const from = new Date(now - 3 * DAY_MS).toISOString().slice(0, 10);
|
||||
const to = new Date(now + 3 * DAY_MS).toISOString().slice(0, 10);
|
||||
const refs = await fetchFifaCalendar(`${from}T00:00:00Z`, `${to}T00:00:00Z`);
|
||||
for (const r of refs) {
|
||||
const num = state.matchFixtureNum(r.home, r.away, 'group', r.date);
|
||||
if (!num) continue;
|
||||
if (!getSourceMap(num, 'fifa')) setSourceMap(num, 'fifa', `${r.idStage}|${r.idMatch}`);
|
||||
setMatchProvider(num, 'fifa-info', { attendance: r.attendance, officials: r.officials, stadium: r.stadium, date: r.date });
|
||||
}
|
||||
logIngest('fifa', 'calendar', true, Date.now() - t0, `${refs.length} matches`);
|
||||
} catch (e) {
|
||||
logIngest('fifa', 'calendar', false, Date.now() - t0, e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
// lineups + timeline: live fixtures hot, finished until settled
|
||||
for (const f of state.allFixtures()) {
|
||||
if (stopped) return;
|
||||
const mapId = getSourceMap(f.num, 'fifa');
|
||||
if (!mapId) continue;
|
||||
const [idStage, idMatch] = mapId.split('|');
|
||||
if (!idStage || !idMatch) continue;
|
||||
const live = f.status === 'live';
|
||||
const imminent = f.status === 'scheduled' && new Date(f.kickoff).getTime() - now < 90 * 60_000 && new Date(f.kickoff).getTime() > now;
|
||||
if (!live && !imminent && f.status !== 'finished') continue;
|
||||
if (f.status === 'finished') {
|
||||
const row = getMatchProvider(f.num, 'fifa');
|
||||
if (row && row.updatedAt > new Date(f.kickoff).getTime() + FOTMOB_SETTLE_MS) continue;
|
||||
}
|
||||
const t1 = Date.now();
|
||||
try {
|
||||
const [liveData, timeline] = [await fetchFifaLive(idStage, idMatch, live), await fetchFifaTimeline(idStage, idMatch, live)];
|
||||
setMatchProvider(f.num, 'fifa', { v: 1, fetchedAt: Date.now(), live: liveData, timeline });
|
||||
logIngest('fifa', live ? 'live' : imminent ? 'lineups' : 'final', true, Date.now() - t1, `M${f.num}`);
|
||||
} catch (e) {
|
||||
logIngest('fifa', 'match', false, Date.now() - t1, e instanceof Error ? e.message : String(e));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let fifaTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
const scheduleFifa = (): void => {
|
||||
if (stopped) return;
|
||||
fifaTimer = setTimeout(async () => { await fifaTick().catch(() => {}); scheduleFifa(); }, state.hasLive() ? 2 * 60_000 : 30 * 60_000);
|
||||
};
|
||||
|
||||
// ---- odds capture: every upcoming fixture inside the horizon, plus imminent
|
||||
// ones near kickoff. Insert-if-changed keeps a clean line-movement history.
|
||||
const oddsTick = async (): Promise<void> => {
|
||||
@@ -168,6 +288,17 @@ export function startScheduler(
|
||||
})) captured++;
|
||||
}
|
||||
} catch { /* breaker/health track it */ }
|
||||
// scorer props (anytime/first/last goalscorer lines) — only near kickoff,
|
||||
// where the lines are most meaningful and the budget stays sane
|
||||
if (new Date(f.kickoff).getTime() - now < 2 * 24 * 60 * 60 * 1000) {
|
||||
try {
|
||||
let propMoves = 0;
|
||||
for (const p of await fetchScorerProps(eid)) {
|
||||
if (recordScorerProp({ fixture_num: f.num, provider: 'DraftKings', market: p.market, athlete: p.athleteName, odds: p.american })) propMoves++;
|
||||
}
|
||||
if (propMoves) logIngest('espn-odds', 'props', true, 0, `M${f.num}: ${propMoves} prop line(s)`);
|
||||
} catch { /* breaker/health track it */ }
|
||||
}
|
||||
}
|
||||
logIngest('espn-odds', 'sweep', true, Date.now() - t0, `${relevant.length} fixtures, ${captured} new lines`);
|
||||
if (captured) console.log(`[ingest] odds: ${captured} new line(s) captured`);
|
||||
@@ -208,6 +339,15 @@ export function startScheduler(
|
||||
await mapAllFixtures().catch(() => {});
|
||||
await liveTick().catch(() => {});
|
||||
scheduleLive(); // live cadence starts now; slow one-time sweeps follow
|
||||
// persist whatever the current scoreboard window knows — results learned
|
||||
// before this table existed would otherwise never be written
|
||||
for (const f of state.allFixtures()) {
|
||||
if (f.status !== 'scheduled') saveFixtureResult(f.num, f.homeScore, f.awayScore, f.status);
|
||||
}
|
||||
await fotmobTick().catch(() => {}); // capture-first: unofficial source, harvest while it lasts
|
||||
scheduleFotmob();
|
||||
await fifaTick().catch(() => {});
|
||||
scheduleFifa();
|
||||
await oddsTick().catch(() => {});
|
||||
await backfillExt().catch(() => {});
|
||||
await enrichTick().catch(() => {});
|
||||
@@ -221,6 +361,8 @@ export function startScheduler(
|
||||
if (liveTimer) clearTimeout(liveTimer);
|
||||
if (enrichTimer) clearTimeout(enrichTimer);
|
||||
if (oddsTimer) clearTimeout(oddsTimer);
|
||||
if (fotmobTimer) clearTimeout(fotmobTimer);
|
||||
if (fifaTimer) clearTimeout(fifaTimer);
|
||||
clearInterval(pruneTimer);
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
// Open-Meteo (keyless, works from the VPS): kickoff-hour weather per venue.
|
||||
// Venues are pre-geocoded at build time into public/data/venues.json —
|
||||
// includes elevation (Mexico City 2,240m), which is its own story.
|
||||
import { cachedFetch } from './fetcher';
|
||||
import { loadStatic } from '../preview';
|
||||
|
||||
const SOURCE = 'open-meteo';
|
||||
|
||||
interface VenuesFile {
|
||||
venues: Record<string, { lat: number; lon: number; elevation: number; tz: string }>;
|
||||
}
|
||||
|
||||
let _venues: VenuesFile | null = null;
|
||||
const venues = (): VenuesFile => (_venues ??= loadStatic<VenuesFile>('venues.json'));
|
||||
|
||||
export interface MatchWeather {
|
||||
v: 1;
|
||||
fetchedAt: number;
|
||||
tempC: number | null;
|
||||
precipProbPct: number | null;
|
||||
windKmh: number | null;
|
||||
elevationM: number;
|
||||
tz: string;
|
||||
}
|
||||
|
||||
interface Forecast {
|
||||
hourly?: { time?: string[]; temperature_2m?: number[]; precipitation_probability?: number[]; wind_speed_10m?: number[] };
|
||||
}
|
||||
|
||||
/** Kickoff-hour forecast for a fixture's venue (forecast horizon ≈16 days). */
|
||||
export async function fetchMatchWeather(venue: string, kickoffIso: string): Promise<MatchWeather | null> {
|
||||
const v = venues().venues[venue];
|
||||
if (!v) return null;
|
||||
const url =
|
||||
`https://api.open-meteo.com/v1/forecast?latitude=${v.lat}&longitude=${v.lon}` +
|
||||
`&hourly=temperature_2m,precipitation_probability,wind_speed_10m&forecast_days=16&timezone=UTC`;
|
||||
const f = await cachedFetch<Forecast>(SOURCE, `weather:${venue}`, url, { ttl: 3 * 60 * 60_000, minInterval: 1_000 });
|
||||
const hours = f.hourly?.time ?? [];
|
||||
const wantHour = kickoffIso.slice(0, 13); // YYYY-MM-DDTHH
|
||||
const idx = hours.findIndex((t) => t.startsWith(wantHour));
|
||||
if (idx < 0) return null;
|
||||
return {
|
||||
v: 1,
|
||||
fetchedAt: Date.now(),
|
||||
tempC: f.hourly?.temperature_2m?.[idx] ?? null,
|
||||
precipProbPct: f.hourly?.precipitation_probability?.[idx] ?? null,
|
||||
windKmh: f.hourly?.wind_speed_10m?.[idx] ?? null,
|
||||
elevationM: v.elevation,
|
||||
tz: v.tz,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user