NilsBriggen
b6f62679c9
- Persistence via Node's built-in node:sqlite (zero native deps) on a Docker volume: response cache, source health, fixture↔provider id map, per-fixture enrichment, ingest log. Runtime bumped to node:24-slim + --experimental-sqlite. - Resilient fetcher: DB cache + per-source rate-limit + jittered backoff + circuit-breaker (blocked sources trip + skip; UI reads DB, never breaks). - ESPN hidden API as the PRIMARY rich source (works from the VPS where SofaScore 403s): scoreboard (live scores w/ clock) + summary (venue, H2H, recent form, lineups, team stats). football-data / SofaScore are fallbacks. - Scheduler: maps all 72 group fixtures to ESPN event ids, polls live on a dynamic cadence, enriches imminent fixtures into match_ext. /api/sources/health. - Verified: DB populates (real H2H + form for opening matches), 22 tests pass, Docker image runs node:sqlite on the volume and persists across restart. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
89 lines
3.1 KiB
TypeScript
89 lines
3.1 KiB
TypeScript
import { cacheGet, cacheSet, healthGet, healthRecord } from '../db/db';
|
|
|
|
// One resilient fetch path for every source: DB-cached, rate-limited, and
|
|
// circuit-broken. A blocked source (e.g. SofaScore 403 from the VPS) trips its
|
|
// breaker and is skipped; callers transparently get stale cache or a thrown
|
|
// error they already handle — the UI, which reads the DB, never blocks.
|
|
|
|
const CIRCUIT_THRESHOLD = 4; // consecutive failures before opening
|
|
const CIRCUIT_COOLDOWN = 10 * 60_000; // base cooldown, escalates with failures
|
|
|
|
const lastCall = new Map<string, number>();
|
|
|
|
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
|
|
export function browserHeaders(extra: Record<string, string> = {}): Record<string, string> {
|
|
return {
|
|
'User-Agent':
|
|
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
|
Accept: 'application/json, text/plain, */*',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
...extra,
|
|
};
|
|
}
|
|
|
|
export interface FetchOpts {
|
|
/** Cache freshness window (ms). */
|
|
ttl: number;
|
|
/** Minimum interval between calls to this source (ms). */
|
|
minInterval?: number;
|
|
headers?: Record<string, string>;
|
|
timeoutMs?: number;
|
|
/** Parse as text instead of JSON. */
|
|
text?: boolean;
|
|
}
|
|
|
|
export function circuitOpen(source: string): boolean {
|
|
const h = healthGet(source);
|
|
return !!h && h.open_until > Date.now();
|
|
}
|
|
|
|
async function rateLimit(source: string, minInterval: number): Promise<void> {
|
|
if (!minInterval) return;
|
|
const last = lastCall.get(source) ?? 0;
|
|
const wait = minInterval - (Date.now() - last);
|
|
if (wait > 0) await sleep(wait + Math.random() * 250); // jitter
|
|
lastCall.set(source, Date.now());
|
|
}
|
|
|
|
/**
|
|
* Fetch JSON (or text) for `source` at `url`, keyed in the cache by `key`.
|
|
* Returns fresh cache when valid, otherwise fetches; on failure returns stale
|
|
* cache if present, else throws.
|
|
*/
|
|
export async function cachedFetch<T = unknown>(
|
|
source: string,
|
|
key: string,
|
|
url: string,
|
|
opts: FetchOpts,
|
|
): Promise<T> {
|
|
const cached = cacheGet(key);
|
|
if (cached?.fresh) return (opts.text ? cached.value : JSON.parse(cached.value)) as T;
|
|
|
|
if (circuitOpen(source)) {
|
|
if (cached) return (opts.text ? cached.value : JSON.parse(cached.value)) as T;
|
|
throw new Error(`${source}: circuit open`);
|
|
}
|
|
|
|
await rateLimit(source, opts.minInterval ?? 0);
|
|
|
|
try {
|
|
const res = await fetch(url, {
|
|
headers: browserHeaders(opts.headers),
|
|
signal: AbortSignal.timeout(opts.timeoutMs ?? 12_000),
|
|
});
|
|
if (!res.ok) throw new Error(`${res.status}`);
|
|
const body = await res.text();
|
|
cacheSet(key, body, opts.ttl);
|
|
healthRecord(source, true);
|
|
return (opts.text ? body : JSON.parse(body)) as T;
|
|
} catch (e) {
|
|
const note = e instanceof Error ? e.message : String(e);
|
|
const consec = (healthGet(source)?.consec_fail ?? 0) + 1;
|
|
const openMs = consec >= CIRCUIT_THRESHOLD ? CIRCUIT_COOLDOWN * Math.min(6, consec - CIRCUIT_THRESHOLD + 1) : undefined;
|
|
healthRecord(source, false, note, openMs);
|
|
if (cached) return (opts.text ? cached.value : JSON.parse(cached.value)) as T; // serve stale
|
|
throw new Error(`${source} fetch failed: ${note}`);
|
|
}
|
|
}
|