diff --git a/deploy/cup26.compose.yml b/deploy/cup26.compose.yml index 5c2e36d..98e0b9f 100644 --- a/deploy/cup26.compose.yml +++ b/deploy/cup26.compose.yml @@ -21,7 +21,11 @@ services: - ENABLE_SOFASCORE=${ENABLE_SOFASCORE:-false} - ENABLE_FOTMOB=${ENABLE_FOTMOB:-false} volumes: - - cup26-data:/data + # HOST_DATA_DIR (in .env) may point at a host path — e.g. the external + # 1TB drive — so the growing SQLite DB stays off the small root disk. + # An absolute path = bind mount; unset = the original named volume. + # The container runs as node (uid 1000): chown 1000:1000 the host dir. + - ${HOST_DATA_DIR:-cup26-data}:/data security_opt: - no-new-privileges:true mem_limit: 384m diff --git a/server/src/db/db.ts b/server/src/db/db.ts index 0ed2712..e915133 100644 --- a/server/src/db/db.ts +++ b/server/src/db/db.ts @@ -135,6 +135,19 @@ export function cacheSet(key: string, value: string, ttl: number): void { .run(key, value, Date.now(), ttl); } +/** Bound DB growth: drop cache entries stale for >7 days (the breaker only ever + * serves recent staleness) and ingest-log rows older than 14 days. Returns the + * number of rows removed. Runs daily from the scheduler. */ +export function prune(now = Date.now()): number { + const cache = db() + .prepare('DELETE FROM kv_cache WHERE ? - fetched_at > ttl + 7 * 86400000') + .run(now); + const log = db() + .prepare('DELETE FROM ingest_log WHERE at < ?') + .run(now - 14 * 86400000); + return Number(cache.changes) + Number(log.changes); +} + // ---- source health / circuit breaker ---- export interface SourceHealthRow { source: string; diff --git a/server/src/ingest/scheduler.ts b/server/src/ingest/scheduler.ts index 7b86582..cc6b8a9 100644 --- a/server/src/ingest/scheduler.ts +++ b/server/src/ingest/scheduler.ts @@ -4,7 +4,7 @@ import { fetchEspnScoreboard, fetchEspnSummary } from './espn'; import { fetchEspnOdds } from './espnOdds'; import { fetchFootballData } from '../footballData'; import { fetchSofascore } from '../sofascore'; -import { setSourceMap, getSourceMap, setMatchExt, recordOdds, logIngest } from '../db/db'; +import { setSourceMap, getSourceMap, setMatchExt, recordOdds, logIngest, prune } from '../db/db'; import type { Fixture } from '../../../src/lib/types'; // The ingestion orchestrator. Replaces the old live loop: ESPN is the primary @@ -161,6 +161,14 @@ export function startScheduler( oddsTimer = setTimeout(async () => { await oddsTick().catch(() => {}); scheduleOdds(); }, ODDS_MS); }; + // Daily prune keeps the faster polling from growing kv_cache/ingest_log forever. + const pruneTimer = setInterval(() => { + try { + const n = prune(); + if (n) console.log(`[db] pruned ${n} stale cache/log rows`); + } catch { /* non-fatal */ } + }, 24 * 60 * 60 * 1000); + const scheduleLive = (): void => { if (stopped) return; const delay = pollDelay(state.hasLive(), state.hasLiveActivity(LIVE_WINDOW_MS)); @@ -188,5 +196,6 @@ export function startScheduler( if (liveTimer) clearTimeout(liveTimer); if (enrichTimer) clearTimeout(enrichTimer); if (oddsTimer) clearTimeout(oddsTimer); + clearInterval(pruneTimer); }; }