diff --git a/deploy/cup26.compose.yml b/deploy/cup26.compose.yml index 98e0b9f..9cf1581 100644 --- a/deploy/cup26.compose.yml +++ b/deploy/cup26.compose.yml @@ -20,12 +20,14 @@ services: # opt-in unofficial sources (usually blocked from a datacenter IP; ESPN is primary) - ENABLE_SOFASCORE=${ENABLE_SOFASCORE:-false} - ENABLE_FOTMOB=${ENABLE_FOTMOB:-false} + - ARCHIVE_DIR=/data/archive volumes: - # HOST_DATA_DIR (in .env) may point at a host path — e.g. the external - # 1TB drive — so the growing SQLite DB stays off the small root disk. - # An absolute path = bind mount; unset = the original named volume. - # The container runs as node (uid 1000): chown 1000:1000 the host dir. + # The LIVE SQLite must stay on a real local filesystem (WAL + locking do + # not survive network mounts). HOST_DATA_DIR can override if ever needed. - ${HOST_DATA_DIR:-cup26-data}:/data + # Nightly VACUUM INTO snapshots land here — point HOST_ARCHIVE_DIR at the + # 1TB storage box (CIFS is fine for sequential snapshot files). + - ${HOST_ARCHIVE_DIR:-cup26-archive}:/data/archive security_opt: - no-new-privileges:true mem_limit: 384m @@ -43,6 +45,7 @@ services: volumes: cup26-data: + cup26-archive: networks: proxy: diff --git a/server/src/db/db.ts b/server/src/db/db.ts index e915133..8bb40b8 100644 --- a/server/src/db/db.ts +++ b/server/src/db/db.ts @@ -1,5 +1,5 @@ import { DatabaseSync } from 'node:sqlite'; -import { mkdirSync } from 'node:fs'; +import { mkdirSync, readdirSync, rmSync } from 'node:fs'; import path from 'node:path'; // Persistence via Node's built-in SQLite (no native deps). One file on a Docker @@ -135,6 +135,29 @@ export function cacheSet(key: string, value: string, ttl: number): void { .run(key, value, Date.now(), ttl); } +/** Nightly consistent snapshot via VACUUM INTO (transaction-safe, works on any + * filesystem — the archive dir may be a CIFS-mounted storage box, where the + * LIVE database could never run). Keeps the newest `keep` dated copies. */ +export function backupDb(keep = 14): string | null { + const dir = process.env.ARCHIVE_DIR ?? path.join(DATA_DIR, 'archive'); + try { + mkdirSync(dir, { recursive: true }); + const stamp = new Date().toISOString().slice(0, 10); + const dest = path.join(dir, `cup26-${stamp}.db`); + rmSync(dest, { force: true }); // VACUUM INTO refuses to overwrite + db().exec(`VACUUM INTO '${dest.replaceAll("'", "''")}'`); + const files = readdirSync(dir).filter((f) => /^cup26-\d{4}-\d{2}-\d{2}\.db$/.test(f)).sort(); + for (const f of files.slice(0, Math.max(0, files.length - keep))) { + rmSync(path.join(dir, f), { force: true }); + } + console.log(`[db] snapshot → ${dest}`); + return dest; + } catch (e) { + console.error('[db] backup failed:', e instanceof Error ? e.message : e); + return null; + } +} + /** Bound DB growth: drop cache entries stale for >7 days (the breaker only ever * serves recent staleness) and ingest-log rows older than 14 days. Returns the * number of rows removed. Runs daily from the scheduler. */ diff --git a/server/src/ingest/scheduler.ts b/server/src/ingest/scheduler.ts index cc6b8a9..4363647 100644 --- a/server/src/ingest/scheduler.ts +++ b/server/src/ingest/scheduler.ts @@ -4,7 +4,7 @@ import { fetchEspnScoreboard, fetchEspnSummary } from './espn'; import { fetchEspnOdds } from './espnOdds'; import { fetchFootballData } from '../footballData'; import { fetchSofascore } from '../sofascore'; -import { setSourceMap, getSourceMap, setMatchExt, recordOdds, logIngest, prune } from '../db/db'; +import { setSourceMap, getSourceMap, setMatchExt, recordOdds, logIngest, prune, backupDb } from '../db/db'; import type { Fixture } from '../../../src/lib/types'; // The ingestion orchestrator. Replaces the old live loop: ESPN is the primary @@ -161,13 +161,17 @@ export function startScheduler( oddsTimer = setTimeout(async () => { await oddsTick().catch(() => {}); scheduleOdds(); }, ODDS_MS); }; - // Daily prune keeps the faster polling from growing kv_cache/ingest_log forever. - const pruneTimer = setInterval(() => { + // Daily housekeeping: prune keeps the faster polling from growing + // kv_cache/ingest_log forever; backupDb snapshots to the archive dir + // (the storage box in prod) so history survives anything. + const housekeeping = (): void => { try { const n = prune(); if (n) console.log(`[db] pruned ${n} stale cache/log rows`); } catch { /* non-fatal */ } - }, 24 * 60 * 60 * 1000); + backupDb(); + }; + const pruneTimer = setInterval(housekeeping, 24 * 60 * 60 * 1000); const scheduleLive = (): void => { if (stopped) return; @@ -189,6 +193,7 @@ export function startScheduler( scheduleLive(); scheduleEnrich(); scheduleOdds(); + backupDb(); // boot snapshot — the daily timer covers the rest })(); return () => {