Nightly DB snapshots to the 1TB storage box (live DB stays local)

The 'external drive' turned out to be a CIFS-mounted Hetzner storage
box — SQLite cannot run on a network mount (WAL shared memory and
locking break), and the live DB is only ~7MB anyway. So: the hot
database stays on the local volume, and a nightly VACUUM INTO snapshot
(transactionally consistent, a plain sequential file write — perfect
for CIFS) lands in /data/archive, which compose binds to the storage
box via HOST_ARCHIVE_DIR. 14-day retention, plus a snapshot at boot.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-06-12 00:36:05 +02:00
parent 716dab2f23
commit ac1de66e6d
3 changed files with 40 additions and 9 deletions
+9 -4
View File
@@ -4,7 +4,7 @@ import { fetchEspnScoreboard, fetchEspnSummary } from './espn';
import { fetchEspnOdds } from './espnOdds';
import { fetchFootballData } from '../footballData';
import { fetchSofascore } from '../sofascore';
import { setSourceMap, getSourceMap, setMatchExt, recordOdds, logIngest, prune } from '../db/db';
import { setSourceMap, getSourceMap, setMatchExt, recordOdds, logIngest, prune, backupDb } from '../db/db';
import type { Fixture } from '../../../src/lib/types';
// The ingestion orchestrator. Replaces the old live loop: ESPN is the primary
@@ -161,13 +161,17 @@ export function startScheduler(
oddsTimer = setTimeout(async () => { await oddsTick().catch(() => {}); scheduleOdds(); }, ODDS_MS);
};
// Daily prune keeps the faster polling from growing kv_cache/ingest_log forever.
const pruneTimer = setInterval(() => {
// Daily housekeeping: prune keeps the faster polling from growing
// kv_cache/ingest_log forever; backupDb snapshots to the archive dir
// (the storage box in prod) so history survives anything.
const housekeeping = (): void => {
try {
const n = prune();
if (n) console.log(`[db] pruned ${n} stale cache/log rows`);
} catch { /* non-fatal */ }
}, 24 * 60 * 60 * 1000);
backupDb();
};
const pruneTimer = setInterval(housekeeping, 24 * 60 * 60 * 1000);
const scheduleLive = (): void => {
if (stopped) return;
@@ -189,6 +193,7 @@ export function startScheduler(
scheduleLive();
scheduleEnrich();
scheduleOdds();
backupDb(); // boot snapshot — the daily timer covers the rest
})();
return () => {