Storage: configurable data dir for the external drive + daily DB prune

HOST_DATA_DIR in .env can now point /data at a host path (the 1TB
external drive) instead of the named volume, so odds history, caches
and snapshots stop eating the 128GB root disk. A daily prune drops
cache entries stale for over a week and ingest-log rows older than 14
days — faster polling no longer means unbounded DB growth.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-06-12 00:10:54 +02:00
parent d3e8df96ef
commit 92fdfdf398
3 changed files with 28 additions and 2 deletions
+5 -1
View File
@@ -21,7 +21,11 @@ services:
- ENABLE_SOFASCORE=${ENABLE_SOFASCORE:-false}
- ENABLE_FOTMOB=${ENABLE_FOTMOB:-false}
volumes:
- cup26-data:/data
# HOST_DATA_DIR (in .env) may point at a host path — e.g. the external
# 1TB drive — so the growing SQLite DB stays off the small root disk.
# An absolute path = bind mount; unset = the original named volume.
# The container runs as node (uid 1000): chown 1000:1000 the host dir.
- ${HOST_DATA_DIR:-cup26-data}:/data
security_opt:
- no-new-privileges:true
mem_limit: 384m
+13
View File
@@ -135,6 +135,19 @@ export function cacheSet(key: string, value: string, ttl: number): void {
.run(key, value, Date.now(), ttl);
}
/** Bound DB growth: drop cache entries stale for >7 days (the breaker only ever
* serves recent staleness) and ingest-log rows older than 14 days. Returns the
* number of rows removed. Runs daily from the scheduler. */
export function prune(now = Date.now()): number {
const cache = db()
.prepare('DELETE FROM kv_cache WHERE ? - fetched_at > ttl + 7 * 86400000')
.run(now);
const log = db()
.prepare('DELETE FROM ingest_log WHERE at < ?')
.run(now - 14 * 86400000);
return Number(cache.changes) + Number(log.changes);
}
// ---- source health / circuit breaker ----
export interface SourceHealthRow {
source: string;
+10 -1
View File
@@ -4,7 +4,7 @@ import { fetchEspnScoreboard, fetchEspnSummary } from './espn';
import { fetchEspnOdds } from './espnOdds';
import { fetchFootballData } from '../footballData';
import { fetchSofascore } from '../sofascore';
import { setSourceMap, getSourceMap, setMatchExt, recordOdds, logIngest } from '../db/db';
import { setSourceMap, getSourceMap, setMatchExt, recordOdds, logIngest, prune } from '../db/db';
import type { Fixture } from '../../../src/lib/types';
// The ingestion orchestrator. Replaces the old live loop: ESPN is the primary
@@ -161,6 +161,14 @@ export function startScheduler(
oddsTimer = setTimeout(async () => { await oddsTick().catch(() => {}); scheduleOdds(); }, ODDS_MS);
};
// Daily prune keeps the faster polling from growing kv_cache/ingest_log forever.
const pruneTimer = setInterval(() => {
try {
const n = prune();
if (n) console.log(`[db] pruned ${n} stale cache/log rows`);
} catch { /* non-fatal */ }
}, 24 * 60 * 60 * 1000);
const scheduleLive = (): void => {
if (stopped) return;
const delay = pollDelay(state.hasLive(), state.hasLiveActivity(LIVE_WINDOW_MS));
@@ -188,5 +196,6 @@ export function startScheduler(
if (liveTimer) clearTimeout(liveTimer);
if (enrichTimer) clearTimeout(enrichTimer);
if (oddsTimer) clearTimeout(oddsTimer);
clearInterval(pruneTimer);
};
}