Storage: configurable data dir for the external drive + daily DB prune
HOST_DATA_DIR in .env can now point /data at a host path (the 1TB external drive) instead of the named volume, so odds history, caches and snapshots stop eating the 128GB root disk. A daily prune drops cache entries stale for over a week and ingest-log rows older than 14 days — faster polling no longer means unbounded DB growth. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -135,6 +135,19 @@ export function cacheSet(key: string, value: string, ttl: number): void {
|
||||
.run(key, value, Date.now(), ttl);
|
||||
}
|
||||
|
||||
/** Bound DB growth: drop cache entries stale for >7 days (the breaker only ever
|
||||
* serves recent staleness) and ingest-log rows older than 14 days. Returns the
|
||||
* number of rows removed. Runs daily from the scheduler. */
|
||||
export function prune(now = Date.now()): number {
|
||||
const cache = db()
|
||||
.prepare('DELETE FROM kv_cache WHERE ? - fetched_at > ttl + 7 * 86400000')
|
||||
.run(now);
|
||||
const log = db()
|
||||
.prepare('DELETE FROM ingest_log WHERE at < ?')
|
||||
.run(now - 14 * 86400000);
|
||||
return Number(cache.changes) + Number(log.changes);
|
||||
}
|
||||
|
||||
// ---- source health / circuit breaker ----
|
||||
export interface SourceHealthRow {
|
||||
source: string;
|
||||
|
||||
@@ -4,7 +4,7 @@ import { fetchEspnScoreboard, fetchEspnSummary } from './espn';
|
||||
import { fetchEspnOdds } from './espnOdds';
|
||||
import { fetchFootballData } from '../footballData';
|
||||
import { fetchSofascore } from '../sofascore';
|
||||
import { setSourceMap, getSourceMap, setMatchExt, recordOdds, logIngest } from '../db/db';
|
||||
import { setSourceMap, getSourceMap, setMatchExt, recordOdds, logIngest, prune } from '../db/db';
|
||||
import type { Fixture } from '../../../src/lib/types';
|
||||
|
||||
// The ingestion orchestrator. Replaces the old live loop: ESPN is the primary
|
||||
@@ -161,6 +161,14 @@ export function startScheduler(
|
||||
oddsTimer = setTimeout(async () => { await oddsTick().catch(() => {}); scheduleOdds(); }, ODDS_MS);
|
||||
};
|
||||
|
||||
// Daily prune keeps the faster polling from growing kv_cache/ingest_log forever.
|
||||
const pruneTimer = setInterval(() => {
|
||||
try {
|
||||
const n = prune();
|
||||
if (n) console.log(`[db] pruned ${n} stale cache/log rows`);
|
||||
} catch { /* non-fatal */ }
|
||||
}, 24 * 60 * 60 * 1000);
|
||||
|
||||
const scheduleLive = (): void => {
|
||||
if (stopped) return;
|
||||
const delay = pollDelay(state.hasLive(), state.hasLiveActivity(LIVE_WINDOW_MS));
|
||||
@@ -188,5 +196,6 @@ export function startScheduler(
|
||||
if (liveTimer) clearTimeout(liveTimer);
|
||||
if (enrichTimer) clearTimeout(enrichTimer);
|
||||
if (oddsTimer) clearTimeout(oddsTimer);
|
||||
clearInterval(pruneTimer);
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user