Storage: configurable data dir for the external drive + daily DB prune
HOST_DATA_DIR in .env can now point /data at a host path (the 1TB external drive) instead of the named volume, so odds history, caches and snapshots stop eating the 128GB root disk. A daily prune drops cache entries stale for over a week and ingest-log rows older than 14 days — faster polling no longer means unbounded DB growth. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -21,7 +21,11 @@ services:
|
|||||||
- ENABLE_SOFASCORE=${ENABLE_SOFASCORE:-false}
|
- ENABLE_SOFASCORE=${ENABLE_SOFASCORE:-false}
|
||||||
- ENABLE_FOTMOB=${ENABLE_FOTMOB:-false}
|
- ENABLE_FOTMOB=${ENABLE_FOTMOB:-false}
|
||||||
volumes:
|
volumes:
|
||||||
- cup26-data:/data
|
# HOST_DATA_DIR (in .env) may point at a host path — e.g. the external
|
||||||
|
# 1TB drive — so the growing SQLite DB stays off the small root disk.
|
||||||
|
# An absolute path = bind mount; unset = the original named volume.
|
||||||
|
# The container runs as node (uid 1000): chown 1000:1000 the host dir.
|
||||||
|
- ${HOST_DATA_DIR:-cup26-data}:/data
|
||||||
security_opt:
|
security_opt:
|
||||||
- no-new-privileges:true
|
- no-new-privileges:true
|
||||||
mem_limit: 384m
|
mem_limit: 384m
|
||||||
|
|||||||
@@ -135,6 +135,19 @@ export function cacheSet(key: string, value: string, ttl: number): void {
|
|||||||
.run(key, value, Date.now(), ttl);
|
.run(key, value, Date.now(), ttl);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Bound DB growth: drop cache entries stale for >7 days (the breaker only ever
|
||||||
|
* serves recent staleness) and ingest-log rows older than 14 days. Returns the
|
||||||
|
* number of rows removed. Runs daily from the scheduler. */
|
||||||
|
export function prune(now = Date.now()): number {
|
||||||
|
const cache = db()
|
||||||
|
.prepare('DELETE FROM kv_cache WHERE ? - fetched_at > ttl + 7 * 86400000')
|
||||||
|
.run(now);
|
||||||
|
const log = db()
|
||||||
|
.prepare('DELETE FROM ingest_log WHERE at < ?')
|
||||||
|
.run(now - 14 * 86400000);
|
||||||
|
return Number(cache.changes) + Number(log.changes);
|
||||||
|
}
|
||||||
|
|
||||||
// ---- source health / circuit breaker ----
|
// ---- source health / circuit breaker ----
|
||||||
export interface SourceHealthRow {
|
export interface SourceHealthRow {
|
||||||
source: string;
|
source: string;
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import { fetchEspnScoreboard, fetchEspnSummary } from './espn';
|
|||||||
import { fetchEspnOdds } from './espnOdds';
|
import { fetchEspnOdds } from './espnOdds';
|
||||||
import { fetchFootballData } from '../footballData';
|
import { fetchFootballData } from '../footballData';
|
||||||
import { fetchSofascore } from '../sofascore';
|
import { fetchSofascore } from '../sofascore';
|
||||||
import { setSourceMap, getSourceMap, setMatchExt, recordOdds, logIngest } from '../db/db';
|
import { setSourceMap, getSourceMap, setMatchExt, recordOdds, logIngest, prune } from '../db/db';
|
||||||
import type { Fixture } from '../../../src/lib/types';
|
import type { Fixture } from '../../../src/lib/types';
|
||||||
|
|
||||||
// The ingestion orchestrator. Replaces the old live loop: ESPN is the primary
|
// The ingestion orchestrator. Replaces the old live loop: ESPN is the primary
|
||||||
@@ -161,6 +161,14 @@ export function startScheduler(
|
|||||||
oddsTimer = setTimeout(async () => { await oddsTick().catch(() => {}); scheduleOdds(); }, ODDS_MS);
|
oddsTimer = setTimeout(async () => { await oddsTick().catch(() => {}); scheduleOdds(); }, ODDS_MS);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Daily prune keeps the faster polling from growing kv_cache/ingest_log forever.
|
||||||
|
const pruneTimer = setInterval(() => {
|
||||||
|
try {
|
||||||
|
const n = prune();
|
||||||
|
if (n) console.log(`[db] pruned ${n} stale cache/log rows`);
|
||||||
|
} catch { /* non-fatal */ }
|
||||||
|
}, 24 * 60 * 60 * 1000);
|
||||||
|
|
||||||
const scheduleLive = (): void => {
|
const scheduleLive = (): void => {
|
||||||
if (stopped) return;
|
if (stopped) return;
|
||||||
const delay = pollDelay(state.hasLive(), state.hasLiveActivity(LIVE_WINDOW_MS));
|
const delay = pollDelay(state.hasLive(), state.hasLiveActivity(LIVE_WINDOW_MS));
|
||||||
@@ -188,5 +196,6 @@ export function startScheduler(
|
|||||||
if (liveTimer) clearTimeout(liveTimer);
|
if (liveTimer) clearTimeout(liveTimer);
|
||||||
if (enrichTimer) clearTimeout(enrichTimer);
|
if (enrichTimer) clearTimeout(enrichTimer);
|
||||||
if (oddsTimer) clearTimeout(oddsTimer);
|
if (oddsTimer) clearTimeout(oddsTimer);
|
||||||
|
clearInterval(pruneTimer);
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user