v3: the lake processed + /data page + glossary terms

- scripts/processStatsbomb.ts: one pass over the 17GB StatsBomb lake (4,235 matches) -> lakestats.json (12.82GB events / 14.9M events / 6,919 players), scorestate.json (goal-rate multipliers by game state — shipped as an insight, NOT wired into in-play: the raw effect is selection-biased toward stronger teams and would distort a calibrated model), fingerprints.json (40/48 nations' shots/xG/set-piece share from real event data). Artifacts committed; raw lake stays local (gitignored). - /data page + /api/data/stats: the gigabytes made visible — lake counters, historical archive, live-collection counts (odds lines, frozen forecasts, enrichment), per-source health with freshness. Footer links. - Term component: plain-language glossary tooltips (RPS, Brier, ECE, Elo, xG, de-vig, ensemble) attached where metrics appear. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-11 17:41:43 +02:00
parent dc52a908c0
commit dda8e1ae42
13 changed files with 430 additions and 7 deletions
@@ -117,8 +117,9 @@ export function RootLayout() {

      <footer className="border-t border-line py-6 pb-24 text-center text-xs text-faint sm:pb-6">
        <p>
-          Cup26 · World Cup 2026 · data from openfootball, football-data.org &amp; StatsBomb open data ·
-          model-driven odds, not betting advice
+          Cup26 · World Cup 2026 · <Link to="/data" className="underline decoration-line underline-offset-2 hover:text-ink">all the data</Link> ·{' '}
+          <Link to="/methodology" className="underline decoration-line underline-offset-2 hover:text-ink">how the model works</Link> ·
+          model odds, not betting advice
        </p>
      </footer>
    </div>
@@ -0,0 +1,25 @@
+import type { ReactNode } from 'react';
+
+// Plain-language explainers for every metric, attached where the metric appears.
+const GLOSSARY: Record<string, string> = {
+  RPS: 'Ranked Probability Score — how far the forecast probabilities were from what actually happened, respecting that home win / draw / away win are ordered. 0 is perfect; lower is better.',
+  Brier: 'Mean squared error of the probabilities across the three outcomes. 0 is perfect; lower is better.',
+  ECE: 'Expected Calibration Error — the average gap between predicted probability and observed frequency. Under 0.05 is excellent.',
+  Elo: 'A strength rating updated after every match: beat a stronger team, gain more points. A 100-point gap ≈ 64% expected score.',
+  xG: 'Expected goals — the probability a shot becomes a goal, judged from historical shots like it. Total xG measures chance quality, not luck.',
+  'de-vig': "Bookmaker odds include a profit margin, so their implied probabilities sum past 100%. De-vigging rescales them to a fair 100% — the bookmaker's true opinion.",
+  ensemble: 'Two independent goal models (Elo-based and team attack/defence) blended by a weight chosen on validation data — the blend beat both members out-of-sample.',
+};
+
+/** A dotted-underline term with a plain-language tooltip from the glossary. */
+export function Term({ k, children }: { k: keyof typeof GLOSSARY | string; children?: ReactNode }) {
+  return (
+    <span
+      tabIndex={0}
+      title={GLOSSARY[k] ?? k}
+      className="cursor-help underline decoration-line decoration-dotted underline-offset-2"
+    >
+      {children ?? k}
+    </span>
+  );
+}
@@ -0,0 +1,153 @@
+import { useEffect, useState } from 'react';
+import { Database, HardDrive, Radio, Scale } from 'lucide-react';
+import { PageHeader } from '@/components/ui/PageHeader';
+import { Card, CardBody, CardHeader } from '@/components/ui/Card';
+import { Badge } from '@/components/ui/Badge';
+
+interface LakeStats {
+  bytes: number; events: number; matchesParsed: number; players: number;
+  eventFiles: number; competitionSeasons: number; nationalTeamsCovered: number; generatedAt: string;
+}
+interface DataStats {
+  lake: LakeStats | null;
+  archive: { results: number; asOf: string; h2hPairings: number } | null;
+  fingerprints: number | null;
+  squadValues: number | null;
+  db: Record<string, number>;
+  sources: { source: string; last_ok: number | null; consec_fail: number; open_until: number }[];
+}
+
+const fmt = (n: number) => n.toLocaleString();
+const gb = (b: number) => `${(b / 1e9).toFixed(2)} GB`;
+const ago = (t: number | null) => {
+  if (!t) return 'never';
+  const m = Math.round((Date.now() - t) / 60000);
+  return m < 1 ? 'just now' : m < 60 ? `${m} min ago` : `${Math.round(m / 60)} h ago`;
+};
+
+function Big({ value, label }: { value: string; label: string }) {
+  return (
+    <div className="text-center">
+      <div className="font-display text-3xl font-extrabold text-accent">{value}</div>
+      <div className="mt-0.5 text-[11px] uppercase tracking-wide text-faint">{label}</div>
+    </div>
+  );
+}
+
+function Counter({ label, value }: { label: string; value: string }) {
+  return (
+    <div className="flex items-center justify-between border-b border-line/60 py-1.5 text-sm last:border-0">
+      <span className="text-muted">{label}</span>
+      <span className="tnum font-semibold text-ink">{value}</span>
+    </div>
+  );
+}
+
+export function DataPage() {
+  const [stats, setStats] = useState<DataStats | null>(null);
+
+  useEffect(() => {
+    let alive = true;
+    fetch('/api/data/stats').then((r) => r.json()).then((d: DataStats) => alive && setStats(d)).catch(() => {});
+    return () => { alive = false; };
+  }, []);
+
+  if (!stats) return <div className="h-96 animate-pulse rounded-xl border border-line bg-panel" />;
+  const lake = stats.lake;
+
+  return (
+    <div className="space-y-5">
+      <PageHeader
+        title="The data"
+        subtitle="Everything this site runs on — sized, sourced and timestamped. No black boxes."
+      />
+
+      {lake && (
+        <Card>
+          <CardHeader className="flex items-center gap-2">
+            <HardDrive size={16} className="text-accent" />
+            <span className="font-display font-bold text-ink">Event-data lake</span>
+            <Badge tone="muted">StatsBomb open data</Badge>
+          </CardHeader>
+          <CardBody>
+            <div className="grid grid-cols-2 gap-4 sm:grid-cols-4">
+              <Big value={gb(lake.bytes)} label="event JSON parsed" />
+              <Big value={`${(lake.events / 1e6).toFixed(1)}M`} label="on-pitch events" />
+              <Big value={fmt(lake.matchesParsed)} label="matches" />
+              <Big value={fmt(lake.players)} label="players" />
+            </div>
+            <p className="mt-4 text-sm text-muted">
+              Every pass, shot and duel from {lake.competitionSeasons} competition-seasons (eight World Cups among them),
+              processed into the style fingerprints ({lake.nationalTeamsCovered} of 48 nations covered) and the
+              score-state analysis shown on the methodology page.
+            </p>
+          </CardBody>
+        </Card>
+      )}
+
+      <div className="grid gap-4 md:grid-cols-2">
+        <Card>
+          <CardHeader className="flex items-center gap-2">
+            <Database size={16} className="text-accent" />
+            <span className="font-display font-bold text-ink">Historical archive</span>
+          </CardHeader>
+          <CardBody>
+            {stats.archive && (
+              <>
+                <Counter label="International results (1872 → now)" value={fmt(stats.archive.results)} />
+                <Counter label="Head-to-head pairings precomputed" value={fmt(stats.archive.h2hPairings)} />
+                <Counter label="Ratings current through" value={stats.archive.asOf} />
+              </>
+            )}
+            {stats.squadValues != null && <Counter label="Squad market values (Transfermarkt)" value={`${stats.squadValues} teams`} />}
+            {stats.fingerprints != null && <Counter label="Event-data team fingerprints" value={`${stats.fingerprints} teams`} />}
+          </CardBody>
+        </Card>
+
+        <Card>
+          <CardHeader className="flex items-center gap-2">
+            <Radio size={16} className="text-accent" />
+            <span className="font-display font-bold text-ink">Live collection (this tournament)</span>
+          </CardHeader>
+          <CardBody>
+            <Counter label="Bookmaker lines captured" value={fmt(stats.db.oddsLines ?? 0)} />
+            <Counter label="Fixtures with market odds" value={fmt(stats.db.oddsFixtures ?? 0)} />
+            <Counter label="Forecasts frozen pre-kickoff" value={fmt(stats.db.frozenForecasts ?? 0)} />
+            <Counter label="Matches enriched (form, H2H, lineups)" value={fmt(stats.db.enrichedMatches ?? 0)} />
+            <Counter label="API responses cached" value={fmt(stats.db.cachedResponses ?? 0)} />
+            <Counter label="Ingestion runs logged" value={fmt(stats.db.ingestEvents ?? 0)} />
+          </CardBody>
+        </Card>
+      </div>
+
+      <Card>
+        <CardHeader className="flex items-center gap-2">
+          <Scale size={16} className="text-accent" />
+          <span className="font-display font-bold text-ink">Source health</span>
+        </CardHeader>
+        <CardBody>
+          {stats.sources.length ? stats.sources.map((s) => (
+            <div key={s.source} className="flex items-center justify-between border-b border-line/60 py-1.5 text-sm last:border-0">
+              <span className="font-medium text-ink">{s.source}</span>
+              <span className="flex items-center gap-2">
+                {s.open_until > Date.now() ? (
+                  <Badge tone="muted">paused (circuit open)</Badge>
+                ) : s.consec_fail > 0 ? (
+                  <Badge tone="gold">{s.consec_fail} recent failure{s.consec_fail > 1 ? 's' : ''}</Badge>
+                ) : (
+                  <Badge tone="accent">healthy</Badge>
+                )}
+                <span className="text-xs text-faint">last OK {ago(s.last_ok)}</span>
+              </span>
+            </div>
+          )) : <p className="text-sm text-faint">No live sources polled yet.</p>}
+          <p className="mt-3 border-t border-line pt-2 text-xs text-faint">
+            Sources: openfootball (fixtures) · ESPN (live scores, form, lineups, bookmaker lines) · football-data.org (fallback) ·
+            martj42 results archive · StatsBomb open data · Transfermarkt (squad values). All scraping is server-side, cached and
+            rate-limited; a blocked source degrades to stale data, never a broken page.
+          </p>
+        </CardBody>
+      </Card>
+    </div>
+  );
+}
@@ -2,6 +2,7 @@ import { useEffect, useState } from 'react';
 import { Activity, Dices, Gauge, TrendingUp } from 'lucide-react';
 import { PageHeader } from '@/components/ui/PageHeader';
 import { Card, CardBody, CardHeader } from '@/components/ui/Card';
+import { Term } from '@/components/ui/Term';
 import { ReliabilityDiagram } from '@/components/ReliabilityDiagram';
 import type { BacktestReport } from '@/lib/types';

@@ -80,6 +81,9 @@ export function MethodologyPage() {
                <Stat value={bt.model.rps.toFixed(3)} label="ranked prob. score" />
                <Stat value={`${(bt.worldCup.accuracy * 100).toFixed(0)}%`} label={`World Cup acc. (${bt.worldCup.tested})`} />
              </div>
+              <p className="text-xs text-faint">
+                Metrics: <Term k="RPS" /> · <Term k="Brier" /> · <Term k="ECE" /> · the shipped model is an <Term k="ensemble" />.
+              </p>
              <div>
                <div className="mb-2 text-xs uppercase tracking-wide text-faint">Ranked Probability Score — lower is better, vs candidates &amp; baselines</div>
                <div className="space-y-1.5">
@@ -3,6 +3,7 @@ import { Link } from '@tanstack/react-router';
 import { Scale } from 'lucide-react';
 import { PageHeader } from '@/components/ui/PageHeader';
 import { Card, CardBody, CardHeader } from '@/components/ui/Card';
+import { Term } from '@/components/ui/Term';
 import { teamFlag } from '@/lib/teams';
 import { kickoffDay, kickoffTime } from '@/lib/format';
 import { cn } from '@/lib/cn';
@@ -119,8 +120,9 @@ export function ScoreboardPage() {
            </p>
          )}
          <p className="mt-3 border-t border-line pt-2 text-xs text-faint">
-            Honest rules: both forecasts frozen pre-kickoff; bookmaker margin removed (de-vig); scored with the Ranked Probability
-            Score; late snapshots excluded. Beating the market over a small sample is noise — watch the gap, not the lead.
+            Honest rules: both forecasts frozen pre-kickoff; bookmaker margin removed (<Term k="de-vig" />); scored with the{' '}
+            <Term k="RPS">Ranked Probability Score</Term>; late snapshots excluded. Beating the market over a small sample is
+            noise — watch the gap, not the lead.
          </p>
        </CardBody>
      </Card>
@@ -11,6 +11,7 @@ import { TeamProfilePage } from './features/team/TeamProfilePage';
 import { MethodologyPage } from './features/methodology/MethodologyPage';
 import { TeamsPage } from './features/teams/TeamsPage';
 import { ScoreboardPage } from './features/scoreboard/ScoreboardPage';
+import { DataPage } from './features/data/DataPage';

 const rootRoute = createRootRoute({
  component: RootLayout,
@@ -27,8 +28,9 @@ const teamRoute = createRoute({ getParentRoute: () => rootRoute, path: '/team/$n
 const methodologyRoute = createRoute({ getParentRoute: () => rootRoute, path: '/methodology', component: MethodologyPage });
 const teamsRoute = createRoute({ getParentRoute: () => rootRoute, path: '/teams', component: TeamsPage });
 const scoreboardRoute = createRoute({ getParentRoute: () => rootRoute, path: '/scoreboard', component: ScoreboardPage });
+const dataRoute = createRoute({ getParentRoute: () => rootRoute, path: '/data', component: DataPage });

-const routeTree = rootRoute.addChildren([indexRoute, groupsRoute, bracketRoute, predictRoute, storyRoute, matchRoute, teamRoute, methodologyRoute, teamsRoute, scoreboardRoute]);
+const routeTree = rootRoute.addChildren([indexRoute, groupsRoute, bracketRoute, predictRoute, storyRoute, matchRoute, teamRoute, methodologyRoute, teamsRoute, scoreboardRoute, dataRoute]);

 export const router = createRouter({ routeTree, defaultPreload: 'intent' });