ttrpg_manager/scripts/fetch_pf2e.ts

/**
 * Fetch Pathfinder 2e reference data from the Archives of Nethys Elasticsearch
 * index (the same data the AoN website uses) and normalize it into per-category
 * JSON files under src/data/srd/pf2e/.
 *
 * Run with: bun run scripts/fetch_pf2e.ts   (or: bunx tsx scripts/fetch_pf2e.ts)
 *
 * The index is public and CC-licensed Paizo content. We keep the structured
 * fields plus the plain-text `text` body, and drop the bulky *_markdown fields.
 */
import { writeFile, mkdir } from 'node:fs/promises';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';

const ES_URL = 'https://elasticsearch.aonprd.com/aon/_search';
// Served as static assets (large files load faster via fetch + native JSON.parse
// than as bundled JS modules), so they live under public/, not src/.
const OUT_DIR = join(dirname(fileURLToPath(import.meta.url)), '..', 'public', 'data', 'pf2e');

/** AoN category -> output filename. */
const CATEGORIES: Record<string, string> = {
  spell: 'spells',
  creature: 'creatures',
  condition: 'conditions',
  feat: 'feats',
  ancestry: 'ancestries',
  heritage: 'heritages',
  background: 'backgrounds',
  archetype: 'archetypes',
  weapon: 'weapons',
  armor: 'armor',
  equipment: 'equipment',
  action: 'actions',
  deity: 'deities',
};

// Fields that are bulky duplicates of `text` or internal — drop them.
const EXCLUDES = [
  'markdown',
  '*_markdown',
  '*_scale',
  '*_scale_number',
  '*_raw',
  'release_date',
  'rarity_id',
  'size_id',
  'remaster_id',
  'is_standard_ancestry_feat',
  'exclude_from_search',
  'npc',
  'hp_scale_number',
];

interface EsHit {
  _source: Record<string, unknown>;
}

const PAGE = 1000;
const MAX_WINDOW = 10000;

async function fetchCategory(category: string): Promise<Record<string, unknown>[]> {
  const all: Record<string, unknown>[] = [];

  // from/size pagination within the 10k window (every category fits under it).
  for (let from = 0; from < MAX_WINDOW; from += PAGE) {
    const body = {
      from,
      size: PAGE,
      _source: { excludes: EXCLUDES },
      query: { bool: { filter: [{ term: { category } }] } },
    };
    const res = await fetch(ES_URL, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify(body),
    });
    if (!res.ok) throw new Error(`ES ${res.status} for ${category}`);
    const json = (await res.json()) as { hits: { hits: EsHit[] } };
    const hits = json.hits.hits;
    if (hits.length === 0) break;
    for (const h of hits) all.push(normalize(h._source));
    process.stdout.write(`\r  ${category}: ${all.length}`);
    if (hits.length < PAGE) break;
  }
  process.stdout.write('\n');
  return all;
}

/** Keep a tidy, predictable shape; coerce traits to a string[]. */
function normalize(src: Record<string, unknown>): Record<string, unknown> {
  const out: Record<string, unknown> = { ...src };
  if (out.trait && !Array.isArray(out.trait)) out.trait = [out.trait];
  // a stable slug for keys/links
  out.slug = String(out.id ?? out.name ?? '').toLowerCase().replace(/[^a-z0-9]+/g, '-');
  return out;
}

/**
 * The AoN `action` index is polluted with action-trait fragments (names like
 * "(concentrate)", "envision", "command") and the same basic actions repeated
 * once per source. Keep only real Title-Cased action names and dedupe by name,
 * preferring the entry with the most complete text.
 */
function isRealAction(x: Record<string, unknown>): boolean {
  const n = String(x.name ?? '').trim();
  return n !== '' && /^[A-Z0-9"]/.test(n) && !n.startsWith('(');
}
function dedupeByName(list: Record<string, unknown>[]): Record<string, unknown>[] {
  const best = new Map<string, Record<string, unknown>>();
  for (const x of list) {
    const k = String(x.name);
    const cur = best.get(k);
    if (!cur || String(x.text ?? '').length > String(cur.text ?? '').length) best.set(k, x);
  }
  return [...best.values()].sort((a, b) => String(a.name).localeCompare(String(b.name)));
}
function postProcess(category: string, list: Record<string, unknown>[]): Record<string, unknown>[] {
  if (category === 'action') return dedupeByName(list.filter(isRealAction));
  return list;
}

async function main() {
  await mkdir(OUT_DIR, { recursive: true });
  const index: { category: string; file: string; count: number }[] = [];

  for (const [category, file] of Object.entries(CATEGORIES)) {
    const entries = postProcess(category, await fetchCategory(category));
    const path = join(OUT_DIR, `${file}.json`);
    await writeFile(path, JSON.stringify(entries));
    index.push({ category, file, count: entries.length });
    console.log(`  wrote ${file}.json (${entries.length})`);
  }

  await writeFile(join(OUT_DIR, 'index.json'), JSON.stringify(index, null, 2));
  console.log('\nDone:', index.map((i) => `${i.file}=${i.count}`).join(', '));
}

main().catch((e) => {
  console.error(e);
  process.exit(1);
});