Phase 2: full compendium across all categories + PF2e data
- Scraped PF2e reference from Archives of Nethys ES into public/data/pf2e/ (spells, creatures, feats, equipment, weapons, armor, ancestries, heritages, backgrounds, archetypes, actions, conditions, deities) via scripts/fetch_pf2e.ts - Registry-driven compendium: system toggle + per-category nav + generic filters - 5e expanded: weapons, armor, feats, conditions added alongside spells/monsters/items - PF2e data served as static assets (fetched, not bundled) for fast native parse - Cross-links: add spell->spellbook / item->inventory to a campaign character; add monster/creature -> open encounter - Condition tooltips in combat sourced from the conditions glossary - Tests: registry filter unit tests, compendium e2e (browse + cross-link) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* Fetch Pathfinder 2e reference data from the Archives of Nethys Elasticsearch
|
||||
* index (the same data the AoN website uses) and normalize it into per-category
|
||||
* JSON files under src/data/srd/pf2e/.
|
||||
*
|
||||
* Run with: bun run scripts/fetch_pf2e.ts (or: bunx tsx scripts/fetch_pf2e.ts)
|
||||
*
|
||||
* The index is public and CC-licensed Paizo content. We keep the structured
|
||||
* fields plus the plain-text `text` body, and drop the bulky *_markdown fields.
|
||||
*/
|
||||
import { writeFile, mkdir } from 'node:fs/promises';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
const ES_URL = 'https://elasticsearch.aonprd.com/aon/_search';
|
||||
// Served as static assets (large files load faster via fetch + native JSON.parse
|
||||
// than as bundled JS modules), so they live under public/, not src/.
|
||||
const OUT_DIR = join(dirname(fileURLToPath(import.meta.url)), '..', 'public', 'data', 'pf2e');
|
||||
|
||||
/** AoN category -> output filename. */
|
||||
const CATEGORIES: Record<string, string> = {
|
||||
spell: 'spells',
|
||||
creature: 'creatures',
|
||||
condition: 'conditions',
|
||||
feat: 'feats',
|
||||
ancestry: 'ancestries',
|
||||
heritage: 'heritages',
|
||||
background: 'backgrounds',
|
||||
archetype: 'archetypes',
|
||||
weapon: 'weapons',
|
||||
armor: 'armor',
|
||||
equipment: 'equipment',
|
||||
action: 'actions',
|
||||
deity: 'deities',
|
||||
};
|
||||
|
||||
// Fields that are bulky duplicates of `text` or internal — drop them.
|
||||
const EXCLUDES = [
|
||||
'markdown',
|
||||
'*_markdown',
|
||||
'*_scale',
|
||||
'*_scale_number',
|
||||
'*_raw',
|
||||
'release_date',
|
||||
'rarity_id',
|
||||
'size_id',
|
||||
'remaster_id',
|
||||
'is_standard_ancestry_feat',
|
||||
'exclude_from_search',
|
||||
'npc',
|
||||
'hp_scale_number',
|
||||
];
|
||||
|
||||
interface EsHit {
|
||||
_source: Record<string, unknown>;
|
||||
}
|
||||
|
||||
const PAGE = 1000;
|
||||
const MAX_WINDOW = 10000;
|
||||
|
||||
async function fetchCategory(category: string): Promise<Record<string, unknown>[]> {
|
||||
const all: Record<string, unknown>[] = [];
|
||||
|
||||
// from/size pagination within the 10k window (every category fits under it).
|
||||
for (let from = 0; from < MAX_WINDOW; from += PAGE) {
|
||||
const body = {
|
||||
from,
|
||||
size: PAGE,
|
||||
_source: { excludes: EXCLUDES },
|
||||
query: { bool: { filter: [{ term: { category } }] } },
|
||||
};
|
||||
const res = await fetch(ES_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
if (!res.ok) throw new Error(`ES ${res.status} for ${category}`);
|
||||
const json = (await res.json()) as { hits: { hits: EsHit[] } };
|
||||
const hits = json.hits.hits;
|
||||
if (hits.length === 0) break;
|
||||
for (const h of hits) all.push(normalize(h._source));
|
||||
process.stdout.write(`\r ${category}: ${all.length}`);
|
||||
if (hits.length < PAGE) break;
|
||||
}
|
||||
process.stdout.write('\n');
|
||||
return all;
|
||||
}
|
||||
|
||||
/** Keep a tidy, predictable shape; coerce traits to a string[]. */
|
||||
function normalize(src: Record<string, unknown>): Record<string, unknown> {
|
||||
const out: Record<string, unknown> = { ...src };
|
||||
if (out.trait && !Array.isArray(out.trait)) out.trait = [out.trait];
|
||||
// a stable slug for keys/links
|
||||
out.slug = String(out.id ?? out.name ?? '').toLowerCase().replace(/[^a-z0-9]+/g, '-');
|
||||
return out;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await mkdir(OUT_DIR, { recursive: true });
|
||||
const index: { category: string; file: string; count: number }[] = [];
|
||||
|
||||
for (const [category, file] of Object.entries(CATEGORIES)) {
|
||||
const entries = await fetchCategory(category);
|
||||
const path = join(OUT_DIR, `${file}.json`);
|
||||
await writeFile(path, JSON.stringify(entries));
|
||||
index.push({ category, file, count: entries.length });
|
||||
console.log(` wrote ${file}.json (${entries.length})`);
|
||||
}
|
||||
|
||||
await writeFile(join(OUT_DIR, 'index.json'), JSON.stringify(index, null, 2));
|
||||
console.log('\nDone:', index.map((i) => `${i.file}=${i.count}`).join(', '));
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user