diff --git a/src/app/_layout.tsx b/src/app/_layout.tsx index 594c0f6..3f078e9 100644 --- a/src/app/_layout.tsx +++ b/src/app/_layout.tsx @@ -21,6 +21,7 @@ export default function RootLayout() { + diff --git a/src/app/bibliography.tsx b/src/app/bibliography.tsx new file mode 100644 index 0000000..f7bfb01 --- /dev/null +++ b/src/app/bibliography.tsx @@ -0,0 +1,220 @@ +import * as Linking from 'expo-linking'; +import { Stack, useLocalSearchParams } from 'expo-router'; +import { useCallback, useState } from 'react'; +import { ActivityIndicator, Pressable, ScrollView, StyleSheet, View } from 'react-native'; + +import { ThemedText } from '@/components/themed-text'; +import { ThemedView } from '@/components/themed-view'; +import { MaxContentWidth, Spacing } from '@/constants/theme'; +import { useTheme } from '@/hooks/use-theme'; +import { getRepo } from '@/lib/db'; +import { downloadText } from '@/lib/download'; +import { + detectCitations, + lookupLinksFor, + resolveCitation, + toBibtex, + toRis, + type Citation, + type LookupLinks, + type RefMetadata, +} from '@/lib/enrich'; + +const ACCENT = '#3c87f7'; + +/** A detected citation paired with its (best-effort) resolved metadata. */ +interface BibEntry { + citation: Citation; + meta?: RefMetadata; +} + +export default function BibliographyScreen() { + const theme = useTheme(); + const { courseId } = useLocalSearchParams<{ courseId?: string }>(); + + const [entries, setEntries] = useState(null); + const [building, setBuilding] = useState(false); + + // Collect citations across the course's (or all) transcripts, then resolve + // each via the metadata APIs. Detection is local; lookups send only ids. + const build = useCallback(async () => { + setBuilding(true); + setEntries(null); + try { + const repo = getRepo(); + const metas = courseId ? await repo.listByCourse(courseId) : await repo.list(); + + // Detect citations across every transcript body. Dedupe by kind+value so + // a reference cited in several lectures shows once. + const byKey = new Map(); + for (const m of metas) { + const full = await repo.get(m.id); + if (!full) continue; + for (const c of detectCitations(full.segments)) { + const key = `${c.kind}:${c.value}`; + if (!byKey.has(key)) byKey.set(key, c); + } + } + const citations = [...byKey.values()]; + // Seed the list immediately (links work without lookups), then resolve. + setEntries(citations.map((citation) => ({ citation }))); + + const resolved = await Promise.all( + citations.map(async (citation) => { + try { + return { citation, meta: await resolveCitation(citation) }; + } catch { + return { citation }; + } + }), + ); + setEntries(resolved); + } finally { + setBuilding(false); + } + }, [courseId]); + + // Resolved metadata feeds the exporters; fall back to the raw citation text + // for entries that couldn't be looked up so nothing is silently dropped. + const exportable = (entries ?? []) + .map((e) => e.meta ?? fallbackMeta(e.citation)) + .filter((m): m is RefMetadata => m !== undefined); + + const exportBibtex = () => { + if (exportable.length === 0) return; + downloadText('bibliography.bib', 'application/x-bibtex', toBibtex(exportable)); + }; + + const exportRis = () => { + if (exportable.length === 0) return; + downloadText('bibliography.ris', 'application/x-research-info-systems', toRis(exportable)); + }; + + const openLink = (url: string) => { + void Linking.openURL(url); + }; + + return ( + + + + + {courseId + ? 'References detected across this course’s lectures.' + : 'References detected across all lectures.'} + + + + void build()} + disabled={building} + style={({ pressed }) => [ + styles.btn, + { backgroundColor: ACCENT, opacity: building ? 0.6 : pressed ? 0.85 : 1 }, + ]}> + + {building ? 'Scanning…' : entries ? 'Re-scan' : 'Build bibliography'} + + + {entries && entries.length > 0 && ( + <> + [ + styles.btn, + { backgroundColor: theme.backgroundElement, opacity: pressed ? 0.7 : 1 }, + ]}> + Export BibTeX + + [ + styles.btn, + { backgroundColor: theme.backgroundElement, opacity: pressed ? 0.7 : 1 }, + ]}> + Export RIS + + + )} + + + {building && entries === null && } + + {entries && entries.length === 0 && ( + + No references found. + + )} + + {entries?.map((e, i) => { + const links = lookupLinksFor(e.citation, e.meta); + return ( + + {e.meta?.title ? ( + {e.meta.title} + ) : ( + {e.citation.raw} + )} + {(e.meta?.authors?.length || e.meta?.year) && ( + + {[e.meta?.authors?.join(', '), e.meta?.year].filter(Boolean).join(' · ')} + + )} + + {e.meta?.source ?? e.citation.kind} + + + {linkEntries(links).map(([label, url]) => ( + openLink(url)} + style={({ pressed }) => [ + styles.linkChip, + { backgroundColor: theme.backgroundSelected, opacity: pressed ? 0.7 : 1 }, + ]}> + {label} + + ))} + + + ); + })} + + + ); +} + +/** Minimal metadata so an unresolved citation still exports something useful. */ +function fallbackMeta(c: Citation): RefMetadata { + return { title: c.raw, source: c.kind }; +} + +/** Ordered, labeled, defined link entries from a LookupLinks bag. */ +function linkEntries(links: LookupLinks): [string, string][] { + const order: [keyof LookupLinks, string][] = [ + ['doi', 'DOI'], + ['arxiv', 'arXiv'], + ['openAlex', 'OpenAlex'], + ['openLibrary', 'Open Library'], + ['googleBooks', 'Google Books'], + ['wikipedia', 'Wikipedia'], + ['annasArchive', "Anna's Archive"], + ['libgen', 'LibGen'], + ]; + return order + .map(([key, label]) => [label, links[key]] as const) + .filter((pair): pair is [string, string] => typeof pair[1] === 'string' && pair[1].length > 0) + .map(([label, url]) => [label, url]); +} + +const styles = StyleSheet.create({ + fill: { flex: 1 }, + content: { padding: Spacing.three, gap: Spacing.two, maxWidth: MaxContentWidth, width: '100%', alignSelf: 'center' }, + row: { flexDirection: 'row', flexWrap: 'wrap', gap: Spacing.two, alignItems: 'center' }, + btn: { paddingHorizontal: Spacing.three, paddingVertical: Spacing.two, borderRadius: Spacing.two, alignItems: 'center' }, + btnText: { color: '#fff', fontWeight: '700', fontSize: 14 }, + card: { padding: Spacing.three, borderRadius: Spacing.three, gap: 2 }, + linkChips: { flexDirection: 'row', flexWrap: 'wrap', gap: Spacing.one, marginTop: Spacing.one }, + linkChip: { paddingHorizontal: Spacing.two, paddingVertical: 2, borderRadius: 999 }, + pad: { paddingVertical: Spacing.four, textAlign: 'center' }, +}); diff --git a/src/app/courses.tsx b/src/app/courses.tsx index 63da31e..8899c4e 100644 --- a/src/app/courses.tsx +++ b/src/app/courses.tsx @@ -1,4 +1,4 @@ -import { Stack, useFocusEffect } from 'expo-router'; +import { Link, Stack, useFocusEffect } from 'expo-router'; import { useCallback, useState } from 'react'; import { Pressable, ScrollView, StyleSheet, TextInput, View } from 'react-native'; @@ -105,6 +105,11 @@ export default function CoursesScreen() { + + + Bibliography + + { setEditing(c.id); diff --git a/src/app/transcript/[id].tsx b/src/app/transcript/[id].tsx index 90c55da..e570851 100644 --- a/src/app/transcript/[id].tsx +++ b/src/app/transcript/[id].tsx @@ -1,3 +1,4 @@ +import * as Linking from 'expo-linking'; import { Link, Stack, useLocalSearchParams } from 'expo-router'; import { useEffect, useMemo, useRef, useState } from 'react'; import { @@ -16,6 +17,17 @@ import { MaxContentWidth, Spacing } from '@/constants/theme'; import { useTheme } from '@/hooks/use-theme'; import { getRepo, type Transcript } from '@/lib/db'; import { downloadText } from '@/lib/download'; +import { + detectCitations, + detectEvents, + lookupLinksFor, + resolveCitation, + toIcs, + type CandidateEvent, + type Citation, + type LookupLinks, + type RefMetadata, +} from '@/lib/enrich'; import { EXPORT_META, formatTranscript, type ExportFormat } from '@/lib/export'; import { formatClock } from '@/lib/format'; import { @@ -54,6 +66,16 @@ export default function TranscriptScreen() { const [creatingCards, setCreatingCards] = useState(false); const [cardsAdded, setCardsAdded] = useState(null); + // Dates & references (Phase 4) — detected on demand, nothing persisted. + // `enrichment === null` means "not scanned yet". + const [enrichment, setEnrichment] = useState<{ + events: CandidateEvent[]; + citations: Citation[]; + } | null>(null); + // Per-citation resolved metadata + in-flight flags, keyed by list index. + const [refMeta, setRefMeta] = useState>({}); + const [refLoading, setRefLoading] = useState>({}); + const scrollRef = useRef(null); // Y offset of each rendered segment row, captured via onLayout, for scroll-to. const segmentYs = useRef>({}); @@ -214,6 +236,43 @@ export default function TranscriptScreen() { } }; + // Scan segments for deadlines + citations. Dates are anchored to the lecture + // date when known, else the transcript's creation time. Pure & local. + const scanEnrichment = () => { + const anchorMs = transcript?.lectureDate ?? transcript?.createdAt ?? Date.now(); + setEnrichment({ + events: detectEvents(segments, anchorMs), + citations: detectCitations(segments), + }); + setRefMeta({}); + setRefLoading({}); + }; + + // Download a calendar file for one or more events. + const exportIcs = (events: CandidateEvent[]) => { + if (events.length === 0) return; + const ics = toIcs( + events.map((e) => ({ title: e.title, dateMs: e.dateMs, allDay: e.allDay })), + ); + downloadText('event.ics', 'text/calendar', ics); + }; + + // Network lookup for a single citation (sends only the id string, never text). + const lookupRef = async (i: number, c: Citation) => { + if (refLoading[i]) return; + setRefLoading((prev) => ({ ...prev, [i]: true })); + try { + const meta = await resolveCitation(c); + if (meta) setRefMeta((prev) => ({ ...prev, [i]: meta })); + } finally { + setRefLoading((prev) => ({ ...prev, [i]: false })); + } + }; + + const openLink = (url: string) => { + void Linking.openURL(url); + }; + if (transcript === undefined) return ; if (transcript === null) return ( @@ -344,6 +403,129 @@ export default function TranscriptScreen() { )} + {/* Dates & references — detected on demand (pure/local detection). */} + + Dates & references + [ + styles.aidBtn, + { backgroundColor: ACCENT, opacity: segments.length === 0 ? 0.5 : pressed ? 0.85 : 1 }, + ]}> + + {enrichment ? 'Re-scan for dates & references' : 'Scan for dates & references'} + + + + {enrichment && ( + <> + {/* --- Dates --- */} + + + Dates + {enrichment.events.length > 0 && ( + exportIcs(enrichment.events)} hitSlop={6}> + Add all ({enrichment.events.length}) + + )} + + {enrichment.events.length === 0 ? ( + No dates found. + ) : ( + enrichment.events.map((e, i) => ( + + e.start !== undefined && seek(e.start)} + disabled={e.start === undefined} + hitSlop={4} + style={styles.flex}> + {e.title} + + {formatEventDate(e)} + {e.start !== undefined ? ` · ${formatClock(e.start)}` : ''} + + + exportIcs([e])} hitSlop={6}> + Add to calendar + + + )) + )} + + + {/* --- References --- */} + + References + {enrichment.citations.length === 0 ? ( + No references found. + ) : ( + enrichment.citations.map((c, i) => { + const meta = refMeta[i]; + const links = lookupLinksFor(c, meta); + return ( + + + c.start !== undefined && seek(c.start)} + disabled={c.start === undefined} + hitSlop={4} + style={styles.flex}> + {c.raw} + {c.start !== undefined && ( + + {formatClock(c.start)} + + )} + + void lookupRef(i, c)} + disabled={refLoading[i]} + hitSlop={6}> + {refLoading[i] ? ( + + ) : ( + + {meta ? 'Looked up' : 'Look up'} + + )} + + + + {meta && ( + + {meta.title && {meta.title}} + {(meta.authors?.length || meta.year) && ( + + {[meta.authors?.join(', '), meta.year].filter(Boolean).join(' · ')} + + )} + {meta.source} + + )} + + + {linkEntries(links).map(([label, url]) => ( + openLink(url)} + style={({ pressed }) => [ + styles.linkChip, + { backgroundColor: theme.backgroundSelected, opacity: pressed ? 0.7 : 1 }, + ]}> + {label} + + ))} + + + ); + }) + )} + + + )} + + {segments.map((s, i) => ( {children}; } +/** Localized date for a candidate event — date-only when it's an all-day item. */ +function formatEventDate(e: CandidateEvent): string { + const d = new Date(e.dateMs); + return e.allDay + ? d.toLocaleDateString() + : d.toLocaleString([], { dateStyle: 'medium', timeStyle: 'short' }); +} + +/** Ordered, labeled, defined link entries from a LookupLinks bag. */ +function linkEntries(links: LookupLinks): [string, string][] { + const order: [keyof LookupLinks, string][] = [ + ['doi', 'DOI'], + ['arxiv', 'arXiv'], + ['openAlex', 'OpenAlex'], + ['openLibrary', 'Open Library'], + ['googleBooks', 'Google Books'], + ['wikipedia', 'Wikipedia'], + ['annasArchive', "Anna's Archive"], + ['libgen', 'LibGen'], + ]; + return order + .map(([key, label]) => [label, links[key]] as const) + .filter((pair): pair is [string, string] => typeof pair[1] === 'string' && pair[1].length > 0) + .map(([label, url]) => [label, url]); +} + const styles = StyleSheet.create({ fill: { flex: 1 }, centered: { alignItems: 'center', justifyContent: 'center' }, @@ -419,6 +627,12 @@ const styles = StyleSheet.create({ aidSection: { gap: Spacing.one, marginTop: Spacing.one }, aidLine: { flexDirection: 'row', gap: Spacing.two, alignItems: 'flex-start', paddingVertical: 2 }, flex: { flex: 1 }, + refHeader: { flexDirection: 'row', alignItems: 'center', justifyContent: 'space-between', gap: Spacing.two }, + refRow: { flexDirection: 'row', gap: Spacing.two, alignItems: 'flex-start', paddingVertical: 2 }, + citCard: { gap: Spacing.one, paddingVertical: Spacing.one }, + metaBox: { gap: 2, paddingLeft: Spacing.two }, + linkChips: { flexDirection: 'row', flexWrap: 'wrap', gap: Spacing.one }, + linkChip: { paddingHorizontal: Spacing.two, paddingVertical: 2, borderRadius: 999 }, segRow: { flexDirection: 'row', gap: Spacing.two, paddingVertical: Spacing.one, paddingHorizontal: Spacing.two, borderRadius: Spacing.two, alignItems: 'flex-start' }, ts: { paddingTop: 4, minWidth: 52 }, segText: { flex: 1, fontSize: 16, lineHeight: 24, padding: 0 }, diff --git a/src/lib/enrich/bib.test.ts b/src/lib/enrich/bib.test.ts new file mode 100644 index 0000000..d672cbd --- /dev/null +++ b/src/lib/enrich/bib.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect } from 'vitest'; +import { toBibtex, toRis } from './bib'; +import type { RefMetadata } from './types'; + +describe('toBibtex', () => { + it('returns "" for []', () => { + expect(toBibtex([])).toBe(''); + }); + + it('emits an @article with a first-author+year key', () => { + const refs: RefMetadata[] = [ + { + title: 'Attention Is All You Need', + authors: ['Ashish Vaswani', 'Noam Shazeer'], + year: 2017, + source: 'Crossref', + url: 'https://doi.org/10.x', + }, + ]; + const out = toBibtex(refs); + expect(out).toContain('@article{vaswani2017,'); + expect(out).toContain('title = {Attention Is All You Need}'); + expect(out).toContain('author = {Ashish Vaswani and Noam Shazeer}'); + expect(out).toContain('year = {2017}'); + expect(out).toContain('url = {https://doi.org/10.x}'); + }); + + it('uses @misc and ref key when no authors', () => { + const out = toBibtex([{ title: 'A book', source: 'Open Library' }]); + expect(out).toContain('@misc{ref0,'); + expect(out).toContain('title = {A book}'); + }); + + it('omits missing fields', () => { + const out = toBibtex([{ source: 'X' }]); + expect(out).toContain('@misc{ref0,'); + expect(out).not.toContain('title ='); + expect(out).not.toContain('year ='); + }); + + it('brace-escapes special characters in values', () => { + const out = toBibtex([{ title: 'a {b} c', source: 'X' }]); + expect(out).toContain('title = {a \\{b\\} c}'); + }); +}); + +describe('toRis', () => { + it('returns "" for []', () => { + expect(toRis([])).toBe(''); + }); + + it('emits a JOUR record with one AU line per author', () => { + const out = toRis([ + { + title: 'T', + authors: ['A One', 'B Two'], + year: 2020, + source: 'X', + url: 'http://u', + }, + ]); + expect(out).toContain('TY - JOUR'); + expect(out).toContain('TI - T'); + expect(out).toContain('AU - A One'); + expect(out).toContain('AU - B Two'); + expect(out).toContain('PY - 2020'); + expect(out).toContain('UR - http://u'); + expect(out.split('\n')).toContain('ER - '); + }); + + it('uses GEN and omits missing fields', () => { + const out = toRis([{ source: 'X' }]); + expect(out).toContain('TY - GEN'); + expect(out).not.toContain('TI -'); + expect(out).not.toContain('AU -'); + expect(out).toContain('ER - '); + }); +}); diff --git a/src/lib/enrich/bib.ts b/src/lib/enrich/bib.ts new file mode 100644 index 0000000..872fb83 --- /dev/null +++ b/src/lib/enrich/bib.ts @@ -0,0 +1,64 @@ +// Pure bibliography serialization: BibTeX and RIS. No network, no I/O. + +import type { RefMetadata } from './types'; + +/** Serialize references to BibTeX. Handles missing fields and []. */ +export function toBibtex(refs: RefMetadata[]): string { + return refs.map((r, i) => bibEntry(r, i)).join('\n\n'); +} + +function bibEntry(r: RefMetadata, index: number): string { + const type = r.authors && r.authors.length ? 'article' : 'misc'; + const key = bibKey(r, index); + + const fields: string[] = []; + if (r.title) fields.push(` title = {${brace(r.title)}}`); + if (r.authors && r.authors.length) { + fields.push(` author = {${brace(r.authors.join(' and '))}}`); + } + if (r.year !== undefined) fields.push(` year = {${r.year}}`); + if (r.url) fields.push(` url = {${brace(r.url)}}`); + + return `@${type}{${key},\n${fields.join(',\n')}\n}`; +} + +function bibKey(r: RefMetadata, index: number): string { + const first = r.authors && r.authors[0]; + if (first) { + // Last token of the first author's name, alphanumerics only. + const tokens = first.trim().split(/[\s,]+/); + const surname = (tokens[tokens.length - 1] ?? first) + .replace(/[^A-Za-z0-9]/g, '') + .toLowerCase(); + if (surname) return `${surname}${r.year ?? ''}`; + } + return `ref${index}`; +} + +/** Escape characters special to BibTeX brace-delimited values. */ +function brace(s: string): string { + // Inside `{...}` we still must escape a stray backslash and the braces. + return s.replace(/\\/g, '\\\\').replace(/[{}]/g, (m) => `\\${m}`); +} + +// --------------------------------------------------------------------------- +// RIS +// --------------------------------------------------------------------------- + +/** Serialize references to RIS. Handles missing fields and []. */ +export function toRis(refs: RefMetadata[]): string { + return refs.map(risEntry).join('\n\n'); +} + +function risEntry(r: RefMetadata): string { + const ty = r.authors && r.authors.length ? 'JOUR' : 'GEN'; + const lines: string[] = [`TY - ${ty}`]; + if (r.title) lines.push(`TI - ${r.title}`); + if (r.authors) { + for (const a of r.authors) lines.push(`AU - ${a}`); + } + if (r.year !== undefined) lines.push(`PY - ${r.year}`); + if (r.url) lines.push(`UR - ${r.url}`); + lines.push('ER - '); + return lines.join('\n'); +} diff --git a/src/lib/enrich/citations.test.ts b/src/lib/enrich/citations.test.ts new file mode 100644 index 0000000..e25d038 --- /dev/null +++ b/src/lib/enrich/citations.test.ts @@ -0,0 +1,84 @@ +import { describe, it, expect } from 'vitest'; +import { detectCitations } from './citations'; +import type { Segment } from '../types'; + +const seg = (start: number, text: string, id?: string): Segment => ({ + id, + start, + end: start + 1, + text, +}); + +describe('detectCitations', () => { + it('returns [] for empty input', () => { + expect(detectCitations([])).toEqual([]); + }); + + it('extracts a bare DOI', () => { + const out = detectCitations([seg(0, 'See 10.1038/nphys1170 for details.')]); + const doi = out.find((c) => c.kind === 'doi'); + expect(doi?.value).toBe('10.1038/nphys1170'); + }); + + it('extracts a DOI from a doi.org URL and a doi: prefix', () => { + const a = detectCitations([seg(0, 'https://doi.org/10.1145/3292500.3330701')]); + expect(a[0]!.value).toBe('10.1145/3292500.3330701'); + const b = detectCitations([seg(0, 'doi: 10.1000/xyz123')]); + expect(b[0]!.value).toBe('10.1000/xyz123'); + }); + + it('trims trailing punctuation from a DOI', () => { + const out = detectCitations([seg(0, '(see 10.1038/nphys1170).')]); + expect(out[0]!.value).toBe('10.1038/nphys1170'); + }); + + it('extracts an explicit arXiv id', () => { + const out = detectCitations([seg(0, 'Read arXiv:2103.00020 today.')]); + const ax = out.find((c) => c.kind === 'arxiv'); + expect(ax?.value).toBe('2103.00020'); + }); + + it('extracts a bare arXiv id', () => { + const out = detectCitations([seg(0, 'the paper 1706.03762 introduced it')]); + const ax = out.find((c) => c.kind === 'arxiv'); + expect(ax?.value).toBe('1706.03762'); + }); + + it('extracts ISBN-13 and ISBN-10, normalized without hyphens', () => { + const a = detectCitations([seg(0, 'ISBN 978-0-13-468599-1')]); + expect(a.find((c) => c.kind === 'isbn')?.value).toBe('9780134685991'); + const b = detectCitations([seg(0, 'ISBN-10: 0-262-03384-4')]); + expect(b.find((c) => c.kind === 'isbn')?.value).toBe('0262033844'); + }); + + it('extracts author-year (parenthesized)', () => { + const out = detectCitations([seg(0, 'As Vaswani (2017) showed,')]); + const ay = out.find((c) => c.kind === 'author-year'); + expect(ay?.value).toBe('Vaswani (2017)'); + }); + + it('extracts "Author et al., Year"', () => { + const out = detectCitations([seg(0, 'per Smith et al., 2021 this holds')]); + const ay = out.find((c) => c.kind === 'author-year'); + expect(ay?.value).toBe('Smith et al. (2021)'); + }); + + it('dedupes by value', () => { + const out = detectCitations([ + seg(0, '10.1038/nphys1170'), + seg(1, 'again 10.1038/nphys1170'), + ]); + expect(out.filter((c) => c.kind === 'doi')).toHaveLength(1); + }); + + it('tags sourceSegmentId and start', () => { + const out = detectCitations([seg(9, 'arXiv:2103.00020', 'segA')]); + expect(out[0]!.sourceSegmentId).toBe('segA'); + expect(out[0]!.start).toBe(9); + }); + + it('does not treat plain prose years as citations', () => { + const out = detectCitations([seg(0, 'In 2017 the weather was nice.')]); + expect(out.filter((c) => c.kind === 'author-year')).toHaveLength(0); + }); +}); diff --git a/src/lib/enrich/citations.ts b/src/lib/enrich/citations.ts new file mode 100644 index 0000000..4919017 --- /dev/null +++ b/src/lib/enrich/citations.ts @@ -0,0 +1,165 @@ +// Pure citation extraction over lecture segments. Regex-only; no network. +// Detects DOIs, arXiv ids, ISBN-10/13, and "Author (Year)" mentions. + +import type { Segment } from '../types'; +import type { Citation } from './types'; + +/** Detect and normalize citations across all segments. Deduped by value. */ +export function detectCitations(segments: Segment[]): Citation[] { + if (!segments.length) return []; + + const out: Citation[] = []; + const seen = new Set(); + + const push = (c: Citation) => { + const key = `${c.kind}:${c.value.toLowerCase()}`; + if (seen.has(key)) return; + seen.add(key); + out.push(c); + }; + + for (const seg of segments) { + const text = seg.text ?? ''; + if (!text) continue; + + for (const c of detectDois(text)) push(tag(c, seg)); + for (const c of detectArxiv(text)) push(tag(c, seg)); + for (const c of detectIsbn(text)) push(tag(c, seg)); + for (const c of detectAuthorYear(text)) push(tag(c, seg)); + } + + return out; +} + +function tag( + c: Omit, + seg: Segment, +): Citation { + return { ...c, sourceSegmentId: seg.id, start: seg.start }; +} + +// --------------------------------------------------------------------------- +// DOI +// --------------------------------------------------------------------------- + +function detectDois(text: string): Omit[] { + const out: Omit[] = []; + // DOIs start with 10./. Allow an optional doi: / URL prefix. + const re = /\b(?:doi:\s*|https?:\/\/(?:dx\.)?doi\.org\/)?(10\.\d{4,9}\/[^\s"<>]+)/gi; + let m: RegExpExecArray | null; + while ((m = re.exec(text))) { + // Trim common trailing punctuation that isn't part of the DOI. + const value = m[1]!.replace(/[.,;)\]]+$/, ''); + out.push({ kind: 'doi', value, raw: m[0] }); + } + return out; +} + +// --------------------------------------------------------------------------- +// arXiv +// --------------------------------------------------------------------------- + +function detectArxiv(text: string): Omit[] { + const out: Omit[] = []; + + // Explicit "arXiv:NNNN.NNNNN" (with optional version) — most reliable. + const reExplicit = /\barxiv:\s*(\d{4}\.\d{4,5}(?:v\d+)?)\b/gi; + let m: RegExpExecArray | null; + while ((m = reExplicit.exec(text))) { + out.push({ kind: 'arxiv', value: m[1]!, raw: m[0] }); + } + + // Bare "NNNN.NNNNN" — only when not already captured as part of an arXiv: hit. + const reBare = /(?[] { + const out: Omit[] = []; + // ISBN-13 (978/979 prefix) and ISBN-10, hyphens/spaces allowed internally. + const re = + /\bISBN(?:-1[03])?:?\s*((?:97[89][-\s]?)?(?:\d[-\s]?){9}[\dXx])\b/g; + let m: RegExpExecArray | null; + while ((m = re.exec(text))) { + const value = m[1]!.replace(/[-\s]/g, '').toUpperCase(); + if (value.length === 10 || value.length === 13) { + out.push({ kind: 'isbn', value, raw: m[0] }); + } + } + return out; +} + +// --------------------------------------------------------------------------- +// Author-year +// --------------------------------------------------------------------------- + +function detectAuthorYear( + text: string, +): Omit[] { + const out: Omit[] = []; + + // "Smith (2020)", "Smith and Jones (2019)", "Smith et al., 2021", + // "Smith et al. (2021)". Author = Capitalized word(s). + const re = + /\b([A-Z][a-z]+(?:[-\s][A-Z][a-z]+)*(?:\s+(?:and|&)\s+[A-Z][a-z]+)?(?:\s+et al\.?)?)\s*[,(]?\s*\(?(\b(?:18|19|20)\d{2})\)?/g; + let m: RegExpExecArray | null; + while ((m = re.exec(text))) { + const author = stripLeadingStopword(m[1]!.trim().replace(/\s+/g, ' ')); + const year = m[2]!; + // Require either a parenthesized year or an "et al"/"and" multi-author cue + // to avoid matching plain prose like "World War 2 1945". + const cue = m[0]; + const looksLikeCitation = + /\(\s*(?:18|19|20)\d{2}\s*\)/.test(cue) || + /et al/i.test(cue) || + /\band\b|&/.test(cue); + if (!looksLikeCitation) continue; + const value = `${author} (${year})`; + out.push({ kind: 'author-year', value, raw: m[0].trim() }); + } + return out; +} + +// Common capitalized sentence-starters that can be mistaken for a surname when +// they immediately precede the real author (e.g. "As Vaswani (2017)"). +const LEADING_STOPWORDS = new Set([ + 'As', + 'In', + 'By', + 'See', + 'Per', + 'From', + 'After', + 'Before', + 'And', + 'The', + 'This', + 'That', + 'These', + 'Those', + 'Both', + 'When', + 'While', + 'Since', +]); + +function stripLeadingStopword(author: string): string { + const tokens = author.split(' '); + // Only strip when something real remains and the next token is itself a name. + if (tokens.length > 1 && tokens[0] && LEADING_STOPWORDS.has(tokens[0])) { + return tokens.slice(1).join(' '); + } + return author; +} diff --git a/src/lib/enrich/dates.test.ts b/src/lib/enrich/dates.test.ts new file mode 100644 index 0000000..2d203ad --- /dev/null +++ b/src/lib/enrich/dates.test.ts @@ -0,0 +1,145 @@ +import { describe, it, expect } from 'vitest'; +import { detectEvents } from './dates'; +import type { Segment } from '../types'; + +const seg = (start: number, text: string, id?: string): Segment => ({ + id, + start, + end: start + 1, + text, +}); + +// Anchor = 2026-06-13 (Saturday) 00:00 UTC. All assertions derive from this. +const ANCHOR = Date.UTC(2026, 5, 13, 0, 0, 0, 0); + +const utc = (y: number, mo: number, d: number, h = 0, mi = 0) => + Date.UTC(y, mo, d, h, mi, 0, 0); + +describe('detectEvents', () => { + it('returns [] for empty input', () => { + expect(detectEvents([], ANCHOR)).toEqual([]); + }); + + it('returns [] for non-finite anchor', () => { + expect(detectEvents([seg(0, 'exam tomorrow')], Number.NaN)).toEqual([]); + }); + + it('resolves ISO YYYY-MM-DD to UTC midnight', () => { + const out = detectEvents([seg(0, 'The exam is on 2026-06-20.')], ANCHOR); + expect(out).toHaveLength(1); + expect(out[0]!.dateMs).toBe(utc(2026, 5, 20)); + expect(out[0]!.allDay).toBe(true); + expect(out[0]!.confidence).toBeGreaterThanOrEqual(0.9); + }); + + it('resolves "March 3" using the anchor year (no roll within 6mo)', () => { + const out = detectEvents([seg(0, 'deadline on March 3')], ANCHOR); + expect(out).toHaveLength(1); + // March 3 2026 is ~3mo before June 13 2026 -> stays in the anchor year. + expect(out[0]!.dateMs).toBe(utc(2026, 2, 3)); + }); + + it('rolls a date more than 6 months before the anchor to next year', () => { + // Anchor late in the year: "January 5" resolves ~10mo back -> roll to next. + const novAnchor = Date.UTC(2026, 10, 1); // 2026-11-01 + const out = detectEvents([seg(0, 'deadline on January 5')], novAnchor); + expect(out[0]!.dateMs).toBe(utc(2027, 0, 5)); + }); + + it('keeps an explicit year even if far past', () => { + const out = detectEvents([seg(0, 'paper from March 3, 2020')], ANCHOR); + expect(out[0]!.dateMs).toBe(utc(2020, 2, 3)); + }); + + it('resolves day-first "3 March"', () => { + const out = detectEvents([seg(0, 'submission due 20 July')], ANCHOR); + expect(out[0]!.dateMs).toBe(utc(2026, 6, 20)); + }); + + it('does NOT roll a date within the next 6 months', () => { + const out = detectEvents([seg(0, 'exam on June 20')], ANCHOR); + expect(out[0]!.dateMs).toBe(utc(2026, 5, 20)); + }); + + it('resolves "on the 14th" within the anchor month', () => { + const out = detectEvents([seg(0, 'assignment due on the 14th')], ANCHOR); + expect(out[0]!.dateMs).toBe(utc(2026, 5, 14)); + }); + + it('resolves relative "tomorrow" from the anchor', () => { + const out = detectEvents([seg(0, 'quiz tomorrow')], ANCHOR); + expect(out[0]!.dateMs).toBe(utc(2026, 5, 14)); + expect(out[0]!.confidence).toBeLessThan(0.9); // relative + }); + + it('resolves "today" to the anchor day', () => { + const out = detectEvents([seg(0, 'homework due today')], ANCHOR); + expect(out[0]!.dateMs).toBe(utc(2026, 5, 13)); + }); + + it('resolves "next week" as +7 days', () => { + const out = detectEvents([seg(0, 'the exam is next week')], ANCHOR); + expect(out[0]!.dateMs).toBe(utc(2026, 5, 20)); + }); + + it('resolves "in 3 days" and "in 2 weeks"', () => { + const a = detectEvents([seg(0, 'due in 3 days')], ANCHOR); + expect(a[0]!.dateMs).toBe(utc(2026, 5, 16)); + const b = detectEvents([seg(0, 'due in 2 weeks')], ANCHOR); + expect(b[0]!.dateMs).toBe(utc(2026, 5, 27)); + }); + + it('resolves "next Monday" to the Monday of the following week', () => { + // Anchor Sat 2026-06-13. Next Monday = 2026-06-22 (skip the 15th). + const out = detectEvents([seg(0, 'exam next Monday')], ANCHOR); + expect(out[0]!.dateMs).toBe(utc(2026, 5, 22)); + }); + + it('captures a clock time and sets allDay=false', () => { + const out = detectEvents([seg(0, 'exam on June 20 at 3pm')], ANCHOR); + expect(out[0]!.allDay).toBe(false); + expect(out[0]!.dateMs).toBe(utc(2026, 5, 20, 15, 0)); + }); + + it('parses 24h time "14:00"', () => { + const out = detectEvents([seg(0, 'meeting 2026-06-20 14:00')], ANCHOR); + expect(out[0]!.allDay).toBe(false); + expect(out[0]!.dateMs).toBe(utc(2026, 5, 20, 14, 0)); + }); + + it('uses the surrounding clause as the title', () => { + const out = detectEvents( + [seg(0, 'Welcome everyone. The final exam is on June 20. See you.')], + ANCHOR, + ); + expect(out[0]!.title.toLowerCase()).toContain('final exam'); + }); + + it('boosts confidence when a deadline keyword is present', () => { + const withKw = detectEvents([seg(0, 'the exam is on June 20')], ANCHOR)[0]!; + const without = detectEvents([seg(0, 'we met on June 20')], ANCHOR)[0]!; + expect(withKw.confidence).toBeGreaterThan(without.confidence); + }); + + it('tags sourceSegmentId and start', () => { + const out = detectEvents([seg(42, 'exam on June 20', 'segX')], ANCHOR); + expect(out[0]!.sourceSegmentId).toBe('segX'); + expect(out[0]!.start).toBe(42); + }); + + it('dedupes identical (dateMs + title)', () => { + const out = detectEvents( + [seg(0, 'exam on June 20'), seg(1, 'exam on June 20')], + ANCHOR, + ); + expect(out).toHaveLength(1); + }); + + it('does not read the wall clock (output depends only on the anchor)', () => { + const a = detectEvents([seg(0, 'quiz tomorrow')], ANCHOR); + const other = Date.UTC(2030, 0, 1); + const b = detectEvents([seg(0, 'quiz tomorrow')], other); + expect(a[0]!.dateMs).toBe(utc(2026, 5, 14)); + expect(b[0]!.dateMs).toBe(Date.UTC(2030, 0, 2)); + }); +}); diff --git a/src/lib/enrich/dates.ts b/src/lib/enrich/dates.ts new file mode 100644 index 0000000..c9cf6d6 --- /dev/null +++ b/src/lib/enrich/dates.ts @@ -0,0 +1,407 @@ +// Pure deadline/date detection over lecture segments. NO wall-clock reads: +// every timestamp is derived from `anchorMs` (the lecture date) plus explicit +// year/month/day/hour components found in the text. Constructing Date objects +// from those explicit components is fine; calling Date.now() / new Date() with +// no args is NOT. + +import type { Segment } from '../types'; +import type { CandidateEvent } from './types'; + +const MONTHS: Record = { + january: 0, + february: 1, + march: 2, + april: 3, + may: 4, + june: 5, + july: 6, + august: 7, + september: 8, + october: 9, + november: 10, + december: 11, + jan: 0, + feb: 1, + mar: 2, + apr: 3, + jun: 5, + jul: 6, + aug: 7, + sep: 8, + sept: 8, + oct: 9, + nov: 10, + dec: 11, +}; + +const WEEKDAYS: Record = { + sunday: 0, + monday: 1, + tuesday: 2, + wednesday: 3, + thursday: 4, + friday: 5, + saturday: 6, +}; + +const DEADLINE_WORDS = [ + 'exam', + 'deadline', + 'due', + 'assignment', + 'midterm', + 'final', + 'quiz', + 'submit', + 'submission', + 'project', + 'homework', + 'test', +]; + +const SIX_MONTHS_MS = 1000 * 60 * 60 * 24 * 30 * 6; + +interface TimeOfDay { + hour: number; + minute: number; +} + +/** A partial match produced by a detector before titling/dating. */ +interface RawMatch { + /** Absolute resolved instant in ms. */ + dateMs: number; + /** Whether a clock time was found (=> not all-day). */ + hasTime: boolean; + /** Matched substring (for `raw`). */ + matched: string; + /** Index of the match within the segment text (for clause extraction). */ + index: number; + /** Base confidence before deadline-keyword boosting. */ + baseConfidence: number; +} + +/** + * Detect candidate dates/deadlines across all segments. + * + * @param segments lecture segments (text + timing). + * @param anchorMs the lecture date in ms; relative dates anchor here and + * absolute dates borrow this year when none is written. + */ +export function detectEvents(segments: Segment[], anchorMs: number): CandidateEvent[] { + if (!segments.length || !Number.isFinite(anchorMs)) return []; + + const out: CandidateEvent[] = []; + const seen = new Set(); + + for (const seg of segments) { + const text = seg.text ?? ''; + if (!text) continue; + + const matches = [ + ...detectIso(text, anchorMs), + ...detectMonthName(text, anchorMs), + ...detectOrdinalOnly(text, anchorMs), + ...detectRelative(text, anchorMs), + ]; + + for (const m of matches) { + const title = clauseAround(text, m.index, m.matched.length); + const hasDeadlineWord = DEADLINE_WORDS.some((w) => + title.toLowerCase().includes(w), + ); + let confidence = m.baseConfidence; + if (hasDeadlineWord) confidence = Math.min(1, confidence + 0.1); + else confidence = Math.max(0, confidence - 0.2); + + const key = `${m.dateMs}|${title.toLowerCase()}`; + if (seen.has(key)) continue; + seen.add(key); + + out.push({ + title, + dateMs: m.dateMs, + allDay: !m.hasTime, + sourceSegmentId: seg.id, + start: seg.start, + raw: m.matched.trim(), + confidence: Number(confidence.toFixed(2)), + }); + } + } + + return out; +} + +// --------------------------------------------------------------------------- +// Absolute detectors +// --------------------------------------------------------------------------- + +/** ISO `YYYY-MM-DD`, optionally followed by a clock time. */ +function detectIso(text: string, anchorMs: number): RawMatch[] { + const out: RawMatch[] = []; + const re = /\b(\d{4})-(\d{2})-(\d{2})\b/g; + let m: RegExpExecArray | null; + while ((m = re.exec(text))) { + const year = Number(m[1]); + const month = Number(m[2]) - 1; + const day = Number(m[3]); + if (month < 0 || month > 11 || day < 1 || day > 31) continue; + const time = findTimeNear(text, m.index + m[0].length); + const dateMs = buildUtc(year, month, day, time); + out.push({ + dateMs, + hasTime: !!time, + matched: m[0], + index: m.index, + baseConfidence: 0.9, + }); + } + return out; +} + +/** `March 3`, `March 3, 2026`, `3 March`, `on March 3rd`. */ +function detectMonthName(text: string, anchorMs: number): RawMatch[] { + const out: RawMatch[] = []; + const monthAlt = Object.keys(MONTHS).join('|'); + + // Month-first: "March 3", "March 3rd, 2026" + const reMD = new RegExp( + `\\b(${monthAlt})\\.?\\s+(\\d{1,2})(?:st|nd|rd|th)?(?:,?\\s+(\\d{4}))?`, + 'gi', + ); + let m: RegExpExecArray | null; + while ((m = reMD.exec(text))) { + const month = MONTHS[m[1]!.toLowerCase()]; + const day = Number(m[2]); + if (month === undefined || day < 1 || day > 31) continue; + const explicitYear = m[3] ? Number(m[3]) : undefined; + pushResolved(out, text, m, month, day, explicitYear, anchorMs); + } + + // Day-first: "3 March", "3rd March 2026" + const reDM = new RegExp( + `\\b(\\d{1,2})(?:st|nd|rd|th)?\\s+(${monthAlt})\\.?(?:,?\\s+(\\d{4}))?`, + 'gi', + ); + while ((m = reDM.exec(text))) { + const day = Number(m[1]); + const month = MONTHS[m[2]!.toLowerCase()]; + if (month === undefined || day < 1 || day > 31) continue; + const explicitYear = m[3] ? Number(m[3]) : undefined; + pushResolved(out, text, m, month, day, explicitYear, anchorMs); + } + + return out; +} + +function pushResolved( + out: RawMatch[], + text: string, + m: RegExpExecArray, + month: number, + day: number, + explicitYear: number | undefined, + anchorMs: number, +): void { + const time = findTimeNear(text, m.index + m[0].length); + let year = explicitYear ?? utcYear(anchorMs); + let dateMs = buildUtc(year, month, day, time); + if (explicitYear === undefined && dateMs < anchorMs - SIX_MONTHS_MS) { + year += 1; + dateMs = buildUtc(year, month, day, time); + } + out.push({ + dateMs, + hasTime: !!time, + matched: m[0], + index: m.index, + baseConfidence: 0.9, + }); +} + +/** `on the 14th` — day-of-month only, month/year from the anchor. */ +function detectOrdinalOnly(text: string, anchorMs: number): RawMatch[] { + const out: RawMatch[] = []; + const re = /\bon the (\d{1,2})(?:st|nd|rd|th)\b/gi; + let m: RegExpExecArray | null; + while ((m = re.exec(text))) { + const day = Number(m[1]); + if (day < 1 || day > 31) continue; + const month = utcMonth(anchorMs); + const year = utcYear(anchorMs); + const time = findTimeNear(text, m.index + m[0].length); + let dateMs = buildUtc(year, month, day, time); + // "on the 14th" means the upcoming 14th; if already well past, roll a month. + if (dateMs < anchorMs - SIX_MONTHS_MS) { + dateMs = buildUtc(year + 1, month, day, time); + } + out.push({ + dateMs, + hasTime: !!time, + matched: m[0], + index: m.index, + baseConfidence: 0.85, + }); + } + return out; +} + +// --------------------------------------------------------------------------- +// Relative detectors (anchored purely to anchorMs) +// --------------------------------------------------------------------------- + +function detectRelative(text: string, anchorMs: number): RawMatch[] { + const out: RawMatch[] = []; + const dayMs = 1000 * 60 * 60 * 24; + + const add = (offsetDays: number, m: RegExpExecArray) => { + const time = findTimeNear(text, m.index + m[0].length); + const dateMs = applyTime(anchorMs + offsetDays * dayMs, time); + out.push({ + dateMs, + hasTime: !!time, + matched: m[0], + index: m.index, + baseConfidence: 0.6, + }); + }; + + let m: RegExpExecArray | null; + + const reToday = /\btoday\b/gi; + while ((m = reToday.exec(text))) add(0, m); + + const reTomorrow = /\btomorrow\b/gi; + while ((m = reTomorrow.exec(text))) add(1, m); + + const reNextWeek = /\bnext week\b/gi; + while ((m = reNextWeek.exec(text))) add(7, m); + + // "in N days" / "in N weeks" + const reInN = /\bin (\d{1,3}) (day|days|week|weeks)\b/gi; + while ((m = reInN.exec(text))) { + const n = Number(m[1]); + const unit = m[2]!.toLowerCase(); + const days = unit.startsWith('week') ? n * 7 : n; + add(days, m); + } + + // "next Monday".."next Friday" (and weekend days) + const reNextDow = /\bnext (sunday|monday|tuesday|wednesday|thursday|friday|saturday)\b/gi; + while ((m = reNextDow.exec(text))) { + const target = WEEKDAYS[m[1]!.toLowerCase()]!; + const anchorDow = utcDow(anchorMs); + // Days until the target weekday in the FOLLOWING week (1..7, never 0). + let delta = (target - anchorDow + 7) % 7; + if (delta === 0) delta = 7; + delta += 7; // "next" => skip the current week's occurrence. + add(delta, m); + } + + return out; +} + +// --------------------------------------------------------------------------- +// Time-of-day extraction +// --------------------------------------------------------------------------- + +/** Look for a clock time shortly after the date phrase (within ~16 chars). */ +function findTimeNear(text: string, from: number): TimeOfDay | undefined { + const window = text.slice(from, from + 18); + return parseTime(window); +} + +function parseTime(s: string): TimeOfDay | undefined { + // "at 3pm", "at 3:30 pm", "3pm" + const re12 = /\b(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm)\b/i; + const m12 = re12.exec(s); + if (m12) { + let hour = Number(m12[1]); + const minute = m12[2] ? Number(m12[2]) : 0; + const mer = m12[3]!.toLowerCase(); + if (hour === 12) hour = 0; + if (mer === 'pm') hour += 12; + if (hour >= 0 && hour <= 23 && minute >= 0 && minute <= 59) { + return { hour, minute }; + } + } + // 24h "14:00", "at 09:30" + const re24 = /\b(?:at\s+)?([01]?\d|2[0-3]):([0-5]\d)\b/; + const m24 = re24.exec(s); + if (m24) { + return { hour: Number(m24[1]), minute: Number(m24[2]) }; + } + return undefined; +} + +// --------------------------------------------------------------------------- +// Pure UTC helpers (constructed only from explicit components / anchorMs) +// --------------------------------------------------------------------------- + +function buildUtc( + year: number, + month: number, + day: number, + time: TimeOfDay | undefined, +): number { + return Date.UTC(year, month, day, time?.hour ?? 0, time?.minute ?? 0, 0, 0); +} + +/** Replace the time-of-day on a relative instant, keeping its UTC calendar day. */ +function applyTime(ms: number, time: TimeOfDay | undefined): number { + if (!time) return ms; + const d = new Date(ms); + return Date.UTC( + d.getUTCFullYear(), + d.getUTCMonth(), + d.getUTCDate(), + time.hour, + time.minute, + 0, + 0, + ); +} + +function utcYear(ms: number): number { + return new Date(ms).getUTCFullYear(); +} +function utcMonth(ms: number): number { + return new Date(ms).getUTCMonth(); +} +function utcDow(ms: number): number { + return new Date(ms).getUTCDay(); +} + +// --------------------------------------------------------------------------- +// Clause extraction for titles +// --------------------------------------------------------------------------- + +/** Extract a short clause around the matched date as the event title. */ +function clauseAround(text: string, index: number, length: number): string { + // Bound to the surrounding sentence by splitting on sentence punctuation. + const before = text.slice(0, index); + const after = text.slice(index + length); + + const startBreak = Math.max( + before.lastIndexOf('. '), + before.lastIndexOf('? '), + before.lastIndexOf('! '), + before.lastIndexOf('; '), + before.lastIndexOf(', '), + ); + const afterBreakRel = firstBreak(after); + + const start = startBreak >= 0 ? startBreak + 2 : 0; + const end = + index + length + (afterBreakRel >= 0 ? afterBreakRel : after.length); + + return text.slice(start, end).trim().replace(/\s+/g, ' '); +} + +function firstBreak(s: string): number { + const idxs = ['. ', '? ', '! ', '; ', ', '] + .map((p) => s.indexOf(p)) + .filter((i) => i >= 0); + if (!idxs.length) return -1; + return Math.min(...idxs); +} diff --git a/src/lib/enrich/ics.test.ts b/src/lib/enrich/ics.test.ts new file mode 100644 index 0000000..13b6523 --- /dev/null +++ b/src/lib/enrich/ics.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect } from 'vitest'; +import { toIcs } from './ics'; + +const utc = (y: number, mo: number, d: number, h = 0, mi = 0) => + Date.UTC(y, mo, d, h, mi, 0, 0); + +describe('toIcs', () => { + it('produces a valid empty calendar for []', () => { + const ics = toIcs([]); + expect(ics).toContain('BEGIN:VCALENDAR'); + expect(ics).toContain('VERSION:2.0'); + expect(ics).toContain('END:VCALENDAR'); + expect(ics).not.toContain('BEGIN:VEVENT'); + }); + + it('uses CRLF line endings', () => { + const ics = toIcs([]); + expect(ics).toContain('\r\n'); + expect(ics.endsWith('\r\n')).toBe(true); + // No bare LF without a preceding CR. + expect(/[^\r]\n/.test(ics)).toBe(false); + }); + + it('emits an all-day VEVENT with VALUE=DATE', () => { + const ics = toIcs([ + { title: 'Final exam', dateMs: utc(2026, 5, 20), allDay: true }, + ]); + expect(ics).toContain('BEGIN:VEVENT'); + expect(ics).toContain('DTSTART;VALUE=DATE:20260620'); + expect(ics).toContain('SUMMARY:Final exam'); + }); + + it('emits a timed VEVENT with a UTC DTSTART', () => { + const ics = toIcs([ + { title: 'Meeting', dateMs: utc(2026, 5, 20, 14, 30), allDay: false }, + ]); + expect(ics).toContain('DTSTART:20260620T143000Z'); + }); + + it('derives DTSTAMP from the event dateMs', () => { + const ics = toIcs([ + { title: 'X', dateMs: utc(2026, 5, 20, 9, 0), allDay: true }, + ]); + expect(ics).toContain('DTSTAMP:20260620T090000Z'); + }); + + it('UID is deterministic (no randomness) and stable across calls', () => { + const ev = { title: 'Exam', dateMs: utc(2026, 5, 20), allDay: true }; + const a = toIcs([ev]); + const b = toIcs([ev]); + expect(a).toBe(b); + const uid = a.split('\r\n').find((l) => l.startsWith('UID:')); + expect(uid).toBeDefined(); + expect(uid).toContain('@wisp'); + }); + + it('changes UID when title or date changes', () => { + const a = toIcs([{ title: 'A', dateMs: utc(2026, 5, 20), allDay: true }]); + const b = toIcs([{ title: 'B', dateMs: utc(2026, 5, 20), allDay: true }]); + const uidA = a.split('\r\n').find((l) => l.startsWith('UID:')); + const uidB = b.split('\r\n').find((l) => l.startsWith('UID:')); + expect(uidA).not.toBe(uidB); + }); + + it('escapes comma, semicolon, backslash and newline in SUMMARY', () => { + const ics = toIcs([ + { + title: 'a, b; c \\ d\ne', + dateMs: utc(2026, 5, 20), + allDay: true, + }, + ]); + const line = ics + .replace(/\r\n /g, '') // unfold first + .split('\r\n') + .find((l) => l.startsWith('SUMMARY:')); + expect(line).toBe('SUMMARY:a\\, b\\; c \\\\ d\\ne'); + }); + + it('folds lines longer than 75 octets with a leading space', () => { + const long = 'X'.repeat(200); + const ics = toIcs([{ title: long, dateMs: utc(2026, 5, 20), allDay: true }]); + for (const line of ics.split('\r\n')) { + // Each physical line must be <=75 octets. + expect(Buffer.byteLength(line, 'utf8')).toBeLessThanOrEqual(75); + } + // Unfolding restores the SUMMARY content. + const unfolded = ics.replace(/\r\n /g, ''); + expect(unfolded).toContain(`SUMMARY:${long}`); + }); + + it('emits one VEVENT per event', () => { + const ics = toIcs([ + { title: 'A', dateMs: utc(2026, 5, 20), allDay: true }, + { title: 'B', dateMs: utc(2026, 5, 21), allDay: true }, + ]); + const count = ics.split('BEGIN:VEVENT').length - 1; + expect(count).toBe(2); + }); +}); diff --git a/src/lib/enrich/ics.ts b/src/lib/enrich/ics.ts new file mode 100644 index 0000000..0f82d9c --- /dev/null +++ b/src/lib/enrich/ics.ts @@ -0,0 +1,142 @@ +// Pure RFC 5545 iCalendar serialization. Deterministic (no randomness, no +// wall-clock): UID and DTSTAMP are derived from the event itself, so the same +// input always yields byte-identical output. CRLF line endings, 75-octet +// folding. + +interface IcsEvent { + title: string; + dateMs: number; + allDay: boolean; +} + +/** Serialize events to a VCALENDAR string. Valid (and non-empty) even for []. */ +export function toIcs(events: IcsEvent[]): string { + const lines: string[] = [ + 'BEGIN:VCALENDAR', + 'VERSION:2.0', + 'PRODID:-//Wisp//Enrich//EN', + 'CALSCALE:GREGORIAN', + ]; + + for (const ev of events) { + const stamp = formatUtc(ev.dateMs); + lines.push('BEGIN:VEVENT'); + lines.push(`UID:${uidFor(ev)}`); + lines.push(`DTSTAMP:${stamp}`); + if (ev.allDay) { + lines.push(`DTSTART;VALUE=DATE:${formatDate(ev.dateMs)}`); + } else { + lines.push(`DTSTART:${stamp}`); + } + lines.push(`SUMMARY:${escapeText(ev.title)}`); + lines.push('END:VEVENT'); + } + + lines.push('END:VCALENDAR'); + + return lines.map(foldLine).join('\r\n') + '\r\n'; +} + +// --------------------------------------------------------------------------- +// Deterministic UID derivation (FNV-1a hash of title + dateMs) +// --------------------------------------------------------------------------- + +function uidFor(ev: IcsEvent): string { + const h = fnv1a(`${ev.title}|${ev.dateMs}`); + return `${h}@wisp`; +} + +function fnv1a(s: string): string { + let hash = 0x811c9dc5; + for (let i = 0; i < s.length; i++) { + hash ^= s.charCodeAt(i); + // 32-bit FNV prime multiply via shifts, kept unsigned. + hash = Math.imul(hash, 0x01000193) >>> 0; + } + return hash.toString(16).padStart(8, '0'); +} + +// --------------------------------------------------------------------------- +// Date formatting from dateMs only (UTC) +// --------------------------------------------------------------------------- + +function formatDate(ms: number): string { + const d = new Date(ms); + return ( + pad(d.getUTCFullYear(), 4) + pad(d.getUTCMonth() + 1) + pad(d.getUTCDate()) + ); +} + +function formatUtc(ms: number): string { + const d = new Date(ms); + return ( + pad(d.getUTCFullYear(), 4) + + pad(d.getUTCMonth() + 1) + + pad(d.getUTCDate()) + + 'T' + + pad(d.getUTCHours()) + + pad(d.getUTCMinutes()) + + pad(d.getUTCSeconds()) + + 'Z' + ); +} + +function pad(n: number, width = 2): string { + return String(n).padStart(width, '0'); +} + +// --------------------------------------------------------------------------- +// RFC 5545 text escaping + line folding +// --------------------------------------------------------------------------- + +/** Escape backslash, semicolon, comma, and newlines per RFC 5545 §3.3.11. */ +function escapeText(s: string): string { + return s + .replace(/\\/g, '\\\\') + .replace(/;/g, '\\;') + .replace(/,/g, '\\,') + .replace(/\r\n|\n|\r/g, '\\n'); +} + +/** + * Fold a content line to <=75 octets, continuation lines starting with a + * single space. UTF-8 aware: never splits a multi-byte character across the + * fold boundary, so every emitted line is valid UTF-8. + */ +function foldLine(line: string): string { + if (utf8Len(line) <= 75) return line; + + const chunks: string[] = []; + // First line: 75 octets. Continuation lines: 74 octets (1 leading space). + let limit = 75; + let current = ''; + let currentBytes = 0; + + for (const ch of line) { + const w = utf8Len(ch); + if (currentBytes + w > limit) { + chunks.push(current); + current = ''; + currentBytes = 0; + limit = 74; + } + current += ch; + currentBytes += w; + } + if (current) chunks.push(current); + + return chunks.map((c, i) => (i === 0 ? c : ' ' + c)).join('\r\n'); +} + +/** Number of UTF-8 octets a string encodes to. */ +function utf8Len(s: string): number { + let n = 0; + for (const ch of s) { + const cp = ch.codePointAt(0)!; + if (cp < 0x80) n += 1; + else if (cp < 0x800) n += 2; + else if (cp < 0x10000) n += 3; + else n += 4; + } + return n; +} diff --git a/src/lib/enrich/index.ts b/src/lib/enrich/index.ts new file mode 100644 index 0000000..c05bc21 --- /dev/null +++ b/src/lib/enrich/index.ts @@ -0,0 +1,11 @@ +// Phase 4 enrichment barrel: pure detection/serialization + network lookups. +// Pure modules contain NO transcript text in any outbound URL; the network +// module sends only the bare identifier/topic. See each file's header. + +export * from './types'; +export * from './dates'; +export * from './citations'; +export * from './ics'; +export * from './bib'; +export * from './links'; +export * from './lookup'; diff --git a/src/lib/enrich/links.test.ts b/src/lib/enrich/links.test.ts new file mode 100644 index 0000000..87d7436 --- /dev/null +++ b/src/lib/enrich/links.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect } from 'vitest'; +import { lookupLinksFor, bookLinksFor } from './links'; +import type { Citation, RefMetadata } from './types'; + +const cite = (kind: Citation['kind'], value: string): Citation => ({ + kind, + value, + raw: value, +}); + +describe('lookupLinksFor', () => { + it('builds a DOI resolver link', () => { + const l = lookupLinksFor(cite('doi', '10.1038/nphys1170')); + expect(l.doi).toBe('https://doi.org/10.1038%2Fnphys1170'); + }); + + it('builds an arXiv abstract link', () => { + const l = lookupLinksFor(cite('arxiv', '1706.03762')); + expect(l.arxiv).toBe('https://arxiv.org/abs/1706.03762'); + }); + + it('points openLibrary at the ISBN page for isbn citations', () => { + const l = lookupLinksFor(cite('isbn', '9780134685991')); + expect(l.openLibrary).toBe('https://openlibrary.org/isbn/9780134685991'); + }); + + it('always includes wikipedia/googleBooks search links', () => { + const l = lookupLinksFor(cite('author-year', 'Vaswani (2017)')); + expect(l.wikipedia).toContain( + 'https://en.wikipedia.org/wiki/Special:Search?search=', + ); + expect(l.googleBooks).toContain('https://www.google.com/search?tbm=bks&q='); + }); + + it('always includes annasArchive and libgen search links', () => { + const l = lookupLinksFor(cite('doi', '10.1/x')); + expect(l.annasArchive).toContain('https://annas-archive.org/search?q='); + expect(l.libgen).toContain('https://libgen.is/search.php?req='); + }); + + it('prefers resolved metadata title for the search query, URL-encoded', () => { + const meta: RefMetadata = { + title: 'Attention Is All You Need', + source: 'Crossref', + }; + const l = lookupLinksFor(cite('arxiv', '1706.03762'), meta); + const q = encodeURIComponent('Attention Is All You Need'); + expect(l.wikipedia).toContain(q); + expect(l.annasArchive).toContain(q); + }); + + it('falls back to the citation value when no metadata title', () => { + const l = lookupLinksFor(cite('author-year', 'Smith et al. (2021)')); + const q = encodeURIComponent('Smith et al. (2021)'); + expect(l.googleBooks).toContain(q); + }); +}); + +describe('bookLinksFor', () => { + it('builds the full search set from a free-text query (no doi/arxiv)', () => { + const l = bookLinksFor('Introduction to Algorithms'); + const q = encodeURIComponent('Introduction to Algorithms'); + expect(l.doi).toBeUndefined(); + expect(l.arxiv).toBeUndefined(); + expect(l.openLibrary).toBe(`https://openlibrary.org/search?q=${q}`); + expect(l.wikipedia).toContain(q); + expect(l.googleBooks).toContain(q); + expect(l.annasArchive).toContain(q); + expect(l.libgen).toContain(q); + }); + + it('URL-encodes special characters', () => { + const l = bookLinksFor('C&A: a/b test'); + expect(l.libgen).toContain(encodeURIComponent('C&A: a/b test')); + expect(l.libgen).not.toContain(' '); + }); +}); diff --git a/src/lib/enrich/links.ts b/src/lib/enrich/links.ts new file mode 100644 index 0000000..b5b24d9 --- /dev/null +++ b/src/lib/enrich/links.ts @@ -0,0 +1,47 @@ +// Pure outbound-link builders. NO network here — these only construct URLs. +// Privacy: the only data placed in a URL is the extracted citation value or a +// resolved title — never transcript text. Anna's Archive / LibGen appear as +// SEARCH urls only; nothing in this app fetches or proxies them. + +import type { Citation, LookupLinks, RefMetadata } from './types'; + +/** Build lookup links for a detected citation, optionally enriched by metadata. */ +export function lookupLinksFor(c: Citation, meta?: RefMetadata): LookupLinks { + const query = (meta?.title || c.value).trim(); + + const links: LookupLinks = { + ...searchLinks(query), + }; + + if (c.kind === 'doi') { + links.doi = `https://doi.org/${encodeURIComponent(c.value)}`; + } + if (c.kind === 'arxiv') { + links.arxiv = `https://arxiv.org/abs/${encodeURIComponent(c.value)}`; + } + if (c.kind === 'isbn') { + // Direct ISBN page beats a free-text Open Library search. + links.openLibrary = `https://openlibrary.org/isbn/${encodeURIComponent( + c.value, + )}`; + } + + return links; +} + +/** Build the same set of links from a free-text query (no DOI/arXiv). */ +export function bookLinksFor(query: string): LookupLinks { + return searchLinks(query.trim()); +} + +/** The query-driven search links shared by both entry points. */ +function searchLinks(query: string): LookupLinks { + const q = encodeURIComponent(query); + return { + wikipedia: `https://en.wikipedia.org/wiki/Special:Search?search=${q}`, + googleBooks: `https://www.google.com/search?tbm=bks&q=${q}`, + openLibrary: `https://openlibrary.org/search?q=${q}`, + annasArchive: `https://annas-archive.org/search?q=${q}`, + libgen: `https://libgen.is/search.php?req=${q}`, + }; +} diff --git a/src/lib/enrich/lookup.ts b/src/lib/enrich/lookup.ts new file mode 100644 index 0000000..f3fa0ea --- /dev/null +++ b/src/lib/enrich/lookup.ts @@ -0,0 +1,290 @@ +// NETWORK metadata client for Phase 4 enrichment. +// +// PRIVACY: every request below sends ONLY the extracted citation id (DOI / +// arXiv id / ISBN) or the topic string — never any transcript text. Detection +// already happened locally; this module just resolves a bare identifier to +// public metadata. +// +// All endpoints are CORS-friendly, legitimate, metadata-only APIs (Crossref, +// OpenAlex, Open Library, Wikipedia REST). Their responses include the +// `Access-Control-Allow-Origin` header, so they are readable even under our +// COEP `require-corp` / COOP `same-origin` isolation (CORS responses satisfy +// require-corp without needing CORP headers). We never fetch or proxy any +// shadow-library content here — those are search-URL only and live in links.ts. +// +// Every function is defensive: ~8s AbortController timeout, try/catch that +// resolves to `undefined`, and never throws. Results are memoised in an +// in-memory Map for the session. + +import type { Citation, RefMetadata } from './types'; + +/** Per-session metadata cache, keyed by `${kind}:${value}` / `wiki:${topic}`. */ +const cache = new Map(); +const wikiCache = new Map< + string, + { title: string; extract: string; url: string } | undefined +>(); + +const TIMEOUT_MS = 8000; + +/** GET JSON with an abort timeout. Returns parsed JSON or undefined on any + * failure (network, non-2xx, timeout, parse). Never throws. */ +async function fetchJson(url: string): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), TIMEOUT_MS); + try { + const res = await fetch(url, { + method: 'GET', + headers: { Accept: 'application/json' }, + signal: controller.signal, + }); + if (!res.ok) return undefined; + return (await res.json()) as unknown; + } catch { + return undefined; + } finally { + clearTimeout(timer); + } +} + +/** Narrow an unknown to a plain record for safe property access. */ +function asRecord(v: unknown): Record | undefined { + return typeof v === 'object' && v !== null + ? (v as Record) + : undefined; +} + +function asString(v: unknown): string | undefined { + return typeof v === 'string' && v.length > 0 ? v : undefined; +} + +function asNumber(v: unknown): number | undefined { + return typeof v === 'number' && Number.isFinite(v) ? v : undefined; +} + +// --------------------------------------------------------------------------- +// Crossref (DOI) +// --------------------------------------------------------------------------- + +function parseCrossref(json: unknown, value: string): RefMetadata | undefined { + const root = asRecord(json); + const message = asRecord(root?.message); + if (!message) return undefined; + + const titleArr = Array.isArray(message.title) ? message.title : []; + const title = asString(titleArr[0]); + + const authors: string[] = []; + if (Array.isArray(message.author)) { + for (const a of message.author) { + const rec = asRecord(a); + if (!rec) continue; + const given = asString(rec.given); + const family = asString(rec.family); + const name = [given, family].filter(Boolean).join(' ').trim(); + if (name) authors.push(name); + } + } + + // Year: prefer published, then published-print/online, then issued. + const year = + crossrefYear(message.published) ?? + crossrefYear(message['published-print']) ?? + crossrefYear(message['published-online']) ?? + crossrefYear(message.issued); + + return { + title, + authors: authors.length ? authors : undefined, + year, + source: 'Crossref', + url: `https://doi.org/${value}`, + }; +} + +/** Crossref date parts live at `['date-parts'][0][0]`. */ +function crossrefYear(field: unknown): number | undefined { + const rec = asRecord(field); + const parts = rec?.['date-parts']; + if (!Array.isArray(parts)) return undefined; + const first = parts[0]; + if (!Array.isArray(first)) return undefined; + return asNumber(first[0]); +} + +// --------------------------------------------------------------------------- +// OpenAlex (arXiv, and DOI fallback) +// --------------------------------------------------------------------------- + +function parseOpenAlex( + json: unknown, + source: string, + url: string, +): RefMetadata | undefined { + const work = asRecord(json); + if (!work) return undefined; + + const title = asString(work.title) ?? asString(work.display_name); + + const authors: string[] = []; + if (Array.isArray(work.authorships)) { + for (const a of work.authorships) { + const rec = asRecord(a); + const author = asRecord(rec?.author); + const name = asString(author?.display_name); + if (name) authors.push(name); + } + } + + const year = asNumber(work.publication_year); + + // OpenAlex returns an error object (with no title) for unknown ids. + if (!title && !authors.length && year === undefined) return undefined; + + return { + title, + authors: authors.length ? authors : undefined, + year, + source, + url, + }; +} + +// --------------------------------------------------------------------------- +// Open Library (ISBN) +// --------------------------------------------------------------------------- + +function parseOpenLibrary( + json: unknown, + value: string, +): RefMetadata | undefined { + const book = asRecord(json); + if (!book) return undefined; + + const title = asString(book.title); + + // Open Library's /isbn/{}.json gives `by_statement` (a free-text credit) + // rather than structured authors; surface it as a single author when present. + const byStatement = asString(book.by_statement); + const authors = byStatement ? [byStatement] : undefined; + + // publish_date is free-text (e.g. "March 2004"); pull a 4-digit year out. + let year: number | undefined; + const publishDate = asString(book.publish_date); + if (publishDate) { + const m = publishDate.match(/\b(\d{4})\b/); + if (m) year = asNumber(Number(m[1])); + } + + if (!title && !authors && year === undefined) return undefined; + + return { + title, + authors, + year, + source: 'Open Library', + url: `https://openlibrary.org/isbn/${value}`, + }; +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Resolve a detected citation to public metadata. + * + * Sends ONLY the citation's identifier (DOI / arXiv id / ISBN) to a + * metadata-only API — never any transcript text. Returns `undefined` on any + * failure or for `author-year` citations (the UI offers search links instead). + * Never throws. + */ +export async function resolveCitation( + c: Citation, +): Promise { + const key = `${c.kind}:${c.value}`; + if (cache.has(key)) return cache.get(key); + + let result: RefMetadata | undefined; + + switch (c.kind) { + case 'doi': { + // Crossref is the authoritative DOI registry and is CORS-enabled. + const json = await fetchJson( + `https://api.crossref.org/works/${encodeURIComponent(c.value)}`, + ); + result = parseCrossref(json, c.value); + // Fallback to OpenAlex if Crossref had nothing (e.g. DataCite DOI). + if (!result) { + const alt = await fetchJson( + `https://api.openalex.org/works/doi:${encodeURIComponent(c.value)}`, + ); + result = parseOpenAlex(alt, 'OpenAlex', `https://doi.org/${c.value}`); + } + break; + } + case 'arxiv': { + // OpenAlex resolves arXiv ids over CORS, avoiding arXiv's + // non-CORS Atom export API. + const json = await fetchJson( + `https://api.openalex.org/works/arxiv:${encodeURIComponent(c.value)}`, + ); + result = parseOpenAlex( + json, + 'OpenAlex', + `https://arxiv.org/abs/${c.value}`, + ); + break; + } + case 'isbn': { + const json = await fetchJson( + `https://openlibrary.org/isbn/${encodeURIComponent(c.value)}.json`, + ); + result = parseOpenLibrary(json, c.value); + break; + } + case 'author-year': + // No reliable id to resolve; UI falls back to search links. + result = undefined; + break; + } + + cache.set(key, result); + return result; +} + +/** + * Fetch a one-paragraph Wikipedia summary for a topic. + * + * Sends ONLY the topic string to Wikipedia's REST summary endpoint (CORS- + * enabled) — never any transcript text. Returns `undefined` on 404 / any + * failure. Never throws. + */ +export async function wikipediaSummary( + topic: string, +): Promise<{ title: string; extract: string; url: string } | undefined> { + const key = `wiki:${topic}`; + if (wikiCache.has(key)) return wikiCache.get(key); + + let result: { title: string; extract: string; url: string } | undefined; + + const json = await fetchJson( + `https://en.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent( + topic, + )}`, + ); + const root = asRecord(json); + if (root) { + const title = asString(root.title); + const extract = asString(root.extract); + const contentUrls = asRecord(root.content_urls); + const desktop = asRecord(contentUrls?.desktop); + const url = asString(desktop?.page); + if (title && extract && url) { + result = { title, extract, url }; + } + } + + wikiCache.set(key, result); + return result; +} diff --git a/src/lib/enrich/types.ts b/src/lib/enrich/types.ts new file mode 100644 index 0000000..1637fd8 --- /dev/null +++ b/src/lib/enrich/types.ts @@ -0,0 +1,55 @@ +// Shared types for Phase 4 enrichment: dates -> calendar, references -> lookups. +// All detection is pure over Segment text; network lookups send only the +// extracted topic/DOI/ISBN string — never the transcript itself. + +/** A detected date/deadline the user can confirm into their calendar. */ +export interface CandidateEvent { + /** Short event title (the surrounding clause, trimmed). */ + title: string; + /** Resolved absolute time, ms since epoch. */ + dateMs: number; + /** All-day if no specific time was detected. */ + allDay: boolean; + /** Source segment for jump-to-lecture. */ + sourceSegmentId?: string; + /** Audio start time of the source segment. */ + start?: number; + /** The matched text. */ + raw: string; + /** 0..1 detection confidence (relative dates are lower). */ + confidence: number; +} + +/** A detected citation/reference in the lecture. */ +export interface Citation { + kind: 'doi' | 'arxiv' | 'isbn' | 'author-year'; + /** Normalized id (DOI, arXiv id, ISBN-13/10) or the raw author-year string. */ + value: string; + sourceSegmentId?: string; + start?: number; + raw: string; +} + +/** Resolved metadata for a citation, from a legitimate metadata API. */ +export interface RefMetadata { + title?: string; + authors?: string[]; + year?: number; + /** Which API answered, e.g. 'Crossref' | 'OpenAlex' | 'Open Library'. */ + source: string; + /** Canonical URL (DOI resolver, work page, etc.). */ + url?: string; +} + +/** Outbound links for finding a reference/book. Legit sources first; the + * shadow-library links (Anna's Archive / LibGen) are search URLs only. */ +export interface LookupLinks { + doi?: string; + arxiv?: string; + openAlex?: string; + openLibrary?: string; + googleBooks?: string; + wikipedia?: string; + annasArchive?: string; + libgen?: string; +}