diff --git a/src/app/_layout.tsx b/src/app/_layout.tsx index b9e47d5..594c0f6 100644 --- a/src/app/_layout.tsx +++ b/src/app/_layout.tsx @@ -19,6 +19,8 @@ export default function RootLayout() { + + diff --git a/src/app/index.tsx b/src/app/index.tsx index fbe3c25..c75c42a 100644 --- a/src/app/index.tsx +++ b/src/app/index.tsx @@ -14,7 +14,7 @@ import { ThemedText } from '@/components/themed-text'; import { ThemedView } from '@/components/themed-view'; import { MaxContentWidth, Spacing } from '@/constants/theme'; import { useTheme } from '@/hooks/use-theme'; -import type { TranscriptMeta } from '@/lib/db'; +import { getRepo, type TranscriptMeta } from '@/lib/db'; import { formatClock } from '@/lib/format'; import { MODELS } from '@/lib/models/catalog'; import { pickAudio, type PickedAudio } from '@/lib/pickAudio'; @@ -31,11 +31,21 @@ export default function LibraryScreen() { const refreshCourses = useCourses((s) => s.refresh); const job = useTranscribe(); const [picked, setPicked] = useState(null); + const [dueCount, setDueCount] = useState(0); useFocusEffect( useCallback(() => { void refresh(); void refreshCourses(); + let alive = true; + void getRepo() + .flashcardCounts() + .then((c) => { + if (alive) setDueCount(c.due); + }); + return () => { + alive = false; + }; }, [refresh, refreshCourses]), ); @@ -82,6 +92,13 @@ export default function LibraryScreen() { Courses + + + + {dueCount > 0 ? `Study (${dueCount})` : 'Study'} + + + ⚙ Settings diff --git a/src/app/quiz.tsx b/src/app/quiz.tsx new file mode 100644 index 0000000..4608832 --- /dev/null +++ b/src/app/quiz.tsx @@ -0,0 +1,188 @@ +import { Stack, useLocalSearchParams, useRouter } from 'expo-router'; +import { useEffect, useMemo, useState } from 'react'; +import { ActivityIndicator, Pressable, ScrollView, StyleSheet, View } from 'react-native'; + +import { ThemedText } from '@/components/themed-text'; +import { ThemedView } from '@/components/themed-view'; +import { MaxContentWidth, Spacing } from '@/constants/theme'; +import { useTheme } from '@/hooks/use-theme'; +import { getRepo, type Transcript } from '@/lib/db'; +import { formatClock } from '@/lib/format'; +import { generateQuiz, glossary, type QuizQuestion } from '@/lib/learn'; + +const ACCENT = '#3c87f7'; +const CORRECT = '#30a46c'; +const WRONG = '#e5484d'; + +export default function QuizScreen() { + const theme = useTheme(); + const router = useRouter(); + const { id } = useLocalSearchParams<{ id: string }>(); + + const [transcript, setTranscript] = useState(undefined); + + useEffect(() => { + let alive = true; + void getRepo() + .get(id) + .then((t) => { + if (alive) setTranscript(t ?? null); + }); + return () => { + alive = false; + }; + }, [id]); + + const questions = useMemo( + () => (transcript ? generateQuiz(glossary(transcript.segments)) : []), + [transcript], + ); + + const [index, setIndex] = useState(0); + const [picked, setPicked] = useState(null); + const [score, setScore] = useState(0); + + const total = questions.length; + const q = index < total ? questions[index] : undefined; + const finished = total > 0 && index >= total; + + const onPick = (optionIndex: number) => { + if (picked !== null || !q) return; + setPicked(optionIndex); + if (optionIndex === q.answerIndex) setScore((s) => s + 1); + }; + + const next = () => { + setPicked(null); + setIndex((i) => i + 1); + }; + + const restart = () => { + setIndex(0); + setPicked(null); + setScore(0); + }; + + if (transcript === undefined) { + return ( + + + + ); + } + + return ( + + + + {transcript === null ? ( + + Transcript not found. + + ) : total === 0 ? ( + + Not enough glossary terms to build a quiz for this lecture. + + ) : finished ? ( + + Quiz complete + + You scored {score} of {total}. + + [styles.primaryBtn, { backgroundColor: ACCENT, opacity: pressed ? 0.85 : 1 }]}> + Try again + + router.push({ pathname: '/transcript/[id]', params: { id } })} + style={({ pressed }) => [styles.secondaryBtn, { backgroundColor: theme.backgroundSelected, opacity: pressed ? 0.85 : 1 }]}> + Back to lecture + + + ) : q ? ( + <> + + Question {index + 1} of {total} + + {q.question} + + + {q.options.map((opt, i) => { + const isAnswer = i === q.answerIndex; + const isPicked = i === picked; + const revealed = picked !== null; + const border = + revealed && isAnswer ? CORRECT : revealed && isPicked ? WRONG : theme.backgroundElement; + return ( + onPick(i)} + disabled={revealed} + style={({ pressed }) => [ + styles.option, + { backgroundColor: theme.backgroundElement, borderColor: border, opacity: pressed && !revealed ? 0.8 : 1 }, + ]}> + {opt} + + ); + })} + + + {picked !== null && ( + + + {picked === q.answerIndex ? 'Correct' : 'Incorrect'} + + {q.start !== undefined && ( + + router.push({ + pathname: '/transcript/[id]', + params: { id, t: String(Math.floor(q.start ?? 0)) }, + }) + } + hitSlop={6}> + Jump to lecture ({formatClock(q.start)}) + + )} + [styles.primaryBtn, { backgroundColor: ACCENT, opacity: pressed ? 0.85 : 1 }]}> + + {index + 1 >= total ? 'See score' : 'Next'} + + + + )} + + ) : null} + + + ); +} + +function Centered({ children }: { children: React.ReactNode }) { + return {children}; +} + +const styles = StyleSheet.create({ + fill: { flex: 1 }, + centered: { alignItems: 'center', justifyContent: 'center' }, + content: { + padding: Spacing.three, + gap: Spacing.three, + maxWidth: MaxContentWidth, + width: '100%', + alignSelf: 'center', + }, + card: { padding: Spacing.four, borderRadius: Spacing.three, gap: Spacing.three }, + question: { fontSize: 24, lineHeight: 32 }, + options: { gap: Spacing.two }, + option: { padding: Spacing.three, borderRadius: Spacing.three, borderWidth: 2 }, + feedback: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.two }, + primaryBtn: { paddingVertical: Spacing.three, borderRadius: Spacing.three, alignItems: 'center' }, + primaryBtnText: { color: '#fff', fontWeight: '700', fontSize: 16 }, + secondaryBtn: { paddingVertical: Spacing.three, borderRadius: Spacing.three, alignItems: 'center' }, + pad: { paddingVertical: Spacing.four, textAlign: 'center' }, +}); diff --git a/src/app/study.tsx b/src/app/study.tsx new file mode 100644 index 0000000..14f9fc0 --- /dev/null +++ b/src/app/study.tsx @@ -0,0 +1,204 @@ +import { Stack, useFocusEffect, useRouter } from 'expo-router'; +import { useCallback, useState } from 'react'; +import { ActivityIndicator, Pressable, ScrollView, StyleSheet, View } from 'react-native'; + +import { ThemedText } from '@/components/themed-text'; +import { ThemedView } from '@/components/themed-view'; +import { MaxContentWidth, Spacing } from '@/constants/theme'; +import { useTheme } from '@/hooks/use-theme'; +import { getRepo, type Flashcard } from '@/lib/db'; +import { downloadText } from '@/lib/download'; +import { review } from '@/lib/learn'; +import { useCourses } from '@/stores/coursesStore'; + +const ACCENT = '#3c87f7'; + +// Course scope for the due queue: 'all' = no filter, null = Unsorted, string = course id. +type Scope = 'all' | string | null; + +const GRADES: { label: string; grade: 0 | 1 | 2 | 3 }[] = [ + { label: 'Again', grade: 0 }, + { label: 'Hard', grade: 1 }, + { label: 'Good', grade: 2 }, + { label: 'Easy', grade: 3 }, +]; + +export default function StudyScreen() { + const theme = useTheme(); + const router = useRouter(); + + const courses = useCourses((s) => s.items); + const refreshCourses = useCourses((s) => s.refresh); + + const [scope, setScope] = useState('all'); + const [queue, setQueue] = useState(null); + const [index, setIndex] = useState(0); + const [revealed, setRevealed] = useState(false); + + const load = useCallback(async (s: Scope) => { + setQueue(null); + setIndex(0); + setRevealed(false); + const cards = await getRepo().listDueFlashcards( + s === 'all' ? {} : { courseId: s }, + ); + setQueue(cards); + }, []); + + // Reload the due queue and courses every time the screen gains focus. + useFocusEffect( + useCallback(() => { + void refreshCourses(); + void load(scope); + }, [refreshCourses, load, scope]), + ); + + const pickScope = (s: Scope) => { + setScope(s); + void load(s); + }; + + const total = queue?.length ?? 0; + const card = queue && index < total ? queue[index] : undefined; + + const grade = async (g: 0 | 1 | 2 | 3) => { + if (!card) return; + await getRepo().updateFlashcardSrs(card.id, review(card.srs, g, Date.now())); + setRevealed(false); + setIndex((i) => i + 1); + }; + + const exportAnki = async () => { + const all = await getRepo().listFlashcards(); + const rows = all.map((c) => `${csvField(c.front)},${csvField(c.back)}`); + const csv = ['front,back', ...rows].join('\r\n'); + downloadText('wisp-flashcards.csv', 'text/csv', csv); + }; + + const courseName = (cid?: string | null) => + cid ? courses.find((c) => c.id === cid)?.name ?? 'Course' : 'Unsorted'; + + return ( + + ( + void exportAnki()} hitSlop={8}> + Export Anki (.csv) + + ), + }} + /> + + + pickScope('all')} /> + pickScope(null)} /> + {courses.map((c) => ( + pickScope(c.id)} /> + ))} + + + {queue === null ? ( + + ) : card === undefined ? ( + + {total === 0 + ? 'No cards due — generate some from a lecture.' + : 'All done — no more cards due right now.'} + + ) : ( + <> + + {index + 1} of {total} + + + + {courseName(card.courseId)} + {card.front} + + {revealed ? ( + <> + + {card.back} + {card.start !== undefined && ( + + router.push({ + pathname: '/transcript/[id]', + params: { id: card.transcriptId, t: String(Math.floor(card.start ?? 0)) }, + }) + } + hitSlop={6}> + Jump to lecture + + )} + + ) : ( + setRevealed(true)} + style={({ pressed }) => [styles.showBtn, { backgroundColor: ACCENT, opacity: pressed ? 0.85 : 1 }]}> + Show answer + + )} + + + {revealed && ( + + {GRADES.map((g) => ( + void grade(g.grade)} + style={({ pressed }) => [ + styles.gradeBtn, + { backgroundColor: theme.backgroundElement, opacity: pressed ? 0.7 : 1 }, + ]}> + {g.label} + + ))} + + )} + + )} + + + ); +} + +function FilterChip({ label, active, onPress }: { label: string; active: boolean; onPress: () => void }) { + const theme = useTheme(); + return ( + + {label} + + ); +} + +/** RFC4180-quote a CSV field: wrap in quotes, double any embedded quotes. */ +function csvField(value: string): string { + return `"${value.replace(/"/g, '""')}"`; +} + +const styles = StyleSheet.create({ + fill: { flex: 1 }, + content: { + padding: Spacing.three, + gap: Spacing.three, + maxWidth: MaxContentWidth, + width: '100%', + alignSelf: 'center', + }, + filterBar: { gap: Spacing.two, paddingVertical: Spacing.one, paddingRight: Spacing.three }, + filterChip: { paddingHorizontal: Spacing.three, paddingVertical: Spacing.one, borderRadius: 999 }, + chipActive: { color: '#fff', fontWeight: '700' }, + card: { padding: Spacing.four, borderRadius: Spacing.three, gap: Spacing.three }, + front: { fontSize: 24, lineHeight: 32 }, + divider: { height: StyleSheet.hairlineWidth, marginVertical: Spacing.one }, + showBtn: { paddingVertical: Spacing.three, borderRadius: Spacing.three, alignItems: 'center' }, + showBtnText: { color: '#fff', fontWeight: '700', fontSize: 16 }, + gradeRow: { flexDirection: 'row', gap: Spacing.two }, + gradeBtn: { flex: 1, paddingVertical: Spacing.three, borderRadius: Spacing.two, alignItems: 'center' }, + pad: { paddingVertical: Spacing.four, textAlign: 'center' }, +}); diff --git a/src/app/transcript/[id].tsx b/src/app/transcript/[id].tsx index 7f16f42..90c55da 100644 --- a/src/app/transcript/[id].tsx +++ b/src/app/transcript/[id].tsx @@ -1,4 +1,4 @@ -import { Stack, useLocalSearchParams } from 'expo-router'; +import { Link, Stack, useLocalSearchParams } from 'expo-router'; import { useEffect, useMemo, useRef, useState } from 'react'; import { ActivityIndicator, @@ -18,9 +18,18 @@ import { getRepo, type Transcript } from '@/lib/db'; import { downloadText } from '@/lib/download'; import { EXPORT_META, formatTranscript, type ExportFormat } from '@/lib/export'; import { formatClock } from '@/lib/format'; +import { + cardsFromGlossary, + glossary, + summarize, + type GlossaryEntry, + type SourcedSentence, +} from '@/lib/learn'; import type { Segment } from '@/lib/types'; import { useTranscribe } from '@/stores/transcribeStore'; +const ACCENT = '#3c87f7'; + export default function TranscriptScreen() { const theme = useTheme(); const { id, t } = useLocalSearchParams<{ id: string; t?: string }>(); @@ -37,6 +46,14 @@ export default function TranscriptScreen() { const [saving, setSaving] = useState(false); const [currentTime, setCurrentTime] = useState(0); + // Study aids (deterministic, computed on demand — no storage, no model). + const [aids, setAids] = useState<{ + summary: SourcedSentence[]; + glossary: GlossaryEntry[]; + } | null>(null); + const [creatingCards, setCreatingCards] = useState(false); + const [cardsAdded, setCardsAdded] = useState(null); + const scrollRef = useRef(null); // Y offset of each rendered segment row, captured via onLayout, for scroll-to. const segmentYs = useRef>({}); @@ -165,6 +182,38 @@ export default function TranscriptScreen() { downloadText(`${safeName}.${meta.ext}`, meta.mime, content); }; + // Compute summary + glossary on demand (pure helpers, nothing persisted). + const generateAids = () => { + setAids({ summary: summarize(segments), glossary: glossary(segments) }); + setCardsAdded(null); + }; + + // Seeds derived from the current glossary — drives the "Create N flashcards" label. + const cardSeeds = useMemo( + () => (aids ? cardsFromGlossary(aids.glossary) : []), + [aids], + ); + + const createCards = async () => { + if (cardSeeds.length === 0 || creatingCards) return; + setCreatingCards(true); + try { + const created = await getRepo().createFlashcards( + cardSeeds.map((s) => ({ + transcriptId: id, + courseId: transcript?.courseId ?? null, + segmentId: s.segmentId, + start: s.start, + front: s.front, + back: s.back, + })), + ); + setCardsAdded(created.length); + } finally { + setCreatingCards(false); + } + }; + if (transcript === undefined) return ; if (transcript === null) return ( @@ -205,6 +254,96 @@ export default function TranscriptScreen() { ))} + {/* Study aids — deterministic helpers, computed on demand. */} + + Study aids + + [ + styles.aidBtn, + { backgroundColor: ACCENT, opacity: segments.length === 0 ? 0.5 : pressed ? 0.85 : 1 }, + ]}> + + {aids ? 'Regenerate summary & glossary' : 'Generate summary & glossary'} + + + + [ + styles.aidBtn, + { backgroundColor: theme.backgroundSelected, opacity: pressed ? 0.85 : 1 }, + ]}> + Quiz this lecture + + + + + {aids && ( + <> + {aids.summary.length > 0 ? ( + + Summary + {aids.summary.map((s, i) => ( + seek(s.start)} hitSlop={4}> + + + {formatClock(s.start)} + + {s.text} + + + ))} + + ) : ( + No summary could be derived. + )} + + {aids.glossary.length > 0 ? ( + + Glossary + {aids.glossary.map((g, i) => ( + seek(g.start)} hitSlop={4}> + + + {formatClock(g.start)} + + + {g.term} + {` — ${g.definition}`} + + + + ))} + + ) : ( + No glossary terms found. + )} + + {cardSeeds.length > 0 && ( + void createCards()} + disabled={creatingCards} + style={({ pressed }) => [ + styles.aidBtn, + { backgroundColor: ACCENT, opacity: creatingCards ? 0.6 : pressed ? 0.85 : 1 }, + ]}> + + {creatingCards ? 'Adding…' : `Create ${cardSeeds.length} flashcards`} + + + )} + + {cardsAdded !== null && ( + + Added {cardsAdded} {cardsAdded === 1 ? 'card' : 'cards'}. + + )} + + )} + + {segments.map((s, i) => ( BLOB codecs -------------------------------------------------- // SQLite stores embeddings as raw little-endian float32 bytes. We bind a // Uint8Array view of the Float32Array's buffer on write and reconstruct a @@ -197,7 +236,7 @@ function dot(a: Float32Array, b: Float32Array): number { // idempotent (IF NOT EXISTS / pragma_table_info guards) so they also no-op on // DBs the original single-table code already created at version 0. -const TARGET_VERSION = 3; +const TARGET_VERSION = 4; type Migration = (db: SQLite.SQLiteDatabase) => Promise; @@ -314,6 +353,30 @@ const MIGRATIONS: Migration[] = [ CREATE INDEX IF NOT EXISTS idx_segvecs_courseId ON segvecs (courseId); `); }, + + // ---- v3 -> v4: flashcards store for Phase 3 spaced repetition ---------- + async (db) => { + // One row per flashcard. srs is JSON; dueAt is a denormalized copy of + // srs.due as an INTEGER so due-card queries can use an index. The + // transcriptId/courseId indexes drive the cascade + scoping paths. + await db.execAsync(` + CREATE TABLE IF NOT EXISTS flashcards ( + id TEXT PRIMARY KEY NOT NULL, + transcriptId TEXT, + courseId TEXT, + segmentId TEXT, + start REAL, + front TEXT, + back TEXT, + createdAt INTEGER, + srs TEXT, + dueAt INTEGER + ); + CREATE INDEX IF NOT EXISTS idx_flashcards_transcriptId ON flashcards (transcriptId); + CREATE INDEX IF NOT EXISTS idx_flashcards_courseId ON flashcards (courseId); + CREATE INDEX IF NOT EXISTS idx_flashcards_dueAt ON flashcards (dueAt); + `); + }, ]; async function runMigrations(db: SQLite.SQLiteDatabase): Promise { @@ -530,8 +593,9 @@ export const repo: StorageRepo = { // Delete persisted media first so we never orphan a file. await this.removeMedia(id); await db.runAsync(`DELETE FROM transcripts WHERE id = ?`, [id]); - // Cascade: drop this transcript's vectors too. + // Cascade: drop this transcript's vectors + flashcards too. await db.runAsync(`DELETE FROM segvecs WHERE transcriptId = ?`, [id]); + await db.runAsync(`DELETE FROM flashcards WHERE transcriptId = ?`, [id]); }, async search(query: string): Promise { @@ -709,6 +773,11 @@ export const repo: StorageRepo = { await db.runAsync(`UPDATE segvecs SET courseId = NULL WHERE courseId = ?`, [ id, ]); + // Flashcards in this course follow their transcripts to Unsorted. + await db.runAsync( + `UPDATE flashcards SET courseId = NULL WHERE courseId = ?`, + [id], + ); await db.runAsync(`DELETE FROM courses WHERE id = ?`, [id]); }); }, @@ -887,4 +956,173 @@ export const repo: StorageRepo = { ); return rows.map((r) => r.id); }, + + // --- flashcards + spaced repetition (Phase 3) --------------------------- + async createFlashcards(drafts: FlashcardDraft[]): Promise { + const db = await getDb(); + const now = Date.now(); + const cards: Flashcard[] = drafts.map((draft) => { + const valid = parseFlashcardDraft(draft); + // Initial SM-2 state for a brand-new card: due immediately (now). We + // compute the SRS inline here rather than importing learn/srs to keep the + // repo storage-only (no dependency on the study logic). + const srs: SrsState = { + ease: 2.5, + intervalDays: 0, + reps: 0, + lapses: 0, + due: now, + }; + return { + id: newId('f_'), + transcriptId: valid.transcriptId, + // courseId defaults to null ("Unsorted") when absent. + courseId: valid.courseId ?? null, + ...(valid.segmentId !== undefined ? { segmentId: valid.segmentId } : {}), + ...(valid.start !== undefined ? { start: valid.start } : {}), + front: valid.front, + back: valid.back, + createdAt: now, + srs, + }; + }); + await db.withTransactionAsync(async () => { + for (const card of cards) { + await db.runAsync( + `INSERT INTO flashcards + (id, transcriptId, courseId, segmentId, start, front, back, + createdAt, srs, dueAt) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + card.id, + card.transcriptId, + card.courseId ?? null, + card.segmentId ?? null, + card.start ?? null, + card.front, + card.back, + card.createdAt, + JSON.stringify(card.srs), + card.srs.due, + ], + ); + } + }); + return cards; + }, + + async listFlashcards(opts?: { + transcriptId?: string; + courseId?: string | null; + }): Promise { + const db = await getDb(); + const where: string[] = []; + const params: (string | null)[] = []; + if (opts?.transcriptId !== undefined) { + where.push('transcriptId = ?'); + params.push(opts.transcriptId); + } + if (opts && opts.courseId !== undefined) { + if (opts.courseId === null) { + where.push('courseId IS NULL'); + } else { + where.push('courseId = ?'); + params.push(opts.courseId); + } + } + const clause = where.length ? ` WHERE ${where.join(' AND ')}` : ''; + const rows = await db.getAllAsync( + `SELECT id, transcriptId, courseId, segmentId, start, front, back, + createdAt, srs, dueAt + FROM flashcards${clause}`, + params, + ); + return rows.map(rowToFlashcard); + }, + + async listDueFlashcards(opts?: { + courseId?: string | null; + now?: number; + limit?: number; + }): Promise { + const db = await getDb(); + const now = opts?.now ?? Date.now(); + // Due means dueAt <= now (the denormalized srs.due column). Soonest first. + const where: string[] = ['dueAt <= ?']; + const params: (string | number)[] = [now]; + if (opts && opts.courseId !== undefined) { + if (opts.courseId === null) { + where.push('courseId IS NULL'); + } else { + where.push('courseId = ?'); + params.push(opts.courseId); + } + } + let sql = `SELECT id, transcriptId, courseId, segmentId, start, front, back, + createdAt, srs, dueAt + FROM flashcards + WHERE ${where.join(' AND ')} + ORDER BY dueAt ASC`; + if (opts?.limit !== undefined) { + sql += ` LIMIT ?`; + params.push(opts.limit); + } + const rows = await db.getAllAsync(sql, params); + return rows.map(rowToFlashcard); + }, + + async updateFlashcardSrs(id: string, srs: SrsState): Promise { + const db = await getDb(); + const row = await db.getFirstAsync( + `SELECT id, transcriptId, courseId, segmentId, start, front, back, + createdAt, srs, dueAt + FROM flashcards WHERE id = ?`, + [id], + ); + if (!row) { + throw new Error(`flashcard not found: ${id}`); + } + // Persist the new schedule, keeping the denormalized dueAt column in sync. + await db.runAsync(`UPDATE flashcards SET srs = ?, dueAt = ? WHERE id = ?`, [ + JSON.stringify(srs), + srs.due, + id, + ]); + return rowToFlashcard({ ...row, srs: JSON.stringify(srs), dueAt: srs.due }); + }, + + async deleteFlashcard(id: string): Promise { + const db = await getDb(); + await db.runAsync(`DELETE FROM flashcards WHERE id = ?`, [id]); + }, + + async flashcardCounts( + courseId?: string | null, + ): Promise<{ total: number; due: number }> { + const db = await getDb(); + const now = Date.now(); + const scopeWhere: string[] = []; + const scopeParams: (string | null)[] = []; + if (courseId !== undefined) { + if (courseId === null) { + scopeWhere.push('courseId IS NULL'); + } else { + scopeWhere.push('courseId = ?'); + scopeParams.push(courseId); + } + } + const scopeClause = scopeWhere.length ? ` WHERE ${scopeWhere.join(' AND ')}` : ''; + const totalRow = await db.getFirstAsync<{ n: number }>( + `SELECT COUNT(*) AS n FROM flashcards${scopeClause}`, + scopeParams, + ); + const dueClause = scopeWhere.length + ? ` WHERE ${scopeWhere.join(' AND ')} AND dueAt <= ?` + : ` WHERE dueAt <= ?`; + const dueRow = await db.getFirstAsync<{ n: number }>( + `SELECT COUNT(*) AS n FROM flashcards${dueClause}`, + [...scopeParams, now], + ); + return { total: totalRow?.n ?? 0, due: dueRow?.n ?? 0 }; + }, }; diff --git a/src/lib/db/repo.test.ts b/src/lib/db/repo.test.ts index 96aa466..e408589 100644 --- a/src/lib/db/repo.test.ts +++ b/src/lib/db/repo.test.ts @@ -521,3 +521,193 @@ describe('vectors (semantic search store)', () => { expect(await repo.searchVectors(E0())).toHaveLength(0); }); }); + +// --------------------------------------------------------------------------- +// Phase 3: flashcards + SM-2 spaced repetition store +// --------------------------------------------------------------------------- +// +// These exercise the storage-only flashcard methods: a brand-new card is due +// immediately (initial srs.due ~ now); listDueFlashcards honours the due cutoff +// and course scope; updateFlashcardSrs persists a recomputed schedule; counts, +// deletes, and the remove(transcript) cascade all behave. + +describe('flashcards', () => { + beforeEach(async () => { + // Wipe transcripts (cascades drop their flashcards) + courses between tests. + const all = await repo.list(); + await Promise.all(all.map((m) => repo.remove(m.id))); + const courses = await repo.listCourses(); + await Promise.all(courses.map((c) => repo.deleteCourse(c.id))); + }); + + it('createFlashcards returns ids + an initial srs due ~ now', async () => { + const t = await repo.create(makeDraft()); + const before = Date.now(); + const [card] = await repo.createFlashcards([ + { + transcriptId: t.id, + front: 'What is Q?', + back: 'The answer.', + segmentId: t.segments[0]!.id, + start: 0, + }, + ]); + const after = Date.now(); + + expect(card!.id).toMatch(/^f_/); + expect(card!.transcriptId).toBe(t.id); + expect(card!.front).toBe('What is Q?'); + expect(card!.back).toBe('The answer.'); + expect(card!.segmentId).toBe(t.segments[0]!.id); + expect(card!.start).toBe(0); + expect(card!.createdAt).toBeTypeOf('number'); + // Initial SM-2 state. + expect(card!.srs.ease).toBe(2.5); + expect(card!.srs.intervalDays).toBe(0); + expect(card!.srs.reps).toBe(0); + expect(card!.srs.lapses).toBe(0); + // Due immediately: srs.due falls within the create() window. + expect(card!.srs.due).toBeGreaterThanOrEqual(before); + expect(card!.srs.due).toBeLessThanOrEqual(after); + + // It round-trips through listFlashcards. + const listed = await repo.listFlashcards({ transcriptId: t.id }); + expect(listed.map((c) => c.id)).toEqual([card!.id]); + }); + + it('listDueFlashcards includes a due card and excludes a future-due one', async () => { + const t = await repo.create(makeDraft()); + const [dueCard, futureCard] = await repo.createFlashcards([ + { transcriptId: t.id, front: 'due', back: 'now' }, + { transcriptId: t.id, front: 'later', back: 'future' }, + ]); + + // Push the second card's due far into the future via a recomputed schedule. + const future = Date.now() + 7 * 24 * 60 * 60 * 1000; + await repo.updateFlashcardSrs(futureCard!.id, { + ease: 2.5, + intervalDays: 7, + reps: 1, + lapses: 0, + due: future, + }); + + const due = await repo.listDueFlashcards(); + expect(due.map((c) => c.id)).toContain(dueCard!.id); + expect(due.map((c) => c.id)).not.toContain(futureCard!.id); + }); + + it('updateFlashcardSrs persists the recomputed schedule', async () => { + const t = await repo.create(makeDraft()); + const [card] = await repo.createFlashcards([ + { transcriptId: t.id, front: 'q', back: 'a' }, + ]); + + const reviewed = Date.now(); + const next = { + ease: 2.6, + intervalDays: 1, + reps: 1, + lapses: 0, + due: reviewed + 24 * 60 * 60 * 1000, + lastReviewed: reviewed, + }; + const updated = await repo.updateFlashcardSrs(card!.id, next); + expect(updated.srs).toEqual(next); + + // The change is durable. + const [reloaded] = await repo.listFlashcards({ transcriptId: t.id }); + expect(reloaded!.srs).toEqual(next); + }); + + it('updateFlashcardSrs throws for an unknown id', async () => { + await expect( + repo.updateFlashcardSrs('f_nope', { + ease: 2.5, + intervalDays: 0, + reps: 0, + lapses: 0, + due: Date.now(), + }), + ).rejects.toThrow(); + }); + + it('flashcardCounts reports total and due', async () => { + const t = await repo.create(makeDraft()); + const [, b, c] = await repo.createFlashcards([ + { transcriptId: t.id, front: '1', back: 'a' }, + { transcriptId: t.id, front: '2', back: 'b' }, + { transcriptId: t.id, front: '3', back: 'c' }, + ]); + + // All three start due. + expect(await repo.flashcardCounts()).toEqual({ total: 3, due: 3 }); + + // Push two out into the future => only one remains due. + const future = Date.now() + 60 * 60 * 1000; + const futureSrs = { + ease: 2.5, + intervalDays: 1, + reps: 1, + lapses: 0, + due: future, + }; + await repo.updateFlashcardSrs(b!.id, futureSrs); + await repo.updateFlashcardSrs(c!.id, futureSrs); + + expect(await repo.flashcardCounts()).toEqual({ total: 3, due: 1 }); + }); + + it('deleteFlashcard removes a single card', async () => { + const t = await repo.create(makeDraft()); + const [a, b] = await repo.createFlashcards([ + { transcriptId: t.id, front: '1', back: 'a' }, + { transcriptId: t.id, front: '2', back: 'b' }, + ]); + + await repo.deleteFlashcard(a!.id); + const remaining = await repo.listFlashcards({ transcriptId: t.id }); + expect(remaining.map((c) => c.id)).toEqual([b!.id]); + }); + + it('remove(transcript) cascades to its flashcards', async () => { + const t = await repo.create(makeDraft()); + await repo.createFlashcards([ + { transcriptId: t.id, front: '1', back: 'a' }, + { transcriptId: t.id, front: '2', back: 'b' }, + ]); + expect(await repo.listFlashcards({ transcriptId: t.id })).toHaveLength(2); + + await repo.remove(t.id); + expect(await repo.listFlashcards({ transcriptId: t.id })).toHaveLength(0); + expect(await repo.flashcardCounts()).toEqual({ total: 0, due: 0 }); + }); + + it('courseId scopes listing, due, and counts; deleteCourse moves cards to Unsorted', async () => { + const course = await repo.createCourse({ name: 'Physics' }); + const inCourse = await repo.create(makeDraft({ courseId: course.id })); + const unsorted = await repo.create(makeDraft()); + + await repo.createFlashcards([ + { transcriptId: inCourse.id, courseId: course.id, front: 'c1', back: 'a' }, + ]); + await repo.createFlashcards([ + { transcriptId: unsorted.id, front: 'u1', back: 'b' }, + ]); + + // Listing is scoped. + expect((await repo.listFlashcards({ courseId: course.id })).map((c) => c.front)).toEqual(['c1']); + expect((await repo.listFlashcards({ courseId: null })).map((c) => c.front)).toEqual(['u1']); + + // Counts + due are scoped. + expect(await repo.flashcardCounts(course.id)).toEqual({ total: 1, due: 1 }); + expect(await repo.flashcardCounts(null)).toEqual({ total: 1, due: 1 }); + expect((await repo.listDueFlashcards({ courseId: course.id })).map((c) => c.front)).toEqual(['c1']); + + // Deleting the course moves its cards to Unsorted (courseId=null). + await repo.deleteCourse(course.id); + const movedToUnsorted = await repo.listFlashcards({ courseId: null }); + expect(movedToUnsorted.map((c) => c.front).sort()).toEqual(['c1', 'u1']); + expect(await repo.listFlashcards({ courseId: course.id })).toHaveLength(0); + }); +}); diff --git a/src/lib/db/repo.ts b/src/lib/db/repo.ts index 8d26a94..7f3e4b7 100644 --- a/src/lib/db/repo.ts +++ b/src/lib/db/repo.ts @@ -12,6 +12,9 @@ import type { TranscriptPatch, Course, CourseDraft, + Flashcard, + FlashcardDraft, + SrsState, } from './schema'; /** Source of audio to persist for a transcript (web passes data, native a uri). */ @@ -127,4 +130,23 @@ export interface StorageRepo { /** Transcript ids that have NO vectors for `model` (need embedding/backfill). */ unembeddedIds(model: string): Promise; + + // --- flashcards + spaced repetition (Phase 3) --------------------------- + /** Validate + persist new flashcards (each gets an id, createdAt, initial SRS). */ + createFlashcards(drafts: FlashcardDraft[]): Promise; + + /** All flashcards, optionally scoped to a transcript or course. */ + listFlashcards(opts?: { transcriptId?: string; courseId?: string | null }): Promise; + + /** Cards due for review (srs.due <= now), optionally course-scoped. */ + listDueFlashcards(opts?: { courseId?: string | null; now?: number; limit?: number }): Promise; + + /** Persist a recomputed SRS schedule after a review. */ + updateFlashcardSrs(id: string, srs: SrsState): Promise; + + /** Delete a flashcard. */ + deleteFlashcard(id: string): Promise; + + /** Counts for badges (optionally course-scoped). */ + flashcardCounts(courseId?: string | null): Promise<{ total: number; due: number }>; } diff --git a/src/lib/db/repo.web.ts b/src/lib/db/repo.web.ts index 0254a96..cfe07b6 100644 --- a/src/lib/db/repo.web.ts +++ b/src/lib/db/repo.web.ts @@ -16,6 +16,7 @@ import { newId } from '../ids'; import { parseDraft, parseCourseDraft, + parseFlashcardDraft, zSegment, type TranscriptDraft, type TranscriptMeta, @@ -23,6 +24,9 @@ import { type TranscriptPatch, type Course, type CourseDraft, + type Flashcard, + type FlashcardDraft, + type SrsState, } from './schema'; import type { StorageRepo, @@ -115,6 +119,9 @@ class WispDexie extends Dexie { // Keyed by the compound [transcriptId+segmentId]; secondary indexes on // transcriptId (cascade), courseId (scoped search) and model (backfill). segvecs!: Dexie.Table; + // Phase 3 flashcards, keyed by id; secondary indexes on transcriptId + // (cascade on remove) and courseId (scoping + cascade on deleteCourse). + flashcards!: Dexie.Table; constructor() { super('wisp'); @@ -151,6 +158,16 @@ class WispDexie extends Dexie { media: 'transcriptId', segvecs: '[transcriptId+segmentId], transcriptId, courseId, model', }); + // v4: add the flashcards store for Phase 3 spaced repetition. Existing + // stores are re-declared unchanged so Dexie keeps them; only the new store + // is added. No backfill — flashcards are produced lazily by the study UI. + this.version(4).stores({ + transcripts: 'id, createdAt, courseId', + courses: 'id, name', + media: 'transcriptId', + segvecs: '[transcriptId+segmentId], transcriptId, courseId, model', + flashcards: 'id, transcriptId, courseId', + }); } } @@ -249,12 +266,20 @@ export const repo: StorageRepo = { }, async remove(id: string): Promise { - // Also delete any persisted media + vectors for this transcript. - await db.transaction('rw', db.transcripts, db.media, db.segvecs, async () => { - await db.transcripts.delete(id); - await db.media.delete(id); - await db.segvecs.where('transcriptId').equals(id).delete(); - }); + // Also delete any persisted media + vectors + flashcards for this transcript. + await db.transaction( + 'rw', + db.transcripts, + db.media, + db.segvecs, + db.flashcards, + async () => { + await db.transcripts.delete(id); + await db.media.delete(id); + await db.segvecs.where('transcriptId').equals(id).delete(); + await db.flashcards.where('transcriptId').equals(id).delete(); + }, + ); }, async search(query: string): Promise { @@ -344,20 +369,29 @@ export const repo: StorageRepo = { async deleteCourse(id: string): Promise { // First reassign this course's transcripts to "Unsorted" (courseId=null), // THEN delete the course row, so no transcript is ever orphaned. - await db.transaction('rw', db.transcripts, db.courses, db.segvecs, async () => { - const now = Date.now(); - const owned = await db.transcripts - .filter((r) => r.courseId === id) - .toArray(); - await Promise.all( - owned.map((r) => - db.transcripts.put({ ...r, courseId: null, updatedAt: now }), - ), - ); - // Keep denormalized vector rows consistent with their now-Unsorted owners. - await db.segvecs.where('courseId').equals(id).modify({ courseId: null }); - await db.courses.delete(id); - }); + await db.transaction( + 'rw', + db.transcripts, + db.courses, + db.segvecs, + db.flashcards, + async () => { + const now = Date.now(); + const owned = await db.transcripts + .filter((r) => r.courseId === id) + .toArray(); + await Promise.all( + owned.map((r) => + db.transcripts.put({ ...r, courseId: null, updatedAt: now }), + ), + ); + // Keep denormalized vector rows consistent with their now-Unsorted owners. + await db.segvecs.where('courseId').equals(id).modify({ courseId: null }); + // Flashcards in this course follow their transcripts to Unsorted. + await db.flashcards.where('courseId').equals(id).modify({ courseId: null }); + await db.courses.delete(id); + }, + ); }, // --- media (persisted source audio) ------------------------------------- @@ -471,4 +505,105 @@ export const repo: StorageRepo = { const ids = await db.transcripts.toCollection().primaryKeys(); return ids.filter((id) => !embedded.has(id)); }, + + // --- flashcards + spaced repetition (Phase 3) --------------------------- + async createFlashcards(drafts: FlashcardDraft[]): Promise { + const now = Date.now(); + const cards: Flashcard[] = drafts.map((draft) => { + const valid = parseFlashcardDraft(draft); + // Initial SM-2 state for a brand-new card: due immediately (now). We + // compute the SRS inline here rather than importing learn/srs to keep the + // repo storage-only (no dependency on the study logic). + const srs: SrsState = { + ease: 2.5, + intervalDays: 0, + reps: 0, + lapses: 0, + due: now, + }; + return { + id: newId('f_'), + transcriptId: valid.transcriptId, + // courseId defaults to null ("Unsorted") when absent. + courseId: valid.courseId ?? null, + ...(valid.segmentId !== undefined ? { segmentId: valid.segmentId } : {}), + ...(valid.start !== undefined ? { start: valid.start } : {}), + front: valid.front, + back: valid.back, + createdAt: now, + srs, + }; + }); + await db.flashcards.bulkPut(cards); + return cards; + }, + + async listFlashcards(opts?: { + transcriptId?: string; + courseId?: string | null; + }): Promise { + let rows = await db.flashcards.toArray(); + if (opts?.transcriptId !== undefined) { + rows = rows.filter((c) => c.transcriptId === opts.transcriptId); + } + if (opts && opts.courseId !== undefined) { + const scope = opts.courseId; + rows = rows.filter((c) => + // null scope => Unsorted (courseId null or undefined). + scope === null ? c.courseId == null : c.courseId === scope, + ); + } + return rows; + }, + + async listDueFlashcards(opts?: { + courseId?: string | null; + now?: number; + limit?: number; + }): Promise { + const now = opts?.now ?? Date.now(); + let rows = await db.flashcards.toArray(); + if (opts && opts.courseId !== undefined) { + const scope = opts.courseId; + rows = rows.filter((c) => + scope === null ? c.courseId == null : c.courseId === scope, + ); + } + // Due means the next-review time is now or in the past. + rows = rows.filter((c) => c.srs.due <= now); + // Soonest-due first. + rows.sort((a, b) => a.srs.due - b.srs.due); + if (opts?.limit !== undefined) { + rows = rows.slice(0, opts.limit); + } + return rows; + }, + + async updateFlashcardSrs(id: string, srs: SrsState): Promise { + const card = await db.flashcards.get(id); + if (!card) { + throw new Error(`flashcard not found: ${id}`); + } + const updated: Flashcard = { ...card, srs }; + await db.flashcards.put(updated); + return updated; + }, + + async deleteFlashcard(id: string): Promise { + await db.flashcards.delete(id); + }, + + async flashcardCounts( + courseId?: string | null, + ): Promise<{ total: number; due: number }> { + const now = Date.now(); + let rows = await db.flashcards.toArray(); + if (courseId !== undefined) { + rows = rows.filter((c) => + courseId === null ? c.courseId == null : c.courseId === courseId, + ); + } + const due = rows.filter((c) => c.srs.due <= now).length; + return { total: rows.length, due }; + }, }; diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index a25eb02..4243b7c 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -136,6 +136,48 @@ export function parseCourseDraft(input: unknown): CourseDraft { return zCourseDraft.parse(input); } +// --------------------------------------------------------------------------- +// Flashcards + spaced repetition (Phase 3) +// --------------------------------------------------------------------------- + +/** SM-2 spaced-repetition state for a card. */ +export interface SrsState { + /** SM-2 ease factor (starts 2.5, floored at 1.3). */ + ease: number; + /** Current interval in days. */ + intervalDays: number; + /** Successful repetitions in a row. */ + reps: number; + /** Times the card was forgotten. */ + lapses: number; + /** Next-due time, ms since epoch. */ + due: number; + /** Last review time, ms since epoch. */ + lastReviewed?: number; +} + +export const zFlashcardDraft = z.object({ + transcriptId: z.string(), + courseId: z.string().nullable().optional(), + /** Source segment for click-to-seek back to the lecture. */ + segmentId: z.string().optional(), + start: zFiniteNonNeg.optional(), + front: z.string().min(1), + back: z.string().min(1), +}); +export type FlashcardDraft = z.infer; + +/** A stored flashcard with its SRS schedule. */ +export interface Flashcard extends FlashcardDraft { + id: string; + createdAt: number; + srs: SrsState; +} + +export function parseFlashcardDraft(input: unknown): FlashcardDraft { + return zFlashcardDraft.parse(input); +} + // --------------------------------------------------------------------------- // Stored types (output of the repo) // --------------------------------------------------------------------------- diff --git a/src/lib/learn/flashcards.test.ts b/src/lib/learn/flashcards.test.ts new file mode 100644 index 0000000..20b4150 --- /dev/null +++ b/src/lib/learn/flashcards.test.ts @@ -0,0 +1,46 @@ +import { describe, it, expect } from 'vitest'; +import { cardsFromGlossary } from './flashcards'; +import type { GlossaryEntry } from './glossary'; + +describe('cardsFromGlossary', () => { + it('maps one card per entry and carries provenance', () => { + const entries: GlossaryEntry[] = [ + { + term: 'Photosynthesis', + definition: 'A process used by plants to make food.', + start: 7, + segmentId: 'segX', + }, + ]; + const cards = cardsFromGlossary(entries); + expect(cards.length).toBe(1); + expect(cards[0]!.back).toBe('A process used by plants to make food.'); + expect(cards[0]!.start).toBe(7); + expect(cards[0]!.segmentId).toBe('segX'); + }); + + it('uses a "What is X?" front when the term is absent from the definition', () => { + const entries: GlossaryEntry[] = [ + { term: 'Photosynthesis', definition: 'A process used by plants.', start: 0 }, + ]; + const [card] = cardsFromGlossary(entries); + expect(card!.front).toBe('What is Photosynthesis?'); + }); + + it('makes a cloze front when the term appears in the definition', () => { + const entries: GlossaryEntry[] = [ + { + term: 'Recursion', + definition: 'Recursion is when a function calls itself.', + start: 0, + }, + ]; + const [card] = cardsFromGlossary(entries); + expect(card!.front).toBe('_____ is when a function calls itself.'); + expect(card!.front).not.toContain('Recursion'); + }); + + it('returns [] for no entries', () => { + expect(cardsFromGlossary([])).toEqual([]); + }); +}); diff --git a/src/lib/learn/flashcards.ts b/src/lib/learn/flashcards.ts new file mode 100644 index 0000000..f0adc9d --- /dev/null +++ b/src/lib/learn/flashcards.ts @@ -0,0 +1,51 @@ +// Turn glossary entries into flashcard seeds. Deterministic, no model. +// A seed is a UI-friendly front/back pair carrying its source provenance; the +// repo later wraps it into a FlashcardDraft + SRS state. +// +// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias). + +import type { GlossaryEntry } from './glossary'; + +/** A front/back pair for a flashcard, with optional source anchor. */ +export interface CardSeed { + front: string; + back: string; + /** Source segment id, for click-to-seek. */ + segmentId?: string; + /** Source start time (seconds). */ + start?: number; +} + +/** + * Build one {@link CardSeed} per glossary entry. + * + * Front: if the term appears verbatim (case-insensitively) inside the + * definition, we make a cloze deletion — blanking the term in the definition so + * the learner recalls it in context. Otherwise we fall back to the plain + * "What is {term}?" prompt. Back is always the full definition. + */ +export function cardsFromGlossary(entries: GlossaryEntry[]): CardSeed[] { + return entries.map((entry) => { + const front = makeFront(entry.term, entry.definition); + return { + front, + back: entry.definition, + segmentId: entry.segmentId, + start: entry.start, + }; + }); +} + +/** "_____" cloze if the term occurs in the definition, else a Q prompt. */ +function makeFront(term: string, definition: string): string { + const re = new RegExp(escapeRegExp(term), 'i'); + if (re.test(definition)) { + return definition.replace(re, '_____'); + } + return `What is ${term}?`; +} + +/** Escape a string for safe use inside a RegExp. */ +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/src/lib/learn/glossary.test.ts b/src/lib/learn/glossary.test.ts new file mode 100644 index 0000000..c8ad0f1 --- /dev/null +++ b/src/lib/learn/glossary.test.ts @@ -0,0 +1,64 @@ +import { describe, it, expect } from 'vitest'; +import { glossary } from './glossary'; +import type { Segment } from '../types'; + +const seg = (start: number, text: string, id?: string): Segment => ({ + id, + start, + end: start + 1, + text, +}); + +describe('glossary', () => { + it('returns [] for no segments', () => { + expect(glossary([])).toEqual([]); + }); + + it('captures an "X is Y" definition pattern', () => { + const segs = [ + seg(3, 'Backpropagation is an algorithm for training neural networks.', 's1'), + ]; + const out = glossary(segs); + const entry = out.find((e) => e.term.toLowerCase() === 'backpropagation'); + expect(entry).toBeDefined(); + expect(entry!.definition).toBe( + 'Backpropagation is an algorithm for training neural networks.', + ); + expect(entry!.start).toBe(3); + expect(entry!.segmentId).toBe('s1'); + }); + + it('recognizes means / refers to / is defined as', () => { + const segs = [ + seg(0, 'Entropy means the amount of disorder in a system.'), + seg(1, 'A token refers to a unit of text.'), + seg(2, 'Latency is defined as the delay before a transfer begins.'), + ]; + const terms = glossary(segs).map((e) => e.term.toLowerCase()); + expect(terms).toContain('entropy'); + expect(terms).toContain('a token'); + expect(terms).toContain('latency'); + }); + + it('dedupes by lowercased term, preferring the explicit definition', () => { + const segs = [ + seg(0, 'Recursion is fun. Recursion appears again here as Recursion.'), + seg(1, 'Recursion is when a function calls itself.', 'def'), + ]; + const out = glossary(segs); + const recursion = out.filter((e) => e.term.toLowerCase() === 'recursion'); + expect(recursion.length).toBe(1); + expect(recursion[0]!.definition).toBe( + 'Recursion is when a function calls itself.', + ); + }); + + it('respects the max cap', () => { + const segs = [ + seg(0, 'Apple is a fruit.'), + seg(1, 'Banana is a fruit.'), + seg(2, 'Cherry is a fruit.'), + ]; + expect(glossary(segs, { max: 2 }).length).toBeLessThanOrEqual(2); + }); +}); diff --git a/src/lib/learn/glossary.ts b/src/lib/learn/glossary.ts new file mode 100644 index 0000000..66f9b0a --- /dev/null +++ b/src/lib/learn/glossary.ts @@ -0,0 +1,183 @@ +// Deterministic glossary extraction: surface candidate terms + definitions from +// a lecture transcript with NO model. Two complementary strategies: +// (a) explicit definition patterns: "X is/are/means/refers to/is defined as Y" +// (b) frequent Capitalized phrases / frequent content nouns as a fallback, +// using their first containing sentence as a stand-in definition. +// +// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias). + +import type { Segment } from '../types'; +import { splitSentences, tokenizeWords, STOPWORDS } from './tokenize'; + +/** One glossary row: a term, a definition, and where it came from. */ +export interface GlossaryEntry { + term: string; + definition: string; + /** Start time (seconds) of the source segment. */ + start: number; + /** Source segment id, when the segment had one. */ + segmentId?: string; +} + +/** Internal accumulator while collecting candidates. */ +interface Candidate extends GlossaryEntry { + /** How "important"/frequent this candidate is, for ranking. */ + frequency: number; + /** Whether it came from an explicit definition pattern (preferred). */ + explicit: boolean; + /** Original discovery order, for stable tie-breaking. */ + order: number; +} + +// "X is/are/means/refers to/is defined as Y" — capture the subject (term) and +// keep the whole sentence as the definition. The subject is the run of words +// before the connective; we cap its length so we don't swallow half a clause. +const DEFINITION_RE = + /^(.{2,60}?)\s+(?:is|are|means|refers? to|is defined as)\s+(.+)$/i; + +// A run of Capitalized words (proper-noun-ish phrase), e.g. "Hidden Markov Model". +const CAPITALIZED_PHRASE_RE = /\b([A-Z][a-z0-9]+(?:\s+[A-Z][a-z0-9]+)*)\b/g; + +/** + * Build a glossary of at most `max` entries from `segments`. + * + * Candidates are deduped by lowercased term (explicit definitions win over + * fallbacks; otherwise the first/most-frequent occurrence is kept), then sorted + * by frequency descending (explicit entries ranked ahead on ties), and capped. + * + * Empty input -> []. + */ +export function glossary( + segments: Segment[], + opts?: { max?: number }, +): GlossaryEntry[] { + const max = opts?.max ?? 12; + if (segments.length === 0 || max <= 0) return []; + + // Document-wide content-word frequencies, used to rank fallback candidates. + const wordFreq = new Map(); + for (const seg of segments) { + for (const tok of tokenizeWords(seg.text)) { + wordFreq.set(tok, (wordFreq.get(tok) ?? 0) + 1); + } + } + + // Frequency of each Capitalized phrase (by lowercased key) across the doc. + const phraseFreq = new Map(); + for (const seg of segments) { + for (const m of seg.text.matchAll(CAPITALIZED_PHRASE_RE)) { + const phrase = m[1]; + if (!phrase) continue; + const key = phrase.toLowerCase(); + phraseFreq.set(key, (phraseFreq.get(key) ?? 0) + 1); + } + } + + // Dedupe by lowercased term. Map preserves first-seen insertion order. + const byTerm = new Map(); + let order = 0; + + const add = (cand: Omit) => { + const key = cand.term.toLowerCase().trim(); + if (key.length === 0) return; + const existing = byTerm.get(key); + if (!existing) { + byTerm.set(key, { ...cand, order: order++ }); + return; + } + // An explicit definition supersedes a fallback for the same term. When both + // are explicit, keep the more informative (longer) definition; this lets a + // substantive "X is when ..." win over a throwaway "X is fun." mention. + const upgrade = + (cand.explicit && !existing.explicit) || + (cand.explicit === existing.explicit && + cand.definition.length > existing.definition.length); + if (upgrade) { + byTerm.set(key, { ...cand, order: existing.order }); + } + }; + + // Pass 1: explicit "X is Y" definitions (highest quality). + for (const seg of segments) { + for (const sentence of splitSentences(seg.text)) { + const m = sentence.match(DEFINITION_RE); + if (!m || !m[1]) continue; + const term = m[1].trim(); + // Skip junk subjects (pure stopword / too short). + const termWords = tokenizeWords(term); + if (termWords.length === 0 && term.length < 3) continue; + add({ + term, + definition: sentence, + start: seg.start, + segmentId: seg.id, + // Frequency from the phrase/word tables, boosted so explicit wins. + frequency: + (phraseFreq.get(term.toLowerCase()) ?? 0) + + (wordFreq.get(term.toLowerCase()) ?? 0) + + 1, + explicit: true, + }); + } + } + + // Pass 2a: frequent multi-word Capitalized phrases as fallback candidates. + for (const seg of segments) { + const sentences = splitSentences(seg.text); + for (const [key, freq] of phraseFreq) { + if (freq < 1) continue; + // Only multi-word phrases here; single Capitalized words are noisy. + if (!key.includes(' ')) continue; + // First sentence in this segment that contains the phrase. + const containing = sentences.find((s) => s.toLowerCase().includes(key)); + if (!containing) continue; + // Reconstruct a Title-Cased display term from the key. + const term = key.replace(/\b\w/g, (c) => c.toUpperCase()); + add({ + term, + definition: containing, + start: seg.start, + segmentId: seg.id, + frequency: freq, + explicit: false, + }); + } + } + + // Pass 2b: frequent single content nouns as a last-resort fallback. + for (const seg of segments) { + const sentences = splitSentences(seg.text); + for (const tok of tokenizeWords(seg.text)) { + if (STOPWORDS.has(tok)) continue; + const freq = wordFreq.get(tok) ?? 0; + if (freq < 2) continue; // require some recurrence to qualify + const containing = sentences.find((s) => + s.toLowerCase().includes(tok), + ); + if (!containing) continue; + add({ + term: tok, + definition: containing, + start: seg.start, + segmentId: seg.id, + frequency: freq, + explicit: false, + }); + } + } + + // Rank: explicit first, then frequency desc, then discovery order (stable). + const ranked = [...byTerm.values()].sort( + (a, b) => + Number(b.explicit) - Number(a.explicit) || + b.frequency - a.frequency || + a.order - b.order, + ); + + return ranked.slice(0, max).map(({ term, definition, start, segmentId }) => ({ + term, + definition, + start, + segmentId, + })); +} diff --git a/src/lib/learn/index.ts b/src/lib/learn/index.ts new file mode 100644 index 0000000..5d8d7c3 --- /dev/null +++ b/src/lib/learn/index.ts @@ -0,0 +1,15 @@ +// Public surface of the deterministic study helpers (Phase 3). All pure: no +// React/Expo/Node-builtins, so the UI and tests can import freely. +// +// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias). + +export { splitSentences, tokenizeWords, STOPWORDS } from './tokenize'; +export { summarize } from './summary'; +export type { SourcedSentence } from './summary'; +export { glossary } from './glossary'; +export type { GlossaryEntry } from './glossary'; +export { cardsFromGlossary } from './flashcards'; +export type { CardSeed } from './flashcards'; +export { initialSrs, review } from './srs'; +export { generateQuiz } from './quiz'; +export type { QuizQuestion } from './quiz'; diff --git a/src/lib/learn/quiz.test.ts b/src/lib/learn/quiz.test.ts new file mode 100644 index 0000000..fa155de --- /dev/null +++ b/src/lib/learn/quiz.test.ts @@ -0,0 +1,66 @@ +import { describe, it, expect } from 'vitest'; +import { generateQuiz } from './quiz'; +import type { GlossaryEntry } from './glossary'; + +const entry = (term: string, definition: string, start = 0): GlossaryEntry => ({ + term, + definition, + start, + segmentId: `seg-${term}`, +}); + +const FOUR: GlossaryEntry[] = [ + entry('Alpha', 'the first letter'), + entry('Beta', 'the second letter'), + entry('Gamma', 'the third letter'), + entry('Delta', 'the fourth letter'), +]; + +describe('generateQuiz', () => { + it('returns [] when there are fewer than 4 terms', () => { + expect(generateQuiz(FOUR.slice(0, 3))).toEqual([]); + expect(generateQuiz([])).toEqual([]); + }); + + it('returns [] when count <= 0', () => { + expect(generateQuiz(FOUR, { count: 0 })).toEqual([]); + }); + + it('each question has 4 options with the answer at answerIndex', () => { + const quiz = generateQuiz(FOUR); + expect(quiz.length).toBe(4); + for (let i = 0; i < quiz.length; i++) { + const q = quiz[i]!; + const src = FOUR[i]!; + expect(q.options.length).toBe(4); + // The option at answerIndex is the correct term for that question. + expect(q.options[q.answerIndex]).toBe(src.term); + // Question wraps the correct definition. + expect(q.question).toBe(`Which term means: "${src.definition}"?`); + } + }); + + it('uses deterministic answer placement at index i % 4', () => { + const quiz = generateQuiz(FOUR); + expect(quiz.map((q) => q.answerIndex)).toEqual([0, 1, 2, 3]); + }); + + it('options are distinct and include 3 distractors from other entries', () => { + const quiz = generateQuiz(FOUR); + for (const q of quiz) { + const unique = new Set(q.options.map((o) => o.toLowerCase())); + expect(unique.size).toBe(4); + } + }); + + it('carries start and segmentId from the correct entry', () => { + const quiz = generateQuiz(FOUR); + expect(quiz[0]!.segmentId).toBe('seg-Alpha'); + expect(quiz[0]!.start).toBe(0); + }); + + it('respects the count cap', () => { + const quiz = generateQuiz(FOUR, { count: 2 }); + expect(quiz.length).toBe(2); + }); +}); diff --git a/src/lib/learn/quiz.ts b/src/lib/learn/quiz.ts new file mode 100644 index 0000000..c59be93 --- /dev/null +++ b/src/lib/learn/quiz.ts @@ -0,0 +1,88 @@ +// Deterministic multiple-choice quiz generation from glossary entries. No model, +// NO randomness — given the same entries, the same quiz is produced every time +// (important for reproducible tests and stable UI across re-renders). +// +// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias). + +import type { GlossaryEntry } from './glossary'; + +/** One MCQ: a definition prompt, term options, and the correct option index. */ +export interface QuizQuestion { + question: string; + options: string[]; + answerIndex: number; + /** Source start time (seconds). */ + start?: number; + /** Source segment id. */ + segmentId?: string; +} + +const OPTION_COUNT = 4; + +/** + * Generate up to `count` questions from `entries`. + * + * Requires at least {@link OPTION_COUNT} (4) distinct terms so every question + * can have one correct answer + 3 distinct distractors; with fewer entries we + * return []. + * + * Determinism: for question `i`, the correct answer is placed at index + * `i % OPTION_COUNT`, and the remaining slots are filled with distractor terms + * drawn from the OTHER entries by rotating through the list (offset by `i`), so + * the layout is a pure function of the input order. + */ +export function generateQuiz( + entries: GlossaryEntry[], + opts?: { count?: number }, +): QuizQuestion[] { + const count = opts?.count ?? 8; + if (entries.length < OPTION_COUNT || count <= 0) return []; + + const questions: QuizQuestion[] = []; + const n = entries.length; + const limit = Math.min(count, n); + + for (let i = 0; i < limit; i++) { + const correct = entries[i]; + if (!correct) continue; // unreachable (i < limit <= n), satisfies the checker + + // Collect 3 distinct distractor terms from other entries, rotating the + // start point by `i` so different questions get different wrong answers. + const distractors: string[] = []; + const seen = new Set([correct.term.toLowerCase()]); + let j = 1; + while (distractors.length < OPTION_COUNT - 1 && j <= n) { + const cand = entries[(i + j) % n]; + j++; + if (!cand) continue; + const key = cand.term.toLowerCase(); + if (!seen.has(key)) { + seen.add(key); + distractors.push(cand.term); + } + } + + // Defensive: if duplicate terms left us short, skip this question rather + // than emit one with fewer than 4 options. (glossary() dedupes, so this is + // effectively unreachable for real input.) + if (distractors.length < OPTION_COUNT - 1) continue; + + // Place the answer at a deterministic index; fill the rest with distractors. + const answerIndex = i % OPTION_COUNT; + const options: string[] = []; + let d = 0; + for (let slot = 0; slot < OPTION_COUNT; slot++) { + options.push(slot === answerIndex ? correct.term : (distractors[d++] as string)); + } + + questions.push({ + question: `Which term means: "${correct.definition}"?`, + options, + answerIndex, + start: correct.start, + segmentId: correct.segmentId, + }); + } + + return questions; +} diff --git a/src/lib/learn/srs.test.ts b/src/lib/learn/srs.test.ts new file mode 100644 index 0000000..3e2b488 --- /dev/null +++ b/src/lib/learn/srs.test.ts @@ -0,0 +1,82 @@ +import { describe, it, expect } from 'vitest'; +import { initialSrs, review } from './srs'; + +const MS_PER_DAY = 86_400_000; +const NOW = 1_700_000_000_000; + +describe('initialSrs', () => { + it('creates a neutral, immediately-due card', () => { + const s = initialSrs(NOW); + expect(s).toEqual({ + ease: 2.5, + intervalDays: 0, + reps: 0, + lapses: 0, + due: NOW, + }); + }); +}); + +describe('review', () => { + it('does not mutate the input state', () => { + const s = initialSrs(NOW); + const frozen = { ...s }; + review(s, 2, NOW); + expect(s).toEqual(frozen); + }); + + it('produces monotonically increasing intervals on repeated "good"', () => { + let s = initialSrs(NOW); + const intervals: number[] = []; + let t = NOW; + for (let i = 0; i < 5; i++) { + s = review(s, 2, t); + intervals.push(s.intervalDays); + t += s.intervalDays * MS_PER_DAY; + } + // 1 day, 6 days, then *ease each time -> strictly increasing. + for (let i = 1; i < intervals.length; i++) { + expect(intervals[i]!).toBeGreaterThan(intervals[i - 1]!); + } + expect(intervals[0]).toBe(1); + expect(intervals[1]).toBe(6); + }); + + it('increments reps on passing grades and resets on "again"', () => { + let s = initialSrs(NOW); + s = review(s, 2, NOW); // good + s = review(s, 2, NOW); // good + expect(s.reps).toBe(2); + s = review(s, 0, NOW); // again + expect(s.reps).toBe(0); + expect(s.lapses).toBe(1); + }); + + it('floors ease at 1.3 even after many "again" reviews', () => { + let s = initialSrs(NOW); + for (let i = 0; i < 20; i++) s = review(s, 0, NOW); + expect(s.ease).toBe(1.3); + }); + + it('lowers ease on hard, raises it on easy', () => { + const good = review(initialSrs(NOW), 2, NOW); + expect(good.ease).toBe(2.5); + const hard = review(initialSrs(NOW), 1, NOW); + expect(hard.ease).toBeCloseTo(2.35, 5); + const easy = review(initialSrs(NOW), 3, NOW); + expect(easy.ease).toBeCloseTo(2.65, 5); + }); + + it('computes due = now + intervalDays * 86400000 and sets lastReviewed', () => { + const s = review(initialSrs(NOW), 2, NOW); + expect(s.intervalDays).toBe(1); + expect(s.due).toBe(NOW + 1 * MS_PER_DAY); + expect(s.lastReviewed).toBe(NOW); + }); + + it('gives "easy" a longer interval than "good" at the same step', () => { + const good = review(initialSrs(NOW), 2, NOW); + const easy = review(initialSrs(NOW), 3, NOW); + expect(easy.intervalDays).toBeGreaterThan(good.intervalDays); + }); +}); diff --git a/src/lib/learn/srs.ts b/src/lib/learn/srs.ts new file mode 100644 index 0000000..d52d394 --- /dev/null +++ b/src/lib/learn/srs.ts @@ -0,0 +1,110 @@ +// Spaced repetition scheduling — a faithful SM-2 variant adapted to a 4-button +// grading UI (Again / Hard / Good / Easy). Pure & deterministic. +// +// The classic SM-2 algorithm grades recall on 0..5 and recomputes an ease +// factor (EF) plus an interval. We collapse the UI to four buttons and map them +// to behaviors that match learner expectations on Anki-style apps: +// +// grade 0 = Again : failed recall. Reset the rep streak, count a lapse, drop +// ease by 0.2, and re-show soon (a short sub-day interval). +// grade 1 = Hard : recalled with difficulty. Grow the interval gently (x1.2) +// and drop ease by 0.15. +// grade 2 = Good : normal recall. Use the SM-2 schedule: +// rep 1 -> 1 day, rep 2 -> 6 days, then interval *= ease. +// grade 3 = Easy : effortless recall. Like Good but with an easy bonus +// (~x1.3) and ease bumped up by 0.15. +// +// Invariants: +// - ease is floored at 1.3 (SM-2's classic minimum). +// - reps increments only on a passing grade (>= 1); Again resets it to 0. +// - due = now + intervalDays * MS_PER_DAY. +// - lastReviewed is set to `now` on every review. +// +// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias). + +import type { SrsState } from '../db/schema'; + +const MS_PER_DAY = 86_400_000; +const MIN_EASE = 1.3; +/** Sub-day interval (in days) used when a card is failed and must re-show soon. */ +const AGAIN_INTERVAL_DAYS = 1 / 1440; // ~1 minute, expressed in days +const HARD_MULTIPLIER = 1.2; +const EASY_BONUS = 1.3; + +/** A brand-new card: due immediately, neutral ease, nothing learned yet. */ +export function initialSrs(now: number): SrsState { + return { + ease: 2.5, + intervalDays: 0, + reps: 0, + lapses: 0, + due: now, + }; +} + +/** + * Apply a review `grade` to `srs` at time `now`, returning the next state. + * Does not mutate the input. + */ +export function review( + srs: SrsState, + grade: 0 | 1 | 2 | 3, + now: number, +): SrsState { + let { ease, intervalDays, reps, lapses } = srs; + + if (grade === 0) { + // Again: the card was forgotten. + ease = floorEase(ease - 0.2); + reps = 0; + lapses += 1; + intervalDays = AGAIN_INTERVAL_DAYS; + } else if (grade === 1) { + // Hard: passed, but gently grow the interval and lower ease. + ease = floorEase(ease - 0.15); + reps += 1; + intervalDays = nextInterval(srs, ease, HARD_MULTIPLIER); + } else if (grade === 2) { + // Good: standard SM-2 progression, ease unchanged. + reps += 1; + intervalDays = nextInterval(srs, ease, 1); + } else { + // Easy: like Good with an easy bonus and an ease bump. + ease = floorEase(ease + 0.15); + reps += 1; + intervalDays = nextInterval(srs, ease, EASY_BONUS); + } + + return { + ease, + intervalDays, + reps, + lapses, + due: now + intervalDays * MS_PER_DAY, + lastReviewed: now, + }; +} + +/** + * SM-2 interval progression for a passing grade. + * - first successful rep (the card had reps 0) -> 1 day + * - second successful rep (had reps 1) -> 6 days + * - thereafter -> previous interval * ease + * The result is scaled by `multiplier` (Hard < 1-ish via lower ease, Easy bonus). + */ +function nextInterval(prev: SrsState, ease: number, multiplier: number): number { + let base: number; + if (prev.reps <= 0) { + base = 1; + } else if (prev.reps === 1) { + base = 6; + } else { + base = prev.intervalDays * ease; + } + return base * multiplier; +} + +/** Clamp an ease factor to SM-2's minimum of 1.3. */ +function floorEase(ease: number): number { + return ease < MIN_EASE ? MIN_EASE : ease; +} diff --git a/src/lib/learn/summary.test.ts b/src/lib/learn/summary.test.ts new file mode 100644 index 0000000..676bfb7 --- /dev/null +++ b/src/lib/learn/summary.test.ts @@ -0,0 +1,57 @@ +import { describe, it, expect } from 'vitest'; +import { summarize } from './summary'; +import type { Segment } from '../types'; + +const seg = (start: number, text: string, id?: string): Segment => ({ + id, + start, + end: start + 1, + text, +}); + +describe('summarize', () => { + it('returns [] for no segments', () => { + expect(summarize([])).toEqual([]); + }); + + it('returns [] when maxSentences <= 0', () => { + expect(summarize([seg(0, 'Neural networks learn.')], { maxSentences: 0 })).toEqual( + [], + ); + }); + + it('picks the top-N highest-frequency sentences', () => { + // "neural" + "networks" dominate; the off-topic sentence should be dropped. + const segs = [ + seg(0, 'Neural networks process data.'), + seg(1, 'Neural networks learn from neural networks.'), + seg(2, 'The weather today is sunny.'), + ]; + const out = summarize(segs, { maxSentences: 2 }); + expect(out.length).toBe(2); + const texts = out.map((s) => s.text); + expect(texts).not.toContain('The weather today is sunny.'); + }); + + it('returns sentences in chronological order with the right start + id', () => { + const segs = [ + seg(10, 'Gradient descent optimizes the gradient.', 'segB'), + seg(2, 'Gradient descent uses the gradient slope.', 'segA'), + ]; + const out = summarize(segs, { maxSentences: 2 }); + // Both kept; chronological order means start 2 comes before start 10. + expect(out.map((s) => s.start)).toEqual([2, 10]); + expect(out[0]!.segmentId).toBe('segA'); + expect(out[1]!.segmentId).toBe('segB'); + }); + + it('flattens multi-sentence segments and tags them with the segment start', () => { + const segs = [seg(5, 'Alpha beta gamma. Alpha beta delta.', 'multi')]; + const out = summarize(segs, { maxSentences: 5 }); + expect(out.length).toBe(2); + for (const s of out) { + expect(s.start).toBe(5); + expect(s.segmentId).toBe('multi'); + } + }); +}); diff --git a/src/lib/learn/summary.ts b/src/lib/learn/summary.ts new file mode 100644 index 0000000..83e697d --- /dev/null +++ b/src/lib/learn/summary.ts @@ -0,0 +1,90 @@ +// Deterministic extractive summary: pick the most "central" sentences of a +// lecture by term-frequency scoring, keeping them in chronological order so the +// summary still reads like a condensed version of the talk. No model. +// +// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias). + +import type { Segment } from '../types'; +import { splitSentences, tokenizeWords } from './tokenize'; + +/** A summary sentence tagged with where it came from (for click-to-seek). */ +export interface SourcedSentence { + text: string; + /** Start time (seconds) of the source segment. */ + start: number; + /** Source segment id, when the segment had one. */ + segmentId?: string; +} + +/** Internal: a sentence carrying its source + original ordering keys. */ +interface ScoredSentence extends SourcedSentence { + /** Original position across the flattened doc, for stable chronological sort. */ + position: number; + /** Term-frequency score, normalized by sentence length. */ + score: number; +} + +/** + * Summarize `segments` into at most `maxSentences` sentences. + * + * Algorithm (fully deterministic): + * 1. Build a document term-frequency table over `tokenizeWords` of all text. + * 2. Flatten every segment into sentences, each tagged with that segment's + * start + id, and a global position index. + * 3. Score each sentence as the sum of its content words' document frequencies, + * normalized by the number of content words (so long sentences don't win by + * length alone). Sentences with no content words score 0. + * 4. Take the top `maxSentences` by score (ties broken by earlier position), + * then RETURN them re-sorted into chronological order (start, then position). + * + * Empty input -> []. + */ +export function summarize( + segments: Segment[], + opts?: { maxSentences?: number }, +): SourcedSentence[] { + const maxSentences = opts?.maxSentences ?? 5; + if (segments.length === 0 || maxSentences <= 0) return []; + + // 1. Document term frequencies. + const freq = new Map(); + for (const seg of segments) { + for (const tok of tokenizeWords(seg.text)) { + freq.set(tok, (freq.get(tok) ?? 0) + 1); + } + } + + // 2. Flatten into sentences with provenance + position. + const sentences: ScoredSentence[] = []; + let position = 0; + for (const seg of segments) { + for (const text of splitSentences(seg.text)) { + // 3. Score this sentence. + const tokens = tokenizeWords(text); + let sum = 0; + for (const tok of tokens) sum += freq.get(tok) ?? 0; + const score = tokens.length > 0 ? sum / tokens.length : 0; + + sentences.push({ + text, + start: seg.start, + segmentId: seg.id, + position: position++, + score, + }); + } + } + + if (sentences.length === 0) return []; + + // 4a. Pick top-N by score, ties broken by earlier position (deterministic). + const ranked = [...sentences].sort( + (a, b) => b.score - a.score || a.position - b.position, + ); + const picked = ranked.slice(0, maxSentences); + + // 4b. Restore chronological order: by start, then original position. + picked.sort((a, b) => a.start - b.start || a.position - b.position); + + return picked.map(({ text, start, segmentId }) => ({ text, start, segmentId })); +} diff --git a/src/lib/learn/tokenize.test.ts b/src/lib/learn/tokenize.test.ts new file mode 100644 index 0000000..5928093 --- /dev/null +++ b/src/lib/learn/tokenize.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect } from 'vitest'; +import { splitSentences, tokenizeWords, STOPWORDS } from './tokenize'; + +describe('splitSentences', () => { + it('splits on . ? and ! and trims', () => { + const out = splitSentences('Hello world. How are you? Great!'); + expect(out).toEqual(['Hello world.', 'How are you?', 'Great!']); + }); + + it('splits on newlines too', () => { + const out = splitSentences('first line\nsecond line\r\nthird line'); + expect(out).toEqual(['first line', 'second line', 'third line']); + }); + + it('drops empty fragments and whitespace-only pieces', () => { + expect(splitSentences(' ')).toEqual([]); + expect(splitSentences('A. . B.')).toEqual(['A.', '.', 'B.']); + expect(splitSentences('')).toEqual([]); + }); + + it('keeps the terminating punctuation with its sentence', () => { + expect(splitSentences('One. Two.')).toEqual(['One.', 'Two.']); + }); +}); + +describe('tokenizeWords', () => { + it('lowercases and splits on non-word runs', () => { + expect(tokenizeWords('Neural-Networks rock')).toEqual([ + 'neural', + 'networks', + 'rock', + ]); + }); + + it('drops empties, stopwords, and tokens shorter than 3 chars', () => { + // "is", "a", "of" are short/stop; "the" is a stopword. + const out = tokenizeWords('The cat is on a mat of fur'); + expect(out).toEqual(['cat', 'mat', 'fur']); + }); + + it('uses the shared STOPWORD set', () => { + expect(STOPWORDS.has('the')).toBe(true); + expect(tokenizeWords('the and for')).toEqual([]); + }); + + it('returns [] for empty / symbol-only input', () => { + expect(tokenizeWords('')).toEqual([]); + expect(tokenizeWords('!!! ??? ...')).toEqual([]); + }); +}); diff --git a/src/lib/learn/tokenize.ts b/src/lib/learn/tokenize.ts new file mode 100644 index 0000000..6046ee5 --- /dev/null +++ b/src/lib/learn/tokenize.ts @@ -0,0 +1,127 @@ +// Text tokenization for the deterministic study helpers (summary, glossary, +// flashcards, quiz). No model, no I/O, no platform deps — pure & testable. +// +// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias). + +/** + * A small, language-agnostic-leaning English stopword set. Kept intentionally + * compact: enough to stop "the/and/of" from dominating term frequency, without + * stripping domain words. Used by both summary scoring and glossary candidate + * selection. + */ +export const STOPWORDS: ReadonlySet = new Set([ + 'the', + 'and', + 'for', + 'are', + 'but', + 'not', + 'you', + 'all', + 'any', + 'can', + 'had', + 'her', + 'was', + 'one', + 'our', + 'out', + 'has', + 'his', + 'how', + 'its', + 'may', + 'new', + 'now', + 'old', + 'see', + 'two', + 'way', + 'who', + 'did', + 'get', + 'him', + 'use', + 'this', + 'that', + 'with', + 'from', + 'they', + 'will', + 'what', + 'when', + 'were', + 'been', + 'have', + 'into', + 'than', + 'them', + 'then', + 'some', + 'such', + 'also', + 'just', + 'like', + 'over', + 'only', + 'most', + 'much', + 'very', + 'each', + 'because', + 'about', + 'which', + 'their', + 'there', + 'these', + 'those', + 'would', + 'could', + 'should', + 'where', + 'while', + 'being', + 'between', + 'here', + 'does', + 'doing', + 'done', + 'made', + 'make', + 'used', + 'using', + 'both', + 'more', + 'less', +]); + +/** + * Split text into sentences on sentence-ending punctuation (`.`, `?`, `!`) and + * newlines. Each result is trimmed; empties are dropped. + * + * This is intentionally simple (no abbreviation handling) — lecture transcripts + * are conversational and rarely contain "Dr." / "e.g." style edge cases, and a + * mis-split sentence only marginally affects summary/glossary scoring. + */ +export function splitSentences(text: string): string[] { + return text + // Break after any run of .?! (keep the run with the sentence it ends), and + // also break on newlines so transcript line breaks become boundaries. + .split(/(?<=[.?!])\s+|[\r\n]+/) + .map((s) => s.trim()) + .filter((s) => s.length > 0); +} + +/** + * Tokenize text into "content words": lowercase, split on non-word runs, + * dropping empties, stopwords, and very short tokens (length < 3). + * + * Used for term-frequency scoring; short tokens and stopwords carry little + * topical signal and would otherwise dominate the frequency counts. + */ +export function tokenizeWords(text: string): string[] { + return text + .toLowerCase() + .split(/\W+/) + .filter((t) => t.length >= 3 && !STOPWORDS.has(t)); +}