feat(phase3): learning helpers — summary, glossary, flashcards (SM-2), quizzes
Deterministic, on-device, no model: - src/lib/learn pure modules (tokenize, summary [TextRank-ish], glossary [definition-pattern + frequency], flashcards [cloze/Q-A], srs [SM-2], quiz [MCQ with distractors]) — 37 unit tests. - Flashcard persistence: Dexie v4 + native v4 `flashcards` table; create/list/ listDue/updateSrs/delete/counts; cascades (transcript delete, course->Unsorted). - UI: transcript "Study aids" (generate summary+glossary, click-to-seek; create flashcards), Study screen (SM-2 review + Anki CSV export), per-lecture Quiz, library Study link with due-count badge. 215 tests green, 0 tsc errors, web export builds. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,127 @@
|
||||
// Text tokenization for the deterministic study helpers (summary, glossary,
|
||||
// flashcards, quiz). No model, no I/O, no platform deps — pure & testable.
|
||||
//
|
||||
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||
|
||||
/**
|
||||
* A small, language-agnostic-leaning English stopword set. Kept intentionally
|
||||
* compact: enough to stop "the/and/of" from dominating term frequency, without
|
||||
* stripping domain words. Used by both summary scoring and glossary candidate
|
||||
* selection.
|
||||
*/
|
||||
export const STOPWORDS: ReadonlySet<string> = new Set([
|
||||
'the',
|
||||
'and',
|
||||
'for',
|
||||
'are',
|
||||
'but',
|
||||
'not',
|
||||
'you',
|
||||
'all',
|
||||
'any',
|
||||
'can',
|
||||
'had',
|
||||
'her',
|
||||
'was',
|
||||
'one',
|
||||
'our',
|
||||
'out',
|
||||
'has',
|
||||
'his',
|
||||
'how',
|
||||
'its',
|
||||
'may',
|
||||
'new',
|
||||
'now',
|
||||
'old',
|
||||
'see',
|
||||
'two',
|
||||
'way',
|
||||
'who',
|
||||
'did',
|
||||
'get',
|
||||
'him',
|
||||
'use',
|
||||
'this',
|
||||
'that',
|
||||
'with',
|
||||
'from',
|
||||
'they',
|
||||
'will',
|
||||
'what',
|
||||
'when',
|
||||
'were',
|
||||
'been',
|
||||
'have',
|
||||
'into',
|
||||
'than',
|
||||
'them',
|
||||
'then',
|
||||
'some',
|
||||
'such',
|
||||
'also',
|
||||
'just',
|
||||
'like',
|
||||
'over',
|
||||
'only',
|
||||
'most',
|
||||
'much',
|
||||
'very',
|
||||
'each',
|
||||
'because',
|
||||
'about',
|
||||
'which',
|
||||
'their',
|
||||
'there',
|
||||
'these',
|
||||
'those',
|
||||
'would',
|
||||
'could',
|
||||
'should',
|
||||
'where',
|
||||
'while',
|
||||
'being',
|
||||
'between',
|
||||
'here',
|
||||
'does',
|
||||
'doing',
|
||||
'done',
|
||||
'made',
|
||||
'make',
|
||||
'used',
|
||||
'using',
|
||||
'both',
|
||||
'more',
|
||||
'less',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Split text into sentences on sentence-ending punctuation (`.`, `?`, `!`) and
|
||||
* newlines. Each result is trimmed; empties are dropped.
|
||||
*
|
||||
* This is intentionally simple (no abbreviation handling) — lecture transcripts
|
||||
* are conversational and rarely contain "Dr." / "e.g." style edge cases, and a
|
||||
* mis-split sentence only marginally affects summary/glossary scoring.
|
||||
*/
|
||||
export function splitSentences(text: string): string[] {
|
||||
return text
|
||||
// Break after any run of .?! (keep the run with the sentence it ends), and
|
||||
// also break on newlines so transcript line breaks become boundaries.
|
||||
.split(/(?<=[.?!])\s+|[\r\n]+/)
|
||||
.map((s) => s.trim())
|
||||
.filter((s) => s.length > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenize text into "content words": lowercase, split on non-word runs,
|
||||
* dropping empties, stopwords, and very short tokens (length < 3).
|
||||
*
|
||||
* Used for term-frequency scoring; short tokens and stopwords carry little
|
||||
* topical signal and would otherwise dominate the frequency counts.
|
||||
*/
|
||||
export function tokenizeWords(text: string): string[] {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.split(/\W+/)
|
||||
.filter((t) => t.length >= 3 && !STOPWORDS.has(t));
|
||||
}
|
||||
Reference in New Issue
Block a user