NilsBriggen
40858e0025
Deterministic, on-device, no model: - src/lib/learn pure modules (tokenize, summary [TextRank-ish], glossary [definition-pattern + frequency], flashcards [cloze/Q-A], srs [SM-2], quiz [MCQ with distractors]) — 37 unit tests. - Flashcard persistence: Dexie v4 + native v4 `flashcards` table; create/list/ listDue/updateSrs/delete/counts; cascades (transcript delete, course->Unsorted). - UI: transcript "Study aids" (generate summary+glossary, click-to-seek; create flashcards), Study screen (SM-2 review + Anki CSV export), per-lecture Quiz, library Study link with due-count badge. 215 tests green, 0 tsc errors, web export builds. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
128 lines
2.5 KiB
TypeScript
128 lines
2.5 KiB
TypeScript
// Text tokenization for the deterministic study helpers (summary, glossary,
|
|
// flashcards, quiz). No model, no I/O, no platform deps — pure & testable.
|
|
//
|
|
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
|
|
|
/**
|
|
* A small, language-agnostic-leaning English stopword set. Kept intentionally
|
|
* compact: enough to stop "the/and/of" from dominating term frequency, without
|
|
* stripping domain words. Used by both summary scoring and glossary candidate
|
|
* selection.
|
|
*/
|
|
export const STOPWORDS: ReadonlySet<string> = new Set([
|
|
'the',
|
|
'and',
|
|
'for',
|
|
'are',
|
|
'but',
|
|
'not',
|
|
'you',
|
|
'all',
|
|
'any',
|
|
'can',
|
|
'had',
|
|
'her',
|
|
'was',
|
|
'one',
|
|
'our',
|
|
'out',
|
|
'has',
|
|
'his',
|
|
'how',
|
|
'its',
|
|
'may',
|
|
'new',
|
|
'now',
|
|
'old',
|
|
'see',
|
|
'two',
|
|
'way',
|
|
'who',
|
|
'did',
|
|
'get',
|
|
'him',
|
|
'use',
|
|
'this',
|
|
'that',
|
|
'with',
|
|
'from',
|
|
'they',
|
|
'will',
|
|
'what',
|
|
'when',
|
|
'were',
|
|
'been',
|
|
'have',
|
|
'into',
|
|
'than',
|
|
'them',
|
|
'then',
|
|
'some',
|
|
'such',
|
|
'also',
|
|
'just',
|
|
'like',
|
|
'over',
|
|
'only',
|
|
'most',
|
|
'much',
|
|
'very',
|
|
'each',
|
|
'because',
|
|
'about',
|
|
'which',
|
|
'their',
|
|
'there',
|
|
'these',
|
|
'those',
|
|
'would',
|
|
'could',
|
|
'should',
|
|
'where',
|
|
'while',
|
|
'being',
|
|
'between',
|
|
'here',
|
|
'does',
|
|
'doing',
|
|
'done',
|
|
'made',
|
|
'make',
|
|
'used',
|
|
'using',
|
|
'both',
|
|
'more',
|
|
'less',
|
|
]);
|
|
|
|
/**
|
|
* Split text into sentences on sentence-ending punctuation (`.`, `?`, `!`) and
|
|
* newlines. Each result is trimmed; empties are dropped.
|
|
*
|
|
* This is intentionally simple (no abbreviation handling) — lecture transcripts
|
|
* are conversational and rarely contain "Dr." / "e.g." style edge cases, and a
|
|
* mis-split sentence only marginally affects summary/glossary scoring.
|
|
*/
|
|
export function splitSentences(text: string): string[] {
|
|
return text
|
|
// Break after any run of .?! (keep the run with the sentence it ends), and
|
|
// also break on newlines so transcript line breaks become boundaries.
|
|
.split(/(?<=[.?!])\s+|[\r\n]+/)
|
|
.map((s) => s.trim())
|
|
.filter((s) => s.length > 0);
|
|
}
|
|
|
|
/**
|
|
* Tokenize text into "content words": lowercase, split on non-word runs,
|
|
* dropping empties, stopwords, and very short tokens (length < 3).
|
|
*
|
|
* Used for term-frequency scoring; short tokens and stopwords carry little
|
|
* topical signal and would otherwise dominate the frequency counts.
|
|
*/
|
|
export function tokenizeWords(text: string): string[] {
|
|
return text
|
|
.toLowerCase()
|
|
.split(/\W+/)
|
|
.filter((t) => t.length >= 3 && !STOPWORDS.has(t));
|
|
}
|