// Text tokenization for the deterministic study helpers (summary, glossary,
// flashcards, quiz). No model, no I/O, no platform deps — pure & testable.
//
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).

/**
 * A small, language-agnostic-leaning English stopword set. Kept intentionally
 * compact: enough to stop "the/and/of" from dominating term frequency, without
 * stripping domain words. Used by both summary scoring and glossary candidate
 * selection.
 */
export const STOPWORDS: ReadonlySet<string> = new Set([
  'the',
  'and',
  'for',
  'are',
  'but',
  'not',
  'you',
  'all',
  'any',
  'can',
  'had',
  'her',
  'was',
  'one',
  'our',
  'out',
  'has',
  'his',
  'how',
  'its',
  'may',
  'new',
  'now',
  'old',
  'see',
  'two',
  'way',
  'who',
  'did',
  'get',
  'him',
  'use',
  'this',
  'that',
  'with',
  'from',
  'they',
  'will',
  'what',
  'when',
  'were',
  'been',
  'have',
  'into',
  'than',
  'them',
  'then',
  'some',
  'such',
  'also',
  'just',
  'like',
  'over',
  'only',
  'most',
  'much',
  'very',
  'each',
  'because',
  'about',
  'which',
  'their',
  'there',
  'these',
  'those',
  'would',
  'could',
  'should',
  'where',
  'while',
  'being',
  'between',
  'here',
  'does',
  'doing',
  'done',
  'made',
  'make',
  'used',
  'using',
  'both',
  'more',
  'less',
]);

/**
 * Split text into sentences on sentence-ending punctuation (`.`, `?`, `!`) and
 * newlines. Each result is trimmed; empties are dropped.
 *
 * This is intentionally simple (no abbreviation handling) — lecture transcripts
 * are conversational and rarely contain "Dr." / "e.g." style edge cases, and a
 * mis-split sentence only marginally affects summary/glossary scoring.
 */
export function splitSentences(text: string): string[] {
  return text
    // Break after any run of .?! (keep the run with the sentence it ends), and
    // also break on newlines so transcript line breaks become boundaries.
    .split(/(?<=[.?!])\s+|[\r\n]+/)
    .map((s) => s.trim())
    .filter((s) => s.length > 0);
}

/**
 * Tokenize text into "content words": lowercase, split on non-word runs,
 * dropping empties, stopwords, and very short tokens (length < 3).
 *
 * Used for term-frequency scoring; short tokens and stopwords carry little
 * topical signal and would otherwise dominate the frequency counts.
 */
export function tokenizeWords(text: string): string[] {
  return text
    .toLowerCase()
    .split(/\W+/)
    .filter((t) => t.length >= 3 && !STOPWORDS.has(t));
}