import { describe, it, expect } from 'vitest';
import { splitSentences, tokenizeWords, STOPWORDS } from './tokenize';

describe('splitSentences', () => {
  it('splits on . ? and ! and trims', () => {
    const out = splitSentences('Hello world. How are you? Great!');
    expect(out).toEqual(['Hello world.', 'How are you?', 'Great!']);
  });

  it('splits on newlines too', () => {
    const out = splitSentences('first line\nsecond line\r\nthird line');
    expect(out).toEqual(['first line', 'second line', 'third line']);
  });

  it('drops empty fragments and whitespace-only pieces', () => {
    expect(splitSentences('   ')).toEqual([]);
    expect(splitSentences('A.  . B.')).toEqual(['A.', '.', 'B.']);
    expect(splitSentences('')).toEqual([]);
  });

  it('keeps the terminating punctuation with its sentence', () => {
    expect(splitSentences('One. Two.')).toEqual(['One.', 'Two.']);
  });
});

describe('tokenizeWords', () => {
  it('lowercases and splits on non-word runs', () => {
    expect(tokenizeWords('Neural-Networks rock')).toEqual([
      'neural',
      'networks',
      'rock',
    ]);
  });

  it('drops empties, stopwords, and tokens shorter than 3 chars', () => {
    // "is", "a", "of" are short/stop; "the" is a stopword.
    const out = tokenizeWords('The cat is on a mat of fur');
    expect(out).toEqual(['cat', 'mat', 'fur']);
  });

  it('uses the shared STOPWORD set', () => {
    expect(STOPWORDS.has('the')).toBe(true);
    expect(tokenizeWords('the and for')).toEqual([]);
  });

  it('returns [] for empty / symbol-only input', () => {
    expect(tokenizeWords('')).toEqual([]);
    expect(tokenizeWords('!!! ??? ...')).toEqual([]);
  });
});
