NilsBriggen
fc24c0875d
Exam-time search over your own lectures, 100% on-device (vectors never leave it): - EmbeddingEngine (transformers.js feature-extraction via the CDN loader, multilingual-e5-small 384-dim, e5 query/passage prefixes); native stub. - Vector store in StorageRepo (Dexie v3 + native v3 segvecs): upsertVectors, brute-force cosine searchVectors (course-scoped), clearVectors, unembeddedIds. Cascades: re-embed on segment edit, reassign updates vector courseId, deletes cascade. - Hybrid search: semantic candidates + lexical rank fused via reciprocal-rank-fusion (pure, tested); searchLectures() returns segment hits tagged semantic/lexical/both. - embeddingStore: build-index/backfill with progress + embed-on-save (fire-and-forget). - Search screen: query -> segment hits (snippet · course · time) -> tap jumps the transcript to that timestamp (seek + scroll-into-view). Per-course stats on Courses. 25 repo tests (incl. cosine ranking + course scoping), 13 search tests, 170 total green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
45 lines
1.4 KiB
TypeScript
45 lines
1.4 KiB
TypeScript
// Pure lexical (keyword) ranking. Complements semantic search by catching exact
|
|
// term matches the embedding model might under-weight (names, acronyms, codes).
|
|
// No I/O, no platform deps — fully unit-testable.
|
|
|
|
/** Lowercase + split on non-word chars, dropping empties. */
|
|
function tokenize(s: string): string[] {
|
|
return s
|
|
.toLowerCase()
|
|
.split(/\W+/)
|
|
.filter((t) => t.length > 0);
|
|
}
|
|
|
|
/**
|
|
* Rank `rows` by how often the query's terms occur in each row's text.
|
|
*
|
|
* Scoring: sum, over each query term, of the number of times that term appears
|
|
* as a token in the row's text. Rows scoring zero are dropped. Result is sorted
|
|
* by score descending (ties keep input order, since Array.sort is stable).
|
|
*/
|
|
export function lexicalRank(
|
|
rows: { id: string; text: string }[],
|
|
query: string,
|
|
): { id: string; score: number }[] {
|
|
const terms = tokenize(query);
|
|
if (terms.length === 0) return [];
|
|
|
|
const out: { id: string; score: number }[] = [];
|
|
for (const row of rows) {
|
|
const tokens = tokenize(row.text);
|
|
if (tokens.length === 0) continue;
|
|
|
|
// Count occurrences per token once, then sum the query terms' counts.
|
|
const counts = new Map<string, number>();
|
|
for (const tok of tokens) counts.set(tok, (counts.get(tok) ?? 0) + 1);
|
|
|
|
let score = 0;
|
|
for (const term of terms) score += counts.get(term) ?? 0;
|
|
|
|
if (score > 0) out.push({ id: row.id, score });
|
|
}
|
|
|
|
out.sort((a, b) => b.score - a.score);
|
|
return out;
|
|
}
|