feat(phase1): on-device semantic recall — embeddings, vector search, hybrid ranking
Exam-time search over your own lectures, 100% on-device (vectors never leave it): - EmbeddingEngine (transformers.js feature-extraction via the CDN loader, multilingual-e5-small 384-dim, e5 query/passage prefixes); native stub. - Vector store in StorageRepo (Dexie v3 + native v3 segvecs): upsertVectors, brute-force cosine searchVectors (course-scoped), clearVectors, unembeddedIds. Cascades: re-embed on segment edit, reassign updates vector courseId, deletes cascade. - Hybrid search: semantic candidates + lexical rank fused via reciprocal-rank-fusion (pure, tested); searchLectures() returns segment hits tagged semantic/lexical/both. - embeddingStore: build-index/backfill with progress + embed-on-save (fire-and-forget). - Search screen: query -> segment hits (snippet · course · time) -> tap jumps the transcript to that timestamp (seek + scroll-into-view). Per-course stats on Courses. 25 repo tests (incl. cosine ranking + course scoping), 13 search tests, 170 total green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -23,7 +23,12 @@ import { useTranscribe } from '@/stores/transcribeStore';
|
||||
|
||||
export default function TranscriptScreen() {
|
||||
const theme = useTheme();
|
||||
const { id } = useLocalSearchParams<{ id: string }>();
|
||||
const { id, t } = useLocalSearchParams<{ id: string; t?: string }>();
|
||||
// Deep-link target time (seconds) from a search hit, if any.
|
||||
const jumpTo = useMemo(() => {
|
||||
const n = Number(t);
|
||||
return Number.isFinite(n) && n >= 0 ? n : null;
|
||||
}, [t]);
|
||||
|
||||
const [transcript, setTranscript] = useState<Transcript | null | undefined>(undefined);
|
||||
const [title, setTitle] = useState('');
|
||||
@@ -32,6 +37,12 @@ export default function TranscriptScreen() {
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [currentTime, setCurrentTime] = useState(0);
|
||||
|
||||
const scrollRef = useRef<ScrollView | null>(null);
|
||||
// Y offset of each rendered segment row, captured via onLayout, for scroll-to.
|
||||
const segmentYs = useRef<Record<number, number>>({});
|
||||
// Ensure we only auto-jump once per (id, t) deep-link.
|
||||
const jumpedRef = useRef(false);
|
||||
|
||||
// Prefer the just-transcribed in-session audio; otherwise load the persisted
|
||||
// source media so playback/scrub works after a reload (ROADMAP Phase 0).
|
||||
const sessionAudioUrl = useTranscribe((s) => (s.lastTranscriptId === id ? s.audioUrl : undefined));
|
||||
@@ -90,13 +101,48 @@ export default function TranscriptScreen() {
|
||||
[segments, currentTime],
|
||||
);
|
||||
|
||||
const seek = (t: number) => {
|
||||
const seek = (time: number) => {
|
||||
const el = audioRef.current;
|
||||
if (!el) return;
|
||||
el.currentTime = t;
|
||||
el.currentTime = time;
|
||||
void el.play();
|
||||
};
|
||||
|
||||
// Reset the one-shot jump guard whenever the deep-link target changes.
|
||||
useEffect(() => {
|
||||
jumpedRef.current = false;
|
||||
}, [id, jumpTo]);
|
||||
|
||||
// Deep-link jump: once the transcript is loaded and we have a finite `t`,
|
||||
// seek the audio (if present), highlight the matching segment, and scroll it
|
||||
// into view. Works even without audio (scroll/highlight by time only).
|
||||
useEffect(() => {
|
||||
if (jumpTo === null || jumpedRef.current) return;
|
||||
if (!transcript || segments.length === 0) return;
|
||||
|
||||
const target = segments.findIndex((s) => jumpTo >= s.start && jumpTo < s.end);
|
||||
const idx = target >= 0 ? target : nearestIndex(segments, jumpTo);
|
||||
if (idx < 0) return;
|
||||
|
||||
jumpedRef.current = true;
|
||||
// Drive the highlight by time even if audio isn't available.
|
||||
setCurrentTime(jumpTo);
|
||||
if (audioRef.current) seek(jumpTo);
|
||||
|
||||
// Scroll once the row's Y offset has been measured (next frames).
|
||||
const scrollToIdx = () => {
|
||||
const y = segmentYs.current[idx];
|
||||
if (y === undefined) return false;
|
||||
scrollRef.current?.scrollTo({ y: Math.max(0, y - Spacing.four), animated: true });
|
||||
return true;
|
||||
};
|
||||
if (!scrollToIdx()) {
|
||||
// Layout may not be measured yet; retry on the next ticks.
|
||||
const timers = [requestAnimationFrame(() => { if (!scrollToIdx()) setTimeout(scrollToIdx, 120); })];
|
||||
return () => timers.forEach(cancelAnimationFrame);
|
||||
}
|
||||
}, [jumpTo, transcript, segments, audioUrl]);
|
||||
|
||||
const editSegment = (i: number, text: string) => {
|
||||
setSegments((prev) => prev.map((s, idx) => (idx === i ? { ...s, text } : s)));
|
||||
setDirty(true);
|
||||
@@ -130,11 +176,11 @@ export default function TranscriptScreen() {
|
||||
return (
|
||||
<ThemedView style={styles.fill}>
|
||||
<Stack.Screen options={{ title: title || 'Transcript' }} />
|
||||
<ScrollView contentContainerStyle={styles.content}>
|
||||
<ScrollView ref={scrollRef} contentContainerStyle={styles.content}>
|
||||
<TextInput
|
||||
value={title}
|
||||
onChangeText={(t) => {
|
||||
setTitle(t);
|
||||
onChangeText={(next) => {
|
||||
setTitle(next);
|
||||
setDirty(true);
|
||||
}}
|
||||
style={[styles.title, { color: theme.text }]}
|
||||
@@ -160,7 +206,12 @@ export default function TranscriptScreen() {
|
||||
</View>
|
||||
|
||||
{segments.map((s, i) => (
|
||||
<View key={i} style={[styles.segRow, i === activeIndex && { backgroundColor: theme.backgroundSelected }]}>
|
||||
<View
|
||||
key={i}
|
||||
onLayout={(e) => {
|
||||
segmentYs.current[i] = e.nativeEvent.layout.y;
|
||||
}}
|
||||
style={[styles.segRow, i === activeIndex && { backgroundColor: theme.backgroundSelected }]}>
|
||||
<Pressable onPress={() => seek(s.start)} hitSlop={6}>
|
||||
<ThemedText type="code" themeColor="textSecondary" style={styles.ts}>
|
||||
{formatClock(s.start)}
|
||||
@@ -168,7 +219,7 @@ export default function TranscriptScreen() {
|
||||
</Pressable>
|
||||
<TextInput
|
||||
value={s.text}
|
||||
onChangeText={(t) => editSegment(i, t)}
|
||||
onChangeText={(next) => editSegment(i, next)}
|
||||
multiline
|
||||
style={[styles.segText, { color: theme.text }]}
|
||||
/>
|
||||
@@ -192,6 +243,24 @@ export default function TranscriptScreen() {
|
||||
);
|
||||
}
|
||||
|
||||
/** Index of the segment whose start is closest to `time` (fallback for jumps
|
||||
* that don't land inside any segment's [start, end) window). */
|
||||
function nearestIndex(segments: Segment[], time: number): number {
|
||||
if (segments.length === 0) return -1;
|
||||
let best = 0;
|
||||
let bestDist = Infinity;
|
||||
for (let i = 0; i < segments.length; i++) {
|
||||
const seg = segments[i];
|
||||
if (!seg) continue;
|
||||
const dist = Math.abs(seg.start - time);
|
||||
if (dist < bestDist) {
|
||||
bestDist = dist;
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
function Centered({ children }: { children: React.ReactNode }) {
|
||||
return <ThemedView style={[styles.fill, styles.centered]}>{children}</ThemedView>;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user