feat(phase5): optional generative RAG — ask your lectures, with citations
Strictly opt-in, gated, with the deterministic features as the always-present floor: - GenerationEngine: WebLLM (Qwen2.5-1.5B, WebGPU, CDN-loaded) + BYO-key cloud (OpenAI-compatible); native stub. Pure grounding prompt builder (4 tests). - rag.askLectures: retrieve Phase-1 hits -> grounded prompt -> answer with citations; refuses when nothing relevant; falls back to search-only when no engine is available. Never sends raw audio/transcripts — only question + snippets. - aiStore (BYO key persisted in localStorage on web), Ask screen (answer + tappable citation chips that jump to the audio + honest "verify" disclaimer), Settings AI section (engine status + bring-your-own-key form). 279 tests green, 0 tsc errors, web export builds. ROADMAP phases 0-5 complete. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,10 @@
|
|||||||
|
// https://docs.expo.dev/guides/using-eslint/
|
||||||
|
const { defineConfig } = require('eslint/config');
|
||||||
|
const expoConfig = require("eslint-config-expo/flat");
|
||||||
|
|
||||||
|
module.exports = defineConfig([
|
||||||
|
expoConfig,
|
||||||
|
{
|
||||||
|
ignores: ["dist/*"],
|
||||||
|
}
|
||||||
|
]);
|
||||||
@@ -38,6 +38,8 @@
|
|||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/node": "^25.9.3",
|
"@types/node": "^25.9.3",
|
||||||
"@types/react": "~19.2.2",
|
"@types/react": "~19.2.2",
|
||||||
|
"eslint": "^9.0.0",
|
||||||
|
"eslint-config-expo": "~56.0.4",
|
||||||
"fake-indexeddb": "^6.2.5",
|
"fake-indexeddb": "^6.2.5",
|
||||||
"fast-check": "^4.8.0",
|
"fast-check": "^4.8.0",
|
||||||
"typescript": "~6.0.3",
|
"typescript": "~6.0.3",
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ export default function RootLayout() {
|
|||||||
<Stack.Screen name="record" options={{ title: 'Record' }} />
|
<Stack.Screen name="record" options={{ title: 'Record' }} />
|
||||||
<Stack.Screen name="transcript/[id]" options={{ title: 'Transcript' }} />
|
<Stack.Screen name="transcript/[id]" options={{ title: 'Transcript' }} />
|
||||||
<Stack.Screen name="search" options={{ title: 'Search' }} />
|
<Stack.Screen name="search" options={{ title: 'Search' }} />
|
||||||
|
<Stack.Screen name="ask" options={{ title: 'Ask' }} />
|
||||||
<Stack.Screen name="courses" options={{ title: 'Courses' }} />
|
<Stack.Screen name="courses" options={{ title: 'Courses' }} />
|
||||||
<Stack.Screen name="study" options={{ title: 'Study' }} />
|
<Stack.Screen name="study" options={{ title: 'Study' }} />
|
||||||
<Stack.Screen name="quiz" options={{ title: 'Quiz' }} />
|
<Stack.Screen name="quiz" options={{ title: 'Quiz' }} />
|
||||||
|
|||||||
+315
@@ -0,0 +1,315 @@
|
|||||||
|
import { Stack, useFocusEffect, useRouter } from 'expo-router';
|
||||||
|
import { useCallback, useState } from 'react';
|
||||||
|
import {
|
||||||
|
ActivityIndicator,
|
||||||
|
Pressable,
|
||||||
|
ScrollView,
|
||||||
|
StyleSheet,
|
||||||
|
TextInput,
|
||||||
|
View,
|
||||||
|
} from 'react-native';
|
||||||
|
|
||||||
|
import { ThemedText } from '@/components/themed-text';
|
||||||
|
import { ThemedView } from '@/components/themed-view';
|
||||||
|
import { MaxContentWidth, Spacing } from '@/constants/theme';
|
||||||
|
import { useTheme } from '@/hooks/use-theme';
|
||||||
|
import type { RagAnswer, RagCitation } from '@/lib/generation/engine';
|
||||||
|
import { formatClock } from '@/lib/format';
|
||||||
|
import { useCourses } from '@/stores/coursesStore';
|
||||||
|
import { useAi } from '@/stores/aiStore';
|
||||||
|
import { useEmbedding } from '@/stores/embeddingStore';
|
||||||
|
|
||||||
|
const ACCENT = '#3c87f7';
|
||||||
|
|
||||||
|
// Course filter is a course id, or `null` for "all courses".
|
||||||
|
type CourseSel = string | null;
|
||||||
|
|
||||||
|
export default function AskScreen() {
|
||||||
|
const theme = useTheme();
|
||||||
|
const router = useRouter();
|
||||||
|
|
||||||
|
const courses = useCourses((s) => s.items);
|
||||||
|
const refreshCourses = useCourses((s) => s.refresh);
|
||||||
|
|
||||||
|
// Whether any lectures have been indexed at all (used to nudge the user to
|
||||||
|
// build the search index when an empty/ungrounded answer comes back).
|
||||||
|
const pending = useEmbedding((s) => s.pending);
|
||||||
|
const refreshPending = useEmbedding((s) => s.refreshPending);
|
||||||
|
|
||||||
|
// AI engine state (model download / readiness) is observed from the store so
|
||||||
|
// the loading bar reflects an in-flight on-device model download.
|
||||||
|
const modelStatus = useAi((s) => s.modelStatus);
|
||||||
|
const progress = useAi((s) => s.progress);
|
||||||
|
|
||||||
|
const [question, setQuestion] = useState('');
|
||||||
|
const [courseId, setCourseId] = useState<CourseSel>(null);
|
||||||
|
const [answering, setAnswering] = useState(false);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const [result, setResult] = useState<RagAnswer | null>(null);
|
||||||
|
|
||||||
|
useFocusEffect(
|
||||||
|
useCallback(() => {
|
||||||
|
void refreshCourses();
|
||||||
|
void refreshPending();
|
||||||
|
}, [refreshCourses, refreshPending]),
|
||||||
|
);
|
||||||
|
|
||||||
|
const onAsk = useCallback(async () => {
|
||||||
|
const q = question.trim();
|
||||||
|
if (!q || answering) return;
|
||||||
|
setAnswering(true);
|
||||||
|
setError(null);
|
||||||
|
setResult(null);
|
||||||
|
try {
|
||||||
|
const ans = await useAi.getState().ask(q, { courseId });
|
||||||
|
setResult(ans);
|
||||||
|
} catch (e) {
|
||||||
|
setError(e instanceof Error ? e.message : 'Something went wrong.');
|
||||||
|
} finally {
|
||||||
|
setAnswering(false);
|
||||||
|
}
|
||||||
|
}, [question, courseId, answering]);
|
||||||
|
|
||||||
|
const openCitation = (c: RagCitation) =>
|
||||||
|
router.push({
|
||||||
|
pathname: '/transcript/[id]',
|
||||||
|
params: { id: c.transcriptId, t: String(Math.floor(c.start)) },
|
||||||
|
});
|
||||||
|
|
||||||
|
// The model is downloading/preparing on-device (WebLLM). Show a progress bar.
|
||||||
|
const loadingModel = modelStatus === 'loading';
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ThemedView style={styles.fill}>
|
||||||
|
<Stack.Screen options={{ title: 'Ask' }} />
|
||||||
|
<ScrollView contentContainerStyle={styles.content} keyboardShouldPersistTaps="handled">
|
||||||
|
<ThemedText type="small" themeColor="textSecondary">
|
||||||
|
Ask a question and get an answer grounded in your lectures — every claim links back to the
|
||||||
|
moment it came from.
|
||||||
|
</ThemedText>
|
||||||
|
|
||||||
|
{courses.length > 0 && (
|
||||||
|
<ScrollView
|
||||||
|
horizontal
|
||||||
|
showsHorizontalScrollIndicator={false}
|
||||||
|
contentContainerStyle={styles.filterBar}>
|
||||||
|
<FilterChip label="All courses" active={courseId === null} onPress={() => setCourseId(null)} />
|
||||||
|
{courses.map((c) => (
|
||||||
|
<FilterChip
|
||||||
|
key={c.id}
|
||||||
|
label={c.name}
|
||||||
|
active={courseId === c.id}
|
||||||
|
onPress={() => setCourseId(c.id)}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</ScrollView>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<TextInput
|
||||||
|
value={question}
|
||||||
|
onChangeText={setQuestion}
|
||||||
|
onSubmitEditing={() => void onAsk()}
|
||||||
|
returnKeyType="search"
|
||||||
|
autoFocus
|
||||||
|
multiline
|
||||||
|
placeholder="Ask your lectures anything…"
|
||||||
|
placeholderTextColor={theme.textSecondary}
|
||||||
|
style={[styles.input, { color: theme.text, backgroundColor: theme.backgroundElement }]}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<Pressable
|
||||||
|
onPress={() => void onAsk()}
|
||||||
|
disabled={answering || question.trim() === ''}
|
||||||
|
style={({ pressed }) => [
|
||||||
|
styles.askBtn,
|
||||||
|
{ opacity: answering || question.trim() === '' ? 0.5 : pressed ? 0.85 : 1 },
|
||||||
|
]}>
|
||||||
|
<ThemedText style={styles.askBtnText}>Ask</ThemedText>
|
||||||
|
</Pressable>
|
||||||
|
|
||||||
|
{loadingModel && (
|
||||||
|
<ThemedView type="backgroundElement" style={styles.card}>
|
||||||
|
<ThemedText type="smallBold">Preparing on-device AI… {Math.round(progress * 100)}%</ThemedText>
|
||||||
|
<ProgressBar value={progress} />
|
||||||
|
<ThemedText type="small" themeColor="textSecondary">
|
||||||
|
The model downloads once, then runs locally on your device.
|
||||||
|
</ThemedText>
|
||||||
|
</ThemedView>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{answering && !loadingModel && (
|
||||||
|
<View style={styles.center}>
|
||||||
|
<ActivityIndicator />
|
||||||
|
<ThemedText type="small" themeColor="textSecondary" style={styles.centerText}>
|
||||||
|
Thinking…
|
||||||
|
</ThemedText>
|
||||||
|
</View>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{error && (
|
||||||
|
<ThemedView type="backgroundElement" style={styles.card}>
|
||||||
|
<ThemedText type="smallBold">Couldn't answer that</ThemedText>
|
||||||
|
<ThemedText type="small" themeColor="textSecondary">
|
||||||
|
{error}
|
||||||
|
</ThemedText>
|
||||||
|
</ThemedView>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{result && !answering && (
|
||||||
|
<Answer
|
||||||
|
result={result}
|
||||||
|
indexEmpty={pending > 0}
|
||||||
|
onOpenCitation={openCitation}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</ScrollView>
|
||||||
|
</ThemedView>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function Answer({
|
||||||
|
result,
|
||||||
|
indexEmpty,
|
||||||
|
onOpenCitation,
|
||||||
|
}: {
|
||||||
|
result: RagAnswer;
|
||||||
|
indexEmpty: boolean;
|
||||||
|
onOpenCitation: (c: RagCitation) => void;
|
||||||
|
}) {
|
||||||
|
// Nothing relevant was retrieved — we refuse to invent an answer.
|
||||||
|
if (!result.grounded) {
|
||||||
|
return (
|
||||||
|
<ThemedView type="backgroundElement" style={styles.card}>
|
||||||
|
<ThemedText type="smallBold">
|
||||||
|
I couldn't find anything about that in your lectures.
|
||||||
|
</ThemedText>
|
||||||
|
{indexEmpty && (
|
||||||
|
<ThemedText type="small" themeColor="textSecondary">
|
||||||
|
Tip: build the search index on the Search screen so your lectures become searchable.
|
||||||
|
</ThemedText>
|
||||||
|
)}
|
||||||
|
</ThemedView>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<View style={styles.answerWrap}>
|
||||||
|
{result.generated ? (
|
||||||
|
<ThemedView type="backgroundElement" style={styles.answerCard}>
|
||||||
|
<ThemedText type="default">{result.answer}</ThemedText>
|
||||||
|
</ThemedView>
|
||||||
|
) : (
|
||||||
|
// Grounded but no LLM available: deterministic search-only fallback.
|
||||||
|
<ThemedView type="backgroundElement" style={styles.card}>
|
||||||
|
<ThemedText type="small" themeColor="textSecondary">
|
||||||
|
On-device AI needs WebGPU, or add an API key in Settings — here are the matching moments:
|
||||||
|
</ThemedText>
|
||||||
|
</ThemedView>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<ThemedText type="smallBold" style={styles.sourcesHeading}>
|
||||||
|
Sources
|
||||||
|
</ThemedText>
|
||||||
|
{result.citations.map((c, i) => (
|
||||||
|
<CitationChip
|
||||||
|
key={`${c.transcriptId}:${c.segmentId ?? i}`}
|
||||||
|
citation={c}
|
||||||
|
onPress={() => onOpenCitation(c)}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
|
||||||
|
<ThemedText type="small" themeColor="textSecondary" style={styles.disclaimer}>
|
||||||
|
AI answers can be wrong — every claim links to the lecture; verify against the source.
|
||||||
|
</ThemedText>
|
||||||
|
</View>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function CitationChip({
|
||||||
|
citation,
|
||||||
|
onPress,
|
||||||
|
}: {
|
||||||
|
citation: RagCitation;
|
||||||
|
onPress: () => void;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<Pressable onPress={onPress} style={({ pressed }) => [pressed && styles.pressed]}>
|
||||||
|
<ThemedView type="backgroundElement" style={styles.card}>
|
||||||
|
<ThemedText type="small" numberOfLines={3}>
|
||||||
|
{citation.text}
|
||||||
|
</ThemedText>
|
||||||
|
<ThemedText type="small" style={styles.citationTime}>
|
||||||
|
({formatClock(citation.start)})
|
||||||
|
</ThemedText>
|
||||||
|
</ThemedView>
|
||||||
|
</Pressable>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function FilterChip({
|
||||||
|
label,
|
||||||
|
active,
|
||||||
|
onPress,
|
||||||
|
}: {
|
||||||
|
label: string;
|
||||||
|
active: boolean;
|
||||||
|
onPress: () => void;
|
||||||
|
}) {
|
||||||
|
const theme = useTheme();
|
||||||
|
return (
|
||||||
|
<Pressable
|
||||||
|
onPress={onPress}
|
||||||
|
style={[styles.filterChip, { backgroundColor: active ? ACCENT : theme.backgroundElement }]}>
|
||||||
|
<ThemedText type="small" style={active ? styles.chipActive : undefined}>
|
||||||
|
{label}
|
||||||
|
</ThemedText>
|
||||||
|
</Pressable>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function ProgressBar({ value }: { value: number }) {
|
||||||
|
return (
|
||||||
|
<View style={styles.track}>
|
||||||
|
<View style={[styles.bar, { width: `${Math.max(2, Math.min(100, value * 100))}%` }]} />
|
||||||
|
</View>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const styles = StyleSheet.create({
|
||||||
|
fill: { flex: 1 },
|
||||||
|
content: {
|
||||||
|
padding: Spacing.three,
|
||||||
|
gap: Spacing.three,
|
||||||
|
maxWidth: MaxContentWidth,
|
||||||
|
width: '100%',
|
||||||
|
alignSelf: 'center',
|
||||||
|
},
|
||||||
|
filterBar: { gap: Spacing.two, paddingVertical: Spacing.one, paddingRight: Spacing.three },
|
||||||
|
filterChip: { paddingHorizontal: Spacing.three, paddingVertical: Spacing.one, borderRadius: 999 },
|
||||||
|
chipActive: { color: '#fff', fontWeight: '700' },
|
||||||
|
input: {
|
||||||
|
borderRadius: Spacing.two,
|
||||||
|
paddingHorizontal: Spacing.three,
|
||||||
|
paddingVertical: Spacing.two,
|
||||||
|
fontSize: 15,
|
||||||
|
minHeight: 48,
|
||||||
|
},
|
||||||
|
askBtn: {
|
||||||
|
backgroundColor: ACCENT,
|
||||||
|
paddingVertical: Spacing.three,
|
||||||
|
borderRadius: Spacing.three,
|
||||||
|
alignItems: 'center',
|
||||||
|
},
|
||||||
|
askBtnText: { color: '#fff', fontWeight: '700', fontSize: 16 },
|
||||||
|
card: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.two },
|
||||||
|
answerWrap: { gap: Spacing.three },
|
||||||
|
answerCard: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.two },
|
||||||
|
sourcesHeading: { marginTop: Spacing.one },
|
||||||
|
citationTime: { color: ACCENT, fontWeight: '700' },
|
||||||
|
disclaimer: { marginTop: Spacing.one, fontStyle: 'italic' },
|
||||||
|
center: { alignItems: 'center', gap: Spacing.two, paddingVertical: Spacing.four },
|
||||||
|
centerText: { textAlign: 'center' },
|
||||||
|
track: { height: 6, borderRadius: 3, backgroundColor: '#88888833', overflow: 'hidden' },
|
||||||
|
bar: { height: 6, borderRadius: 3, backgroundColor: ACCENT },
|
||||||
|
pressed: { opacity: 0.7 },
|
||||||
|
});
|
||||||
@@ -87,6 +87,11 @@ export default function LibraryScreen() {
|
|||||||
<ThemedText type="link" themeColor="textSecondary">Search</ThemedText>
|
<ThemedText type="link" themeColor="textSecondary">Search</ThemedText>
|
||||||
</Pressable>
|
</Pressable>
|
||||||
</Link>
|
</Link>
|
||||||
|
<Link href="/ask" asChild>
|
||||||
|
<Pressable hitSlop={8}>
|
||||||
|
<ThemedText type="link" themeColor="textSecondary">Ask</ThemedText>
|
||||||
|
</Pressable>
|
||||||
|
</Link>
|
||||||
<Link href="/courses" asChild>
|
<Link href="/courses" asChild>
|
||||||
<Pressable hitSlop={8}>
|
<Pressable hitSlop={8}>
|
||||||
<ThemedText type="link" themeColor="textSecondary">Courses</ThemedText>
|
<ThemedText type="link" themeColor="textSecondary">Courses</ThemedText>
|
||||||
|
|||||||
+189
-3
@@ -1,12 +1,18 @@
|
|||||||
import { ScrollView, StyleSheet, Pressable, View } from 'react-native';
|
import { useEffect, useState } from 'react';
|
||||||
|
import { ScrollView, StyleSheet, Pressable, TextInput, View } from 'react-native';
|
||||||
|
|
||||||
import { ThemedText } from '@/components/themed-text';
|
import { ThemedText } from '@/components/themed-text';
|
||||||
import { ThemedView } from '@/components/themed-view';
|
import { ThemedView } from '@/components/themed-view';
|
||||||
import { MaxContentWidth, Spacing } from '@/constants/theme';
|
import { MaxContentWidth, Spacing } from '@/constants/theme';
|
||||||
import { useTheme } from '@/hooks/use-theme';
|
import { useTheme } from '@/hooks/use-theme';
|
||||||
import { listModels } from '@/lib/models/catalog';
|
import { listModels } from '@/lib/models/catalog';
|
||||||
|
import { useAi } from '@/stores/aiStore';
|
||||||
import { useTranscribe } from '@/stores/transcribeStore';
|
import { useTranscribe } from '@/stores/transcribeStore';
|
||||||
|
|
||||||
|
const ACCENT = '#3c87f7';
|
||||||
|
const DEFAULT_BASE_URL = 'https://api.openai.com/v1';
|
||||||
|
const DEFAULT_MODEL = 'gpt-4o-mini';
|
||||||
|
|
||||||
export default function SettingsScreen() {
|
export default function SettingsScreen() {
|
||||||
const theme = useTheme();
|
const theme = useTheme();
|
||||||
const modelId = useTranscribe((s) => s.modelId);
|
const modelId = useTranscribe((s) => s.modelId);
|
||||||
@@ -28,10 +34,10 @@ export default function SettingsScreen() {
|
|||||||
<Pressable key={m.id} onPress={() => setModel(m.id)}>
|
<Pressable key={m.id} onPress={() => setModel(m.id)}>
|
||||||
<ThemedView
|
<ThemedView
|
||||||
type={selected ? 'backgroundSelected' : 'backgroundElement'}
|
type={selected ? 'backgroundSelected' : 'backgroundElement'}
|
||||||
style={[styles.card, selected && { borderColor: '#3c87f7', borderWidth: 1 }]}>
|
style={[styles.card, selected && { borderColor: ACCENT, borderWidth: 1 }]}>
|
||||||
<View style={styles.rowBetween}>
|
<View style={styles.rowBetween}>
|
||||||
<ThemedText type="smallBold">{m.label}</ThemedText>
|
<ThemedText type="smallBold">{m.label}</ThemedText>
|
||||||
{selected && <ThemedText type="small" style={{ color: '#3c87f7' }}>✓ selected</ThemedText>}
|
{selected && <ThemedText type="small" style={{ color: ACCENT }}>✓ selected</ThemedText>}
|
||||||
</View>
|
</View>
|
||||||
<ThemedText type="small" themeColor="textSecondary">
|
<ThemedText type="small" themeColor="textSecondary">
|
||||||
{cap(m.tier)} · ~{m.approxMB} MB · {m.multilingual ? 'multilingual' : 'English-only'}
|
{cap(m.tier)} · ~{m.approxMB} MB · {m.multilingual ? 'multilingual' : 'English-only'}
|
||||||
@@ -41,6 +47,9 @@ export default function SettingsScreen() {
|
|||||||
);
|
);
|
||||||
})}
|
})}
|
||||||
|
|
||||||
|
<View style={styles.spacer} />
|
||||||
|
<AiSection />
|
||||||
|
|
||||||
<View style={styles.spacer} />
|
<View style={styles.spacer} />
|
||||||
<ThemedText type="subtitle">Privacy</ThemedText>
|
<ThemedText type="subtitle">Privacy</ThemedText>
|
||||||
<ThemedText type="small" themeColor="textSecondary">
|
<ThemedText type="small" themeColor="textSecondary">
|
||||||
@@ -53,6 +62,159 @@ export default function SettingsScreen() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function AiSection() {
|
||||||
|
const theme = useTheme();
|
||||||
|
|
||||||
|
const cloud = useAi((s) => s.cloud);
|
||||||
|
const engineKind = useAi((s) => s.engineKind);
|
||||||
|
const setCloud = useAi((s) => s.setCloud);
|
||||||
|
const refreshAvailability = useAi((s) => s.refreshAvailability);
|
||||||
|
|
||||||
|
// Form fields seeded from any saved cloud config.
|
||||||
|
const [baseUrl, setBaseUrl] = useState(cloud?.baseUrl ?? DEFAULT_BASE_URL);
|
||||||
|
const [model, setModel] = useState(cloud?.model ?? DEFAULT_MODEL);
|
||||||
|
const [apiKey, setApiKey] = useState(cloud?.apiKey ?? '');
|
||||||
|
|
||||||
|
// Probe availability (WebGPU / key set) when the screen mounts.
|
||||||
|
useEffect(() => {
|
||||||
|
void refreshAvailability();
|
||||||
|
}, [refreshAvailability]);
|
||||||
|
|
||||||
|
// Keep the form in sync if the saved config changes elsewhere.
|
||||||
|
useEffect(() => {
|
||||||
|
setBaseUrl(cloud?.baseUrl ?? DEFAULT_BASE_URL);
|
||||||
|
setModel(cloud?.model ?? DEFAULT_MODEL);
|
||||||
|
setApiKey(cloud?.apiKey ?? '');
|
||||||
|
}, [cloud]);
|
||||||
|
|
||||||
|
const engineLabel =
|
||||||
|
engineKind === 'cloud'
|
||||||
|
? 'Cloud (your key)'
|
||||||
|
: engineKind === 'webllm'
|
||||||
|
? 'On-device (WebGPU)'
|
||||||
|
: 'Not available';
|
||||||
|
|
||||||
|
const onSave = () => {
|
||||||
|
const url = baseUrl.trim() || DEFAULT_BASE_URL;
|
||||||
|
const mdl = model.trim() || DEFAULT_MODEL;
|
||||||
|
const key = apiKey.trim();
|
||||||
|
if (!key) return;
|
||||||
|
setCloud({ baseUrl: url, apiKey: key, model: mdl });
|
||||||
|
void refreshAvailability();
|
||||||
|
};
|
||||||
|
|
||||||
|
const onClear = () => {
|
||||||
|
setCloud(undefined);
|
||||||
|
setApiKey('');
|
||||||
|
setBaseUrl(DEFAULT_BASE_URL);
|
||||||
|
setModel(DEFAULT_MODEL);
|
||||||
|
void refreshAvailability();
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<ThemedText type="subtitle">AI (optional)</ThemedText>
|
||||||
|
<ThemedText type="small" themeColor="textSecondary">
|
||||||
|
Wisp can answer questions about your lectures with cited answers. This is fully optional —
|
||||||
|
without it, you still get on-device semantic search. It runs on-device when your browser
|
||||||
|
supports WebGPU, or through your own API key below.
|
||||||
|
</ThemedText>
|
||||||
|
|
||||||
|
<ThemedView type="backgroundElement" style={styles.card}>
|
||||||
|
<View style={styles.rowBetween}>
|
||||||
|
<ThemedText type="smallBold">Active engine</ThemedText>
|
||||||
|
<ThemedText type="small" style={{ color: engineKind === 'none' ? theme.textSecondary : ACCENT }}>
|
||||||
|
{engineLabel}
|
||||||
|
</ThemedText>
|
||||||
|
</View>
|
||||||
|
<ThemedText type="small" themeColor="textSecondary">
|
||||||
|
On-device generation needs a WebGPU-capable browser. If that isn't available, add an
|
||||||
|
API key below to use a cloud model instead.
|
||||||
|
</ThemedText>
|
||||||
|
</ThemedView>
|
||||||
|
|
||||||
|
<ThemedText type="smallBold">Bring your own key</ThemedText>
|
||||||
|
<ThemedText type="small" themeColor="textSecondary">
|
||||||
|
Use any OpenAI-compatible endpoint. Your key is stored locally on this device and is never
|
||||||
|
shared with Wisp. Only your question and the retrieved lecture snippets are sent to the
|
||||||
|
endpoint — never your raw audio or full transcripts.
|
||||||
|
</ThemedText>
|
||||||
|
|
||||||
|
<Field label="Base URL">
|
||||||
|
<TextInput
|
||||||
|
value={baseUrl}
|
||||||
|
onChangeText={setBaseUrl}
|
||||||
|
autoCapitalize="none"
|
||||||
|
autoCorrect={false}
|
||||||
|
keyboardType="url"
|
||||||
|
placeholder={DEFAULT_BASE_URL}
|
||||||
|
placeholderTextColor={theme.textSecondary}
|
||||||
|
style={[styles.input, { color: theme.text, backgroundColor: theme.backgroundElement }]}
|
||||||
|
/>
|
||||||
|
</Field>
|
||||||
|
|
||||||
|
<Field label="Model">
|
||||||
|
<TextInput
|
||||||
|
value={model}
|
||||||
|
onChangeText={setModel}
|
||||||
|
autoCapitalize="none"
|
||||||
|
autoCorrect={false}
|
||||||
|
placeholder={DEFAULT_MODEL}
|
||||||
|
placeholderTextColor={theme.textSecondary}
|
||||||
|
style={[styles.input, { color: theme.text, backgroundColor: theme.backgroundElement }]}
|
||||||
|
/>
|
||||||
|
</Field>
|
||||||
|
|
||||||
|
<Field label="API key">
|
||||||
|
<TextInput
|
||||||
|
value={apiKey}
|
||||||
|
onChangeText={setApiKey}
|
||||||
|
autoCapitalize="none"
|
||||||
|
autoCorrect={false}
|
||||||
|
secureTextEntry
|
||||||
|
placeholder="sk-…"
|
||||||
|
placeholderTextColor={theme.textSecondary}
|
||||||
|
style={[styles.input, { color: theme.text, backgroundColor: theme.backgroundElement }]}
|
||||||
|
/>
|
||||||
|
</Field>
|
||||||
|
|
||||||
|
<View style={styles.btnRow}>
|
||||||
|
<Pressable
|
||||||
|
onPress={onSave}
|
||||||
|
disabled={apiKey.trim() === ''}
|
||||||
|
style={({ pressed }) => [
|
||||||
|
styles.saveBtn,
|
||||||
|
{ opacity: apiKey.trim() === '' ? 0.5 : pressed ? 0.85 : 1 },
|
||||||
|
]}>
|
||||||
|
<ThemedText style={styles.saveBtnText}>Save key</ThemedText>
|
||||||
|
</Pressable>
|
||||||
|
<Pressable
|
||||||
|
onPress={onClear}
|
||||||
|
disabled={!cloud}
|
||||||
|
style={({ pressed }) => [
|
||||||
|
styles.clearBtn,
|
||||||
|
{ borderColor: theme.textSecondary, opacity: !cloud ? 0.5 : pressed ? 0.85 : 1 },
|
||||||
|
]}>
|
||||||
|
<ThemedText type="smallBold" themeColor="textSecondary">
|
||||||
|
Clear
|
||||||
|
</ThemedText>
|
||||||
|
</Pressable>
|
||||||
|
</View>
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function Field({ label, children }: { label: string; children: React.ReactNode }) {
|
||||||
|
return (
|
||||||
|
<View style={styles.field}>
|
||||||
|
<ThemedText type="small" themeColor="textSecondary">
|
||||||
|
{label}
|
||||||
|
</ThemedText>
|
||||||
|
{children}
|
||||||
|
</View>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function cap(s: string) {
|
function cap(s: string) {
|
||||||
return s.charAt(0).toUpperCase() + s.slice(1);
|
return s.charAt(0).toUpperCase() + s.slice(1);
|
||||||
}
|
}
|
||||||
@@ -63,4 +225,28 @@ const styles = StyleSheet.create({
|
|||||||
card: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.one },
|
card: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.one },
|
||||||
rowBetween: { flexDirection: 'row', alignItems: 'center', justifyContent: 'space-between' },
|
rowBetween: { flexDirection: 'row', alignItems: 'center', justifyContent: 'space-between' },
|
||||||
spacer: { height: Spacing.three },
|
spacer: { height: Spacing.three },
|
||||||
|
field: { gap: Spacing.one },
|
||||||
|
input: {
|
||||||
|
borderRadius: Spacing.two,
|
||||||
|
paddingHorizontal: Spacing.three,
|
||||||
|
paddingVertical: Spacing.two,
|
||||||
|
fontSize: 15,
|
||||||
|
},
|
||||||
|
btnRow: { flexDirection: 'row', gap: Spacing.two, marginTop: Spacing.one },
|
||||||
|
saveBtn: {
|
||||||
|
flex: 1,
|
||||||
|
backgroundColor: ACCENT,
|
||||||
|
paddingVertical: Spacing.two,
|
||||||
|
borderRadius: Spacing.two,
|
||||||
|
alignItems: 'center',
|
||||||
|
},
|
||||||
|
saveBtnText: { color: '#fff', fontWeight: '700' },
|
||||||
|
clearBtn: {
|
||||||
|
paddingHorizontal: Spacing.four,
|
||||||
|
paddingVertical: Spacing.two,
|
||||||
|
borderRadius: Spacing.two,
|
||||||
|
borderWidth: 1,
|
||||||
|
alignItems: 'center',
|
||||||
|
justifyContent: 'center',
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,95 @@
|
|||||||
|
// Bring-your-own-key cloud generation engine (OpenAI-compatible chat
|
||||||
|
// completions). Used when the user has supplied a CloudConfig (baseUrl, apiKey,
|
||||||
|
// model). Works against OpenAI itself or any OpenAI-compatible endpoint.
|
||||||
|
//
|
||||||
|
// PRIVACY: this engine sends ONLY the system prompt + the user's question +
|
||||||
|
// the retrieved lecture snippets (the caller assembles these via prompt.ts).
|
||||||
|
// Raw audio and full transcripts are NEVER sent — that is a hard Phase 5 rule.
|
||||||
|
//
|
||||||
|
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||||
|
|
||||||
|
import type { CloudConfig, GenerationEngine, GenOptions } from './engine';
|
||||||
|
|
||||||
|
interface ChatMessage {
|
||||||
|
role: 'system' | 'user';
|
||||||
|
content: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ChatCompletionResponse {
|
||||||
|
choices?: { message?: { content?: string } }[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createCloudEngine(cfg: CloudConfig): GenerationEngine {
|
||||||
|
// Normalize: drop a trailing slash so we can append the path cleanly.
|
||||||
|
const base = cfg.baseUrl.replace(/\/+$/, '');
|
||||||
|
|
||||||
|
return {
|
||||||
|
kind: 'cloud',
|
||||||
|
label: 'Cloud (your key)',
|
||||||
|
|
||||||
|
async isAvailable(): Promise<boolean> {
|
||||||
|
return !!cfg.apiKey;
|
||||||
|
},
|
||||||
|
|
||||||
|
// Cloud models need no local download; nothing to load.
|
||||||
|
async loadModel(): Promise<void> {
|
||||||
|
/* no-op */
|
||||||
|
},
|
||||||
|
|
||||||
|
isLoaded(): boolean {
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
|
||||||
|
async generate(prompt: string, opts?: GenOptions): Promise<string> {
|
||||||
|
if (!cfg.apiKey) {
|
||||||
|
throw new Error('Cloud generation requires an API key.');
|
||||||
|
}
|
||||||
|
|
||||||
|
const messages: ChatMessage[] = [];
|
||||||
|
if (opts?.system) messages.push({ role: 'system', content: opts.system });
|
||||||
|
messages.push({ role: 'user', content: prompt });
|
||||||
|
|
||||||
|
let res: Response;
|
||||||
|
try {
|
||||||
|
res = await fetch(`${base}/chat/completions`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
Authorization: `Bearer ${cfg.apiKey}`,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
model: cfg.model,
|
||||||
|
messages,
|
||||||
|
max_tokens: opts?.maxTokens ?? 512,
|
||||||
|
}),
|
||||||
|
signal: opts?.signal,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof DOMException && err.name === 'AbortError') throw err;
|
||||||
|
throw new Error(
|
||||||
|
`Cloud request failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
const detail = await res.text().catch(() => '');
|
||||||
|
throw new Error(
|
||||||
|
`Cloud request failed (${res.status} ${res.statusText})${detail ? `: ${detail}` : ''}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let data: ChatCompletionResponse;
|
||||||
|
try {
|
||||||
|
data = (await res.json()) as ChatCompletionResponse;
|
||||||
|
} catch {
|
||||||
|
throw new Error('Cloud response was not valid JSON.');
|
||||||
|
}
|
||||||
|
|
||||||
|
const content = data.choices?.[0]?.message?.content;
|
||||||
|
if (typeof content !== 'string') {
|
||||||
|
throw new Error('Cloud response did not contain a message.');
|
||||||
|
}
|
||||||
|
return content;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
// Optional generative layer (ROADMAP Phase 5). STRICTLY opt-in and gated: a
|
||||||
|
// local WebLLM model (WebGPU) or a bring-your-own-key cloud model. When neither
|
||||||
|
// is available the app falls back to the deterministic Phase 1/3 features — the
|
||||||
|
// LLM is never a hard dependency, and raw audio/transcripts are NEVER sent to a
|
||||||
|
// cloud: only the user's question + the retrieved snippets go out (BYO-key path).
|
||||||
|
|
||||||
|
export interface GenOptions {
|
||||||
|
system?: string;
|
||||||
|
maxTokens?: number;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
/** Streamed token callback (optional). */
|
||||||
|
onToken?: (delta: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface GenerationEngine {
|
||||||
|
/** Which backend this is. */
|
||||||
|
readonly kind: 'webllm' | 'cloud' | 'none';
|
||||||
|
/** Human label, e.g. 'On-device (Qwen2.5-1.5B)' or 'Cloud (your key)'. */
|
||||||
|
readonly label: string;
|
||||||
|
/** Whether this engine can run here right now (WebGPU present / key set). */
|
||||||
|
isAvailable(): Promise<boolean>;
|
||||||
|
/** Load/prepare the model. onProgress in [0,1] (for the WebLLM download). */
|
||||||
|
loadModel(onProgress?: (p: number) => void): Promise<void>;
|
||||||
|
isLoaded(): boolean;
|
||||||
|
/** Generate a completion for `prompt`. */
|
||||||
|
generate(prompt: string, opts?: GenOptions): Promise<string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Cloud (BYO-key) configuration — OpenAI-compatible chat completions. */
|
||||||
|
export interface CloudConfig {
|
||||||
|
/** e.g. https://api.openai.com/v1 (or any OpenAI-compatible base). */
|
||||||
|
baseUrl: string;
|
||||||
|
apiKey: string;
|
||||||
|
model: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Default local WebLLM model (small instruct, q4) — pulled from the CDN. */
|
||||||
|
export const WEBLLM_MODEL = 'Qwen2.5-1.5B-Instruct-q4f16_1-MLC';
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// RAG ("ask your lectures") result shape — produced by rag.ts.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export interface RagCitation {
|
||||||
|
transcriptId: string;
|
||||||
|
segmentId?: string;
|
||||||
|
start: number;
|
||||||
|
text: string;
|
||||||
|
/** Retrieval score from Phase 1 search. */
|
||||||
|
score: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RagAnswer {
|
||||||
|
/** The generated answer (grounded in the citations). */
|
||||||
|
answer: string;
|
||||||
|
/** The lecture moments the answer is based on (always shown verbatim). */
|
||||||
|
citations: RagCitation[];
|
||||||
|
/** False when no relevant lecture content was retrieved (we refuse to answer). */
|
||||||
|
grounded: boolean;
|
||||||
|
/** True when produced by an LLM; false when this is the search-only fallback. */
|
||||||
|
generated: boolean;
|
||||||
|
}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
// NATIVE on-device generation stub. Running a local LLM on device (e.g. via
|
||||||
|
// llama.rn / MLC's native runtime) is a Phase 5 follow-up; until then the native
|
||||||
|
// webllm engine reports unavailable so callers transparently fall back to the
|
||||||
|
// search-only path (or a BYO-key cloud model). It NEVER produces a fake answer.
|
||||||
|
//
|
||||||
|
// This module is selected by Metro for native (.native.ts) and is never imported
|
||||||
|
// by any vitest test.
|
||||||
|
|
||||||
|
import type { GenerationEngine } from './engine';
|
||||||
|
|
||||||
|
const NOT_AVAILABLE = 'On-device generation is not available on native yet';
|
||||||
|
|
||||||
|
export const webllm: GenerationEngine = {
|
||||||
|
kind: 'webllm',
|
||||||
|
label: 'On-device (Qwen2.5-1.5B)',
|
||||||
|
|
||||||
|
async isAvailable(): Promise<boolean> {
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
|
||||||
|
async loadModel(): Promise<void> {
|
||||||
|
throw new Error(NOT_AVAILABLE);
|
||||||
|
},
|
||||||
|
|
||||||
|
isLoaded(): boolean {
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
|
||||||
|
async generate(): Promise<string> {
|
||||||
|
throw new Error(NOT_AVAILABLE);
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
// Base resolver for TypeScript. Metro picks engineImpl.web.ts / engineImpl.native.ts
|
||||||
|
// by platform extension at build time; tsc resolves this file (defaults to web).
|
||||||
|
export { webllm } from './engineImpl.web';
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
// WEB-ONLY on-device generation engine, backed by @mlc-ai/web-llm running a
|
||||||
|
// small instruct model (Qwen2.5-1.5B, q4) entirely in the browser via WebGPU.
|
||||||
|
//
|
||||||
|
// WHY WE LOAD IT FROM A CDN AT RUNTIME (not a static import):
|
||||||
|
// web-llm ships WASM and uses dynamic imports that Metro (Expo's web bundler)
|
||||||
|
// cannot statically bundle. As with transcription/engineImpl.web.ts, we never
|
||||||
|
// let Metro see the package: we load the ESM build from a CDN at runtime via a
|
||||||
|
// dynamic import hidden behind `new Function` (so Metro's static analyzer can't
|
||||||
|
// trip over it). The browser resolves it natively. This keeps the JS bundle
|
||||||
|
// small — the model itself is a ~1GB download, cached by the browser after the
|
||||||
|
// first run — and means web-llm is NEVER bundled.
|
||||||
|
//
|
||||||
|
// Requires WebGPU. On a device without a WebGPU adapter, isAvailable() is false
|
||||||
|
// and callers fall back to the search-only path (no fake answers).
|
||||||
|
//
|
||||||
|
// This module is web-only and is NEVER imported by any vitest test.
|
||||||
|
|
||||||
|
import { WEBLLM_MODEL } from './engine';
|
||||||
|
import type { GenerationEngine, GenOptions } from './engine';
|
||||||
|
|
||||||
|
// Pin the web-llm ESM build we load at runtime.
|
||||||
|
const WEBLLM_CDN = 'https://esm.run/@mlc-ai/web-llm';
|
||||||
|
|
||||||
|
// `new Function` hides the dynamic import() specifier from Metro's bundler so it
|
||||||
|
// never tries to resolve/transform web-llm or its WASM.
|
||||||
|
const runtimeImport = new Function('u', 'return import(u)') as (u: string) => Promise<WebLlmModule>;
|
||||||
|
|
||||||
|
// Minimal structural types for the bits of web-llm we use.
|
||||||
|
interface InitProgressReport {
|
||||||
|
progress: number;
|
||||||
|
text?: string;
|
||||||
|
}
|
||||||
|
interface ChatMessage {
|
||||||
|
role: 'system' | 'user' | 'assistant';
|
||||||
|
content: string;
|
||||||
|
}
|
||||||
|
interface CompletionChunk {
|
||||||
|
choices: { delta: { content?: string } }[];
|
||||||
|
}
|
||||||
|
interface CompletionResponse {
|
||||||
|
choices: { message: { content: string } }[];
|
||||||
|
}
|
||||||
|
interface ChatCompletions {
|
||||||
|
create(req: {
|
||||||
|
messages: ChatMessage[];
|
||||||
|
stream: false;
|
||||||
|
max_tokens: number;
|
||||||
|
}): Promise<CompletionResponse>;
|
||||||
|
create(req: {
|
||||||
|
messages: ChatMessage[];
|
||||||
|
stream: true;
|
||||||
|
max_tokens: number;
|
||||||
|
}): Promise<AsyncIterable<CompletionChunk>>;
|
||||||
|
}
|
||||||
|
interface MLCEngine {
|
||||||
|
chat: { completions: ChatCompletions };
|
||||||
|
}
|
||||||
|
interface WebLlmModule {
|
||||||
|
CreateMLCEngine(
|
||||||
|
model: string,
|
||||||
|
opts?: { initProgressCallback?: (p: InitProgressReport) => void },
|
||||||
|
): Promise<MLCEngine>;
|
||||||
|
}
|
||||||
|
|
||||||
|
let libPromise: Promise<WebLlmModule> | null = null;
|
||||||
|
function lib(): Promise<WebLlmModule> {
|
||||||
|
if (!libPromise) libPromise = runtimeImport(WEBLLM_CDN);
|
||||||
|
return libPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
let engineInstance: MLCEngine | null = null;
|
||||||
|
|
||||||
|
async function hasWebGpu(): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
if (typeof navigator === 'undefined' || !('gpu' in navigator)) return false;
|
||||||
|
const gpu = (navigator as { gpu?: { requestAdapter(): Promise<unknown> } }).gpu;
|
||||||
|
const adapter = await gpu?.requestAdapter();
|
||||||
|
return adapter != null;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const webllm: GenerationEngine = {
|
||||||
|
kind: 'webllm',
|
||||||
|
label: 'On-device (Qwen2.5-1.5B)',
|
||||||
|
|
||||||
|
async isAvailable(): Promise<boolean> {
|
||||||
|
return hasWebGpu();
|
||||||
|
},
|
||||||
|
|
||||||
|
async loadModel(onProgress?: (p: number) => void): Promise<void> {
|
||||||
|
if (engineInstance) return; // idempotent
|
||||||
|
const { CreateMLCEngine } = await lib();
|
||||||
|
engineInstance = await CreateMLCEngine(WEBLLM_MODEL, {
|
||||||
|
initProgressCallback: (p) => onProgress?.(p.progress),
|
||||||
|
});
|
||||||
|
},
|
||||||
|
|
||||||
|
isLoaded(): boolean {
|
||||||
|
return engineInstance != null;
|
||||||
|
},
|
||||||
|
|
||||||
|
async generate(prompt: string, opts?: GenOptions): Promise<string> {
|
||||||
|
if (!engineInstance) {
|
||||||
|
throw new Error('On-device model is not loaded; call loadModel() first.');
|
||||||
|
}
|
||||||
|
const messages: ChatMessage[] = [];
|
||||||
|
if (opts?.system) messages.push({ role: 'system', content: opts.system });
|
||||||
|
messages.push({ role: 'user', content: prompt });
|
||||||
|
|
||||||
|
const maxTokens = opts?.maxTokens ?? 512;
|
||||||
|
|
||||||
|
if (opts?.onToken) {
|
||||||
|
// Stream: surface each delta and accumulate the full text to return.
|
||||||
|
const stream = await engineInstance.chat.completions.create({
|
||||||
|
messages,
|
||||||
|
stream: true,
|
||||||
|
max_tokens: maxTokens,
|
||||||
|
});
|
||||||
|
let full = '';
|
||||||
|
for await (const chunk of stream) {
|
||||||
|
const delta = chunk.choices[0]?.delta?.content ?? '';
|
||||||
|
if (delta) {
|
||||||
|
full += delta;
|
||||||
|
opts.onToken(delta);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return full;
|
||||||
|
}
|
||||||
|
|
||||||
|
const res = await engineInstance.chat.completions.create({
|
||||||
|
messages,
|
||||||
|
stream: false,
|
||||||
|
max_tokens: maxTokens,
|
||||||
|
});
|
||||||
|
return res.choices[0]?.message?.content ?? '';
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
// Public entry point for the optional generative ("ask your lectures") layer.
|
||||||
|
//
|
||||||
|
// `./engineImpl` is resolved by Metro to engineImpl.web.ts or
|
||||||
|
// engineImpl.native.ts by platform extension; the base engineImpl.ts re-export
|
||||||
|
// (web) is what TypeScript resolves for typechecking. Consumers call
|
||||||
|
// getGenerationEngine() and stay platform-agnostic.
|
||||||
|
//
|
||||||
|
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||||
|
|
||||||
|
import type { CloudConfig, GenerationEngine } from './engine';
|
||||||
|
import { createCloudEngine } from './cloud';
|
||||||
|
import { webllm } from './engineImpl';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An engine that does nothing useful — represents "no generation backend here".
|
||||||
|
* isAvailable() is always false; generate()/loadModel() throw. Callers should
|
||||||
|
* check isAvailable() and fall back to the search-only path (no fake answers).
|
||||||
|
*/
|
||||||
|
export const noneEngine: GenerationEngine = {
|
||||||
|
kind: 'none',
|
||||||
|
label: 'No model',
|
||||||
|
async isAvailable(): Promise<boolean> {
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
async loadModel(): Promise<void> {
|
||||||
|
throw new Error('No generation engine is available.');
|
||||||
|
},
|
||||||
|
isLoaded(): boolean {
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
async generate(): Promise<string> {
|
||||||
|
throw new Error('No generation engine is available.');
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pick the generation engine.
|
||||||
|
*
|
||||||
|
* - If a cloud config WITH an apiKey is given, use the BYO-key cloud engine.
|
||||||
|
* - Otherwise return the platform on-device (webllm) engine. On a device without
|
||||||
|
* WebGPU (web) or on native, that engine's isAvailable() resolves false, which
|
||||||
|
* is how the "none" state is represented in practice — callers MUST check
|
||||||
|
* isAvailable() before loading/generating, and fall back to search-only.
|
||||||
|
*/
|
||||||
|
export function getGenerationEngine(cloud?: CloudConfig): GenerationEngine {
|
||||||
|
if (cloud?.apiKey) return createCloudEngine(cloud);
|
||||||
|
return webllm;
|
||||||
|
}
|
||||||
|
|
||||||
|
export { createCloudEngine, webllm };
|
||||||
|
export * from './engine';
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
import { describe, it, expect } from 'vitest';
|
||||||
|
import { buildRagPrompt } from './prompt';
|
||||||
|
|
||||||
|
describe('buildRagPrompt', () => {
|
||||||
|
it('numbers excerpts 1-based with formatted times and includes the question', () => {
|
||||||
|
const { system, prompt } = buildRagPrompt('What is entropy?', [
|
||||||
|
{ text: 'Entropy measures disorder.', start: 65, transcriptId: 't1' },
|
||||||
|
{ text: 'It always increases in a closed system.', start: 130, transcriptId: 't1' },
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Excerpts are numbered and carry a mm:ss timecode from `start`.
|
||||||
|
expect(prompt).toContain('[1] (1:05) Entropy measures disorder.');
|
||||||
|
expect(prompt).toContain('[2] (2:10) It always increases in a closed system.');
|
||||||
|
// Question is present.
|
||||||
|
expect(prompt).toContain('Question: What is entropy?');
|
||||||
|
// System prompt enforces grounding + citations.
|
||||||
|
expect(system).toMatch(/only/i);
|
||||||
|
expect(system).toMatch(/\[1\]|square brackets/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('trims the question', () => {
|
||||||
|
const { prompt } = buildRagPrompt(' hello? ', [
|
||||||
|
{ text: 'hi', start: 0, transcriptId: 't1' },
|
||||||
|
]);
|
||||||
|
expect(prompt).toContain('Question: hello?');
|
||||||
|
expect(prompt).not.toContain('Question: hello?');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('is deterministic for identical inputs', () => {
|
||||||
|
const snips = [{ text: 'a', start: 5, transcriptId: 't1' }];
|
||||||
|
const a = buildRagPrompt('q', snips);
|
||||||
|
const b = buildRagPrompt('q', snips);
|
||||||
|
expect(a).toEqual(b);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('yields a refusal-style prompt with no snippets (no excerpts -> admit not found)', () => {
|
||||||
|
const { system, prompt } = buildRagPrompt('Anything?', []);
|
||||||
|
expect(prompt).toContain('(no lecture excerpts were found)');
|
||||||
|
expect(prompt).toContain('Question: Anything?');
|
||||||
|
// Still must not contain a fabricated [1] excerpt.
|
||||||
|
expect(prompt).not.toMatch(/^\[1\]/m);
|
||||||
|
// The system rule that drives the refusal is present.
|
||||||
|
expect(system).toMatch(/could not find it in the lectures/i);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
// PURE, deterministic RAG prompt builder for the optional "ask your lectures"
|
||||||
|
// generative layer (Phase 5). No I/O, no platform deps — just string assembly,
|
||||||
|
// so it is fully unit-testable and produces byte-identical output for identical
|
||||||
|
// inputs.
|
||||||
|
//
|
||||||
|
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||||
|
|
||||||
|
import { formatClock } from '../format';
|
||||||
|
|
||||||
|
/** A retrieved lecture snippet to ground the answer in. */
|
||||||
|
export interface PromptSnippet {
|
||||||
|
text: string;
|
||||||
|
/** Snippet start, in seconds (rendered as mm:ss in the prompt). */
|
||||||
|
start: number;
|
||||||
|
transcriptId: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the system + user messages for a grounded, cited RAG answer.
|
||||||
|
*
|
||||||
|
* The system prompt forces the model to answer ONLY from the supplied excerpts,
|
||||||
|
* to admit when the excerpts don't cover the question (no hallucinated answers),
|
||||||
|
* to stay concise, and to cite excerpt numbers like [1], [2].
|
||||||
|
*
|
||||||
|
* The user prompt lists the excerpts as `[i] (mm:ss) text` (1-based), then the
|
||||||
|
* question. With zero snippets the excerpt block is replaced by an explicit
|
||||||
|
* "(no lecture excerpts were found)" marker, which — combined with the grounding
|
||||||
|
* rule — steers the model to a refusal rather than an invented answer.
|
||||||
|
*
|
||||||
|
* Deterministic: same inputs -> same output.
|
||||||
|
*/
|
||||||
|
export function buildRagPrompt(
|
||||||
|
question: string,
|
||||||
|
snippets: PromptSnippet[],
|
||||||
|
): { system: string; prompt: string } {
|
||||||
|
const system = [
|
||||||
|
'You are a study assistant that answers questions about a student\'s lecture recordings.',
|
||||||
|
'Answer ONLY using the numbered lecture excerpts provided below.',
|
||||||
|
'If the excerpts do not contain the answer, say you could not find it in the lectures — do not use outside knowledge and do not guess.',
|
||||||
|
'Be concise.',
|
||||||
|
'Cite the excerpts you used by their number in square brackets, like [1] or [2][3].',
|
||||||
|
].join(' ');
|
||||||
|
|
||||||
|
const q = question.trim();
|
||||||
|
|
||||||
|
const excerptBlock =
|
||||||
|
snippets.length === 0
|
||||||
|
? '(no lecture excerpts were found)'
|
||||||
|
: snippets
|
||||||
|
.map((s, i) => `[${i + 1}] (${formatClock(s.start)}) ${s.text.trim()}`)
|
||||||
|
.join('\n');
|
||||||
|
|
||||||
|
const prompt = `Lecture excerpts:\n${excerptBlock}\n\nQuestion: ${q}`;
|
||||||
|
|
||||||
|
return { system, prompt };
|
||||||
|
}
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
// RAG orchestrator for the optional "ask your lectures" feature (Phase 5).
|
||||||
|
//
|
||||||
|
// Flow: retrieve the most relevant lecture moments with the existing on-device
|
||||||
|
// semantic search, then (only if a generation engine is available) ask the model
|
||||||
|
// to write a grounded, cited answer from those moments. The retrieved moments are
|
||||||
|
// ALWAYS returned as citations so the UI can show them verbatim.
|
||||||
|
//
|
||||||
|
// Hard rules honoured here:
|
||||||
|
// - Opt-in + gated: if no engine is available we return the search hits as a
|
||||||
|
// "search-only" fallback (generated:false) — never a fake/ungrounded answer.
|
||||||
|
// - Privacy: we only ever pass the question + retrieved snippets to the engine
|
||||||
|
// (the cloud path is OpenAI-compatible BYO-key); raw audio/transcripts never
|
||||||
|
// leave the device.
|
||||||
|
// - Defensive: any generation failure degrades to the search-only fallback so
|
||||||
|
// the user still gets the moments.
|
||||||
|
//
|
||||||
|
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||||
|
|
||||||
|
import { searchLectures } from '../search/search';
|
||||||
|
import type { GenerationEngine, RagAnswer, RagCitation } from './engine';
|
||||||
|
import { buildRagPrompt } from './prompt';
|
||||||
|
|
||||||
|
/** How many lecture moments to retrieve and feed the model. */
|
||||||
|
const RAG_LIMIT = 6;
|
||||||
|
|
||||||
|
const EMPTY: RagAnswer = {
|
||||||
|
answer: '',
|
||||||
|
citations: [],
|
||||||
|
grounded: false,
|
||||||
|
generated: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Answer `question` from the user's lectures, grounded in retrieved moments.
|
||||||
|
*
|
||||||
|
* - Empty/whitespace question -> empty, ungrounded result.
|
||||||
|
* - No relevant moments retrieved -> empty, ungrounded result (we refuse to
|
||||||
|
* answer with nothing to cite).
|
||||||
|
* - Engine unavailable / generation fails -> the retrieved moments as a
|
||||||
|
* search-only fallback (grounded:true, generated:false).
|
||||||
|
* - Otherwise -> the model's grounded, cited answer (generated:true).
|
||||||
|
*/
|
||||||
|
export async function askLectures(
|
||||||
|
question: string,
|
||||||
|
engine: GenerationEngine,
|
||||||
|
opts?: { courseId?: string | null; signal?: AbortSignal },
|
||||||
|
): Promise<RagAnswer> {
|
||||||
|
const q = question.trim();
|
||||||
|
if (q.length === 0) return EMPTY;
|
||||||
|
|
||||||
|
const hits = await searchLectures(q, {
|
||||||
|
courseId: opts?.courseId,
|
||||||
|
limit: RAG_LIMIT,
|
||||||
|
});
|
||||||
|
if (hits.length === 0) return EMPTY;
|
||||||
|
|
||||||
|
// The retrieved moments are shown verbatim regardless of whether we generate.
|
||||||
|
const citations: RagCitation[] = hits.map((h) => ({
|
||||||
|
transcriptId: h.transcriptId,
|
||||||
|
segmentId: h.segmentId,
|
||||||
|
start: h.start,
|
||||||
|
text: h.text,
|
||||||
|
score: h.score,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Search-only fallback: no engine here right now (no WebGPU / no key set).
|
||||||
|
if (!(await engine.isAvailable())) {
|
||||||
|
return { answer: '', citations, grounded: true, generated: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (!engine.isLoaded()) await engine.loadModel();
|
||||||
|
const { system, prompt } = buildRagPrompt(
|
||||||
|
q,
|
||||||
|
hits.map((h) => ({
|
||||||
|
text: h.text,
|
||||||
|
start: h.start,
|
||||||
|
transcriptId: h.transcriptId,
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
const answer = await engine.generate(prompt, {
|
||||||
|
system,
|
||||||
|
maxTokens: 512,
|
||||||
|
signal: opts?.signal,
|
||||||
|
});
|
||||||
|
return { answer, citations, grounded: true, generated: true };
|
||||||
|
} catch {
|
||||||
|
// Any failure (model load, generation, abort) -> still give the moments.
|
||||||
|
return { answer: '', citations, grounded: true, generated: false };
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,159 @@
|
|||||||
|
// Session/UI state for the optional generative ("ask your lectures") layer
|
||||||
|
// (Phase 5). Holds the BYO-key cloud config (if any), the on-device model load
|
||||||
|
// status, and which backend is selected, and exposes a single ask() that wires
|
||||||
|
// retrieval -> generation via askLectures.
|
||||||
|
//
|
||||||
|
// The feature is GATED BY AVAILABILITY, not a manual switch: `enabled` defaults
|
||||||
|
// true, but ask() still degrades to the search-only fallback when no engine is
|
||||||
|
// available (no WebGPU and no cloud key). Raw audio/transcripts never leave the
|
||||||
|
// device — only the question + retrieved snippets do (cloud path).
|
||||||
|
//
|
||||||
|
// Persistence: on web the cloud config is saved to localStorage under
|
||||||
|
// 'wisp:ai'; native keeps it in-memory (no localStorage). Everything here is
|
||||||
|
// defensive — refreshAvailability never throws.
|
||||||
|
import { create } from 'zustand';
|
||||||
|
|
||||||
|
import {
|
||||||
|
getGenerationEngine,
|
||||||
|
type CloudConfig,
|
||||||
|
type GenerationEngine,
|
||||||
|
type RagAnswer,
|
||||||
|
} from '@/lib/generation';
|
||||||
|
import { askLectures } from '@/lib/generation/rag';
|
||||||
|
|
||||||
|
const STORAGE_KEY = 'wisp:ai';
|
||||||
|
|
||||||
|
type ModelStatus = 'idle' | 'loading' | 'ready';
|
||||||
|
type EngineKind = 'webllm' | 'cloud' | 'none';
|
||||||
|
|
||||||
|
interface PersistedState {
|
||||||
|
enabled: boolean;
|
||||||
|
cloud?: CloudConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Web-only localStorage handle (undefined on native / SSR). */
|
||||||
|
function ls(): Storage | undefined {
|
||||||
|
try {
|
||||||
|
return typeof localStorage !== 'undefined' ? localStorage : undefined;
|
||||||
|
} catch {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Read persisted enabled + cloud config from localStorage (web only). */
|
||||||
|
function loadPersisted(): PersistedState {
|
||||||
|
const store = ls();
|
||||||
|
if (!store) return { enabled: true };
|
||||||
|
try {
|
||||||
|
const raw = store.getItem(STORAGE_KEY);
|
||||||
|
if (!raw) return { enabled: true };
|
||||||
|
const parsed = JSON.parse(raw) as Partial<PersistedState>;
|
||||||
|
return {
|
||||||
|
enabled: parsed.enabled ?? true,
|
||||||
|
cloud: parsed.cloud,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return { enabled: true };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Persist (or clear) the cloud config + enabled flag on web. No-op on native. */
|
||||||
|
function persist(state: PersistedState): void {
|
||||||
|
const store = ls();
|
||||||
|
if (!store) return;
|
||||||
|
try {
|
||||||
|
store.setItem(STORAGE_KEY, JSON.stringify(state));
|
||||||
|
} catch {
|
||||||
|
/* quota / private mode — ignore, just stay in-memory */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const initial = loadPersisted();
|
||||||
|
|
||||||
|
interface AiState {
|
||||||
|
/** Feature flag — defaults true; real gating is via availability. */
|
||||||
|
enabled: boolean;
|
||||||
|
/** BYO-key cloud config, if the user supplied one. */
|
||||||
|
cloud?: CloudConfig;
|
||||||
|
/** On-device model load status (WebLLM). */
|
||||||
|
modelStatus: ModelStatus;
|
||||||
|
/** WebLLM download progress in [0,1]. */
|
||||||
|
progress: number;
|
||||||
|
/** Which backend the current config resolves to. */
|
||||||
|
engineKind: EngineKind;
|
||||||
|
|
||||||
|
/** Set (or clear, with no arg) the cloud config; persists on web. */
|
||||||
|
setCloud: (cfg?: CloudConfig) => void;
|
||||||
|
/** Recompute engineKind and probe availability. Never throws. */
|
||||||
|
refreshAvailability: () => Promise<void>;
|
||||||
|
/** Resolve an engine, loading the on-device model if needed. */
|
||||||
|
ensureReady: () => Promise<GenerationEngine>;
|
||||||
|
/** Retrieve + (optionally) generate a grounded, cited answer. */
|
||||||
|
ask: (
|
||||||
|
question: string,
|
||||||
|
opts?: { courseId?: string | null; signal?: AbortSignal },
|
||||||
|
) => Promise<RagAnswer>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const useAi = create<AiState>((set, get) => ({
|
||||||
|
enabled: initial.enabled,
|
||||||
|
cloud: initial.cloud,
|
||||||
|
modelStatus: 'idle',
|
||||||
|
progress: 0,
|
||||||
|
// Resolve the initial backend synchronously from the persisted config.
|
||||||
|
engineKind: getGenerationEngine(initial.cloud).kind,
|
||||||
|
|
||||||
|
setCloud: (cfg) => {
|
||||||
|
persist({ enabled: get().enabled, cloud: cfg });
|
||||||
|
// A cloud config swap invalidates any in-progress on-device load state.
|
||||||
|
set({
|
||||||
|
cloud: cfg,
|
||||||
|
engineKind: getGenerationEngine(cfg).kind,
|
||||||
|
modelStatus: 'idle',
|
||||||
|
progress: 0,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
|
||||||
|
refreshAvailability: async () => {
|
||||||
|
try {
|
||||||
|
const engine = getGenerationEngine(get().cloud);
|
||||||
|
set({ engineKind: engine.kind });
|
||||||
|
// Probe availability (WebGPU adapter / configured key). The result isn't
|
||||||
|
// stored beyond engineKind; ask()/ensureReady() re-check at call time.
|
||||||
|
await engine.isAvailable();
|
||||||
|
} catch {
|
||||||
|
// Stay defensive: never throw out of availability refresh.
|
||||||
|
set({ engineKind: 'none' });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
ensureReady: async () => {
|
||||||
|
const engine = getGenerationEngine(get().cloud);
|
||||||
|
set({ engineKind: engine.kind });
|
||||||
|
// Only the on-device WebLLM engine has a model to download — and only when
|
||||||
|
// it can actually run here (WebGPU). On a machine without WebGPU we skip the
|
||||||
|
// (doomed, ~1GB) load and let askLectures degrade to search-only.
|
||||||
|
if (
|
||||||
|
engine.kind === 'webllm' &&
|
||||||
|
!engine.isLoaded() &&
|
||||||
|
(await engine.isAvailable())
|
||||||
|
) {
|
||||||
|
set({ modelStatus: 'loading', progress: 0 });
|
||||||
|
try {
|
||||||
|
await engine.loadModel((p) => set({ progress: p }));
|
||||||
|
set({ modelStatus: 'ready', progress: 1 });
|
||||||
|
} catch (err) {
|
||||||
|
set({ modelStatus: 'idle', progress: 0 });
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
set({ modelStatus: 'ready' });
|
||||||
|
}
|
||||||
|
return engine;
|
||||||
|
},
|
||||||
|
|
||||||
|
ask: async (question, opts) => {
|
||||||
|
const engine = await get().ensureReady();
|
||||||
|
return askLectures(question, engine, opts);
|
||||||
|
},
|
||||||
|
}));
|
||||||
Reference in New Issue
Block a user