feat(phase5): optional generative RAG — ask your lectures, with citations
CI / test (push) Successful in 20s
CI / build-apk (push) Has been skipped
CI / deploy-web (push) Successful in 33s

Strictly opt-in, gated, with the deterministic features as the always-present floor:
- GenerationEngine: WebLLM (Qwen2.5-1.5B, WebGPU, CDN-loaded) + BYO-key cloud
  (OpenAI-compatible); native stub. Pure grounding prompt builder (4 tests).
- rag.askLectures: retrieve Phase-1 hits -> grounded prompt -> answer with
  citations; refuses when nothing relevant; falls back to search-only when no
  engine is available. Never sends raw audio/transcripts — only question + snippets.
- aiStore (BYO key persisted in localStorage on web), Ask screen (answer +
  tappable citation chips that jump to the audio + honest "verify" disclaimer),
  Settings AI section (engine status + bring-your-own-key form).

279 tests green, 0 tsc errors, web export builds. ROADMAP phases 0-5 complete.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-14 15:56:57 +02:00
parent 97aee3e4b3
commit 7611ada001
17 changed files with 1752 additions and 12 deletions
+497 -9
View File
File diff suppressed because it is too large Load Diff
+10
View File
@@ -0,0 +1,10 @@
// https://docs.expo.dev/guides/using-eslint/
const { defineConfig } = require('eslint/config');
const expoConfig = require("eslint-config-expo/flat");
module.exports = defineConfig([
expoConfig,
{
ignores: ["dist/*"],
}
]);
+2
View File
@@ -38,6 +38,8 @@
"devDependencies": { "devDependencies": {
"@types/node": "^25.9.3", "@types/node": "^25.9.3",
"@types/react": "~19.2.2", "@types/react": "~19.2.2",
"eslint": "^9.0.0",
"eslint-config-expo": "~56.0.4",
"fake-indexeddb": "^6.2.5", "fake-indexeddb": "^6.2.5",
"fast-check": "^4.8.0", "fast-check": "^4.8.0",
"typescript": "~6.0.3", "typescript": "~6.0.3",
+1
View File
@@ -18,6 +18,7 @@ export default function RootLayout() {
<Stack.Screen name="record" options={{ title: 'Record' }} /> <Stack.Screen name="record" options={{ title: 'Record' }} />
<Stack.Screen name="transcript/[id]" options={{ title: 'Transcript' }} /> <Stack.Screen name="transcript/[id]" options={{ title: 'Transcript' }} />
<Stack.Screen name="search" options={{ title: 'Search' }} /> <Stack.Screen name="search" options={{ title: 'Search' }} />
<Stack.Screen name="ask" options={{ title: 'Ask' }} />
<Stack.Screen name="courses" options={{ title: 'Courses' }} /> <Stack.Screen name="courses" options={{ title: 'Courses' }} />
<Stack.Screen name="study" options={{ title: 'Study' }} /> <Stack.Screen name="study" options={{ title: 'Study' }} />
<Stack.Screen name="quiz" options={{ title: 'Quiz' }} /> <Stack.Screen name="quiz" options={{ title: 'Quiz' }} />
+315
View File
@@ -0,0 +1,315 @@
import { Stack, useFocusEffect, useRouter } from 'expo-router';
import { useCallback, useState } from 'react';
import {
ActivityIndicator,
Pressable,
ScrollView,
StyleSheet,
TextInput,
View,
} from 'react-native';
import { ThemedText } from '@/components/themed-text';
import { ThemedView } from '@/components/themed-view';
import { MaxContentWidth, Spacing } from '@/constants/theme';
import { useTheme } from '@/hooks/use-theme';
import type { RagAnswer, RagCitation } from '@/lib/generation/engine';
import { formatClock } from '@/lib/format';
import { useCourses } from '@/stores/coursesStore';
import { useAi } from '@/stores/aiStore';
import { useEmbedding } from '@/stores/embeddingStore';
const ACCENT = '#3c87f7';
// Course filter is a course id, or `null` for "all courses".
type CourseSel = string | null;
export default function AskScreen() {
const theme = useTheme();
const router = useRouter();
const courses = useCourses((s) => s.items);
const refreshCourses = useCourses((s) => s.refresh);
// Whether any lectures have been indexed at all (used to nudge the user to
// build the search index when an empty/ungrounded answer comes back).
const pending = useEmbedding((s) => s.pending);
const refreshPending = useEmbedding((s) => s.refreshPending);
// AI engine state (model download / readiness) is observed from the store so
// the loading bar reflects an in-flight on-device model download.
const modelStatus = useAi((s) => s.modelStatus);
const progress = useAi((s) => s.progress);
const [question, setQuestion] = useState('');
const [courseId, setCourseId] = useState<CourseSel>(null);
const [answering, setAnswering] = useState(false);
const [error, setError] = useState<string | null>(null);
const [result, setResult] = useState<RagAnswer | null>(null);
useFocusEffect(
useCallback(() => {
void refreshCourses();
void refreshPending();
}, [refreshCourses, refreshPending]),
);
const onAsk = useCallback(async () => {
const q = question.trim();
if (!q || answering) return;
setAnswering(true);
setError(null);
setResult(null);
try {
const ans = await useAi.getState().ask(q, { courseId });
setResult(ans);
} catch (e) {
setError(e instanceof Error ? e.message : 'Something went wrong.');
} finally {
setAnswering(false);
}
}, [question, courseId, answering]);
const openCitation = (c: RagCitation) =>
router.push({
pathname: '/transcript/[id]',
params: { id: c.transcriptId, t: String(Math.floor(c.start)) },
});
// The model is downloading/preparing on-device (WebLLM). Show a progress bar.
const loadingModel = modelStatus === 'loading';
return (
<ThemedView style={styles.fill}>
<Stack.Screen options={{ title: 'Ask' }} />
<ScrollView contentContainerStyle={styles.content} keyboardShouldPersistTaps="handled">
<ThemedText type="small" themeColor="textSecondary">
Ask a question and get an answer grounded in your lectures every claim links back to the
moment it came from.
</ThemedText>
{courses.length > 0 && (
<ScrollView
horizontal
showsHorizontalScrollIndicator={false}
contentContainerStyle={styles.filterBar}>
<FilterChip label="All courses" active={courseId === null} onPress={() => setCourseId(null)} />
{courses.map((c) => (
<FilterChip
key={c.id}
label={c.name}
active={courseId === c.id}
onPress={() => setCourseId(c.id)}
/>
))}
</ScrollView>
)}
<TextInput
value={question}
onChangeText={setQuestion}
onSubmitEditing={() => void onAsk()}
returnKeyType="search"
autoFocus
multiline
placeholder="Ask your lectures anything…"
placeholderTextColor={theme.textSecondary}
style={[styles.input, { color: theme.text, backgroundColor: theme.backgroundElement }]}
/>
<Pressable
onPress={() => void onAsk()}
disabled={answering || question.trim() === ''}
style={({ pressed }) => [
styles.askBtn,
{ opacity: answering || question.trim() === '' ? 0.5 : pressed ? 0.85 : 1 },
]}>
<ThemedText style={styles.askBtnText}>Ask</ThemedText>
</Pressable>
{loadingModel && (
<ThemedView type="backgroundElement" style={styles.card}>
<ThemedText type="smallBold">Preparing on-device AI {Math.round(progress * 100)}%</ThemedText>
<ProgressBar value={progress} />
<ThemedText type="small" themeColor="textSecondary">
The model downloads once, then runs locally on your device.
</ThemedText>
</ThemedView>
)}
{answering && !loadingModel && (
<View style={styles.center}>
<ActivityIndicator />
<ThemedText type="small" themeColor="textSecondary" style={styles.centerText}>
Thinking
</ThemedText>
</View>
)}
{error && (
<ThemedView type="backgroundElement" style={styles.card}>
<ThemedText type="smallBold">Couldn&apos;t answer that</ThemedText>
<ThemedText type="small" themeColor="textSecondary">
{error}
</ThemedText>
</ThemedView>
)}
{result && !answering && (
<Answer
result={result}
indexEmpty={pending > 0}
onOpenCitation={openCitation}
/>
)}
</ScrollView>
</ThemedView>
);
}
function Answer({
result,
indexEmpty,
onOpenCitation,
}: {
result: RagAnswer;
indexEmpty: boolean;
onOpenCitation: (c: RagCitation) => void;
}) {
// Nothing relevant was retrieved — we refuse to invent an answer.
if (!result.grounded) {
return (
<ThemedView type="backgroundElement" style={styles.card}>
<ThemedText type="smallBold">
I couldn&apos;t find anything about that in your lectures.
</ThemedText>
{indexEmpty && (
<ThemedText type="small" themeColor="textSecondary">
Tip: build the search index on the Search screen so your lectures become searchable.
</ThemedText>
)}
</ThemedView>
);
}
return (
<View style={styles.answerWrap}>
{result.generated ? (
<ThemedView type="backgroundElement" style={styles.answerCard}>
<ThemedText type="default">{result.answer}</ThemedText>
</ThemedView>
) : (
// Grounded but no LLM available: deterministic search-only fallback.
<ThemedView type="backgroundElement" style={styles.card}>
<ThemedText type="small" themeColor="textSecondary">
On-device AI needs WebGPU, or add an API key in Settings here are the matching moments:
</ThemedText>
</ThemedView>
)}
<ThemedText type="smallBold" style={styles.sourcesHeading}>
Sources
</ThemedText>
{result.citations.map((c, i) => (
<CitationChip
key={`${c.transcriptId}:${c.segmentId ?? i}`}
citation={c}
onPress={() => onOpenCitation(c)}
/>
))}
<ThemedText type="small" themeColor="textSecondary" style={styles.disclaimer}>
AI answers can be wrong every claim links to the lecture; verify against the source.
</ThemedText>
</View>
);
}
function CitationChip({
citation,
onPress,
}: {
citation: RagCitation;
onPress: () => void;
}) {
return (
<Pressable onPress={onPress} style={({ pressed }) => [pressed && styles.pressed]}>
<ThemedView type="backgroundElement" style={styles.card}>
<ThemedText type="small" numberOfLines={3}>
{citation.text}
</ThemedText>
<ThemedText type="small" style={styles.citationTime}>
({formatClock(citation.start)})
</ThemedText>
</ThemedView>
</Pressable>
);
}
function FilterChip({
label,
active,
onPress,
}: {
label: string;
active: boolean;
onPress: () => void;
}) {
const theme = useTheme();
return (
<Pressable
onPress={onPress}
style={[styles.filterChip, { backgroundColor: active ? ACCENT : theme.backgroundElement }]}>
<ThemedText type="small" style={active ? styles.chipActive : undefined}>
{label}
</ThemedText>
</Pressable>
);
}
function ProgressBar({ value }: { value: number }) {
return (
<View style={styles.track}>
<View style={[styles.bar, { width: `${Math.max(2, Math.min(100, value * 100))}%` }]} />
</View>
);
}
const styles = StyleSheet.create({
fill: { flex: 1 },
content: {
padding: Spacing.three,
gap: Spacing.three,
maxWidth: MaxContentWidth,
width: '100%',
alignSelf: 'center',
},
filterBar: { gap: Spacing.two, paddingVertical: Spacing.one, paddingRight: Spacing.three },
filterChip: { paddingHorizontal: Spacing.three, paddingVertical: Spacing.one, borderRadius: 999 },
chipActive: { color: '#fff', fontWeight: '700' },
input: {
borderRadius: Spacing.two,
paddingHorizontal: Spacing.three,
paddingVertical: Spacing.two,
fontSize: 15,
minHeight: 48,
},
askBtn: {
backgroundColor: ACCENT,
paddingVertical: Spacing.three,
borderRadius: Spacing.three,
alignItems: 'center',
},
askBtnText: { color: '#fff', fontWeight: '700', fontSize: 16 },
card: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.two },
answerWrap: { gap: Spacing.three },
answerCard: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.two },
sourcesHeading: { marginTop: Spacing.one },
citationTime: { color: ACCENT, fontWeight: '700' },
disclaimer: { marginTop: Spacing.one, fontStyle: 'italic' },
center: { alignItems: 'center', gap: Spacing.two, paddingVertical: Spacing.four },
centerText: { textAlign: 'center' },
track: { height: 6, borderRadius: 3, backgroundColor: '#88888833', overflow: 'hidden' },
bar: { height: 6, borderRadius: 3, backgroundColor: ACCENT },
pressed: { opacity: 0.7 },
});
+5
View File
@@ -87,6 +87,11 @@ export default function LibraryScreen() {
<ThemedText type="link" themeColor="textSecondary">Search</ThemedText> <ThemedText type="link" themeColor="textSecondary">Search</ThemedText>
</Pressable> </Pressable>
</Link> </Link>
<Link href="/ask" asChild>
<Pressable hitSlop={8}>
<ThemedText type="link" themeColor="textSecondary">Ask</ThemedText>
</Pressable>
</Link>
<Link href="/courses" asChild> <Link href="/courses" asChild>
<Pressable hitSlop={8}> <Pressable hitSlop={8}>
<ThemedText type="link" themeColor="textSecondary">Courses</ThemedText> <ThemedText type="link" themeColor="textSecondary">Courses</ThemedText>
+189 -3
View File
@@ -1,12 +1,18 @@
import { ScrollView, StyleSheet, Pressable, View } from 'react-native'; import { useEffect, useState } from 'react';
import { ScrollView, StyleSheet, Pressable, TextInput, View } from 'react-native';
import { ThemedText } from '@/components/themed-text'; import { ThemedText } from '@/components/themed-text';
import { ThemedView } from '@/components/themed-view'; import { ThemedView } from '@/components/themed-view';
import { MaxContentWidth, Spacing } from '@/constants/theme'; import { MaxContentWidth, Spacing } from '@/constants/theme';
import { useTheme } from '@/hooks/use-theme'; import { useTheme } from '@/hooks/use-theme';
import { listModels } from '@/lib/models/catalog'; import { listModels } from '@/lib/models/catalog';
import { useAi } from '@/stores/aiStore';
import { useTranscribe } from '@/stores/transcribeStore'; import { useTranscribe } from '@/stores/transcribeStore';
const ACCENT = '#3c87f7';
const DEFAULT_BASE_URL = 'https://api.openai.com/v1';
const DEFAULT_MODEL = 'gpt-4o-mini';
export default function SettingsScreen() { export default function SettingsScreen() {
const theme = useTheme(); const theme = useTheme();
const modelId = useTranscribe((s) => s.modelId); const modelId = useTranscribe((s) => s.modelId);
@@ -28,10 +34,10 @@ export default function SettingsScreen() {
<Pressable key={m.id} onPress={() => setModel(m.id)}> <Pressable key={m.id} onPress={() => setModel(m.id)}>
<ThemedView <ThemedView
type={selected ? 'backgroundSelected' : 'backgroundElement'} type={selected ? 'backgroundSelected' : 'backgroundElement'}
style={[styles.card, selected && { borderColor: '#3c87f7', borderWidth: 1 }]}> style={[styles.card, selected && { borderColor: ACCENT, borderWidth: 1 }]}>
<View style={styles.rowBetween}> <View style={styles.rowBetween}>
<ThemedText type="smallBold">{m.label}</ThemedText> <ThemedText type="smallBold">{m.label}</ThemedText>
{selected && <ThemedText type="small" style={{ color: '#3c87f7' }}> selected</ThemedText>} {selected && <ThemedText type="small" style={{ color: ACCENT }}> selected</ThemedText>}
</View> </View>
<ThemedText type="small" themeColor="textSecondary"> <ThemedText type="small" themeColor="textSecondary">
{cap(m.tier)} · ~{m.approxMB} MB · {m.multilingual ? 'multilingual' : 'English-only'} {cap(m.tier)} · ~{m.approxMB} MB · {m.multilingual ? 'multilingual' : 'English-only'}
@@ -41,6 +47,9 @@ export default function SettingsScreen() {
); );
})} })}
<View style={styles.spacer} />
<AiSection />
<View style={styles.spacer} /> <View style={styles.spacer} />
<ThemedText type="subtitle">Privacy</ThemedText> <ThemedText type="subtitle">Privacy</ThemedText>
<ThemedText type="small" themeColor="textSecondary"> <ThemedText type="small" themeColor="textSecondary">
@@ -53,6 +62,159 @@ export default function SettingsScreen() {
); );
} }
function AiSection() {
const theme = useTheme();
const cloud = useAi((s) => s.cloud);
const engineKind = useAi((s) => s.engineKind);
const setCloud = useAi((s) => s.setCloud);
const refreshAvailability = useAi((s) => s.refreshAvailability);
// Form fields seeded from any saved cloud config.
const [baseUrl, setBaseUrl] = useState(cloud?.baseUrl ?? DEFAULT_BASE_URL);
const [model, setModel] = useState(cloud?.model ?? DEFAULT_MODEL);
const [apiKey, setApiKey] = useState(cloud?.apiKey ?? '');
// Probe availability (WebGPU / key set) when the screen mounts.
useEffect(() => {
void refreshAvailability();
}, [refreshAvailability]);
// Keep the form in sync if the saved config changes elsewhere.
useEffect(() => {
setBaseUrl(cloud?.baseUrl ?? DEFAULT_BASE_URL);
setModel(cloud?.model ?? DEFAULT_MODEL);
setApiKey(cloud?.apiKey ?? '');
}, [cloud]);
const engineLabel =
engineKind === 'cloud'
? 'Cloud (your key)'
: engineKind === 'webllm'
? 'On-device (WebGPU)'
: 'Not available';
const onSave = () => {
const url = baseUrl.trim() || DEFAULT_BASE_URL;
const mdl = model.trim() || DEFAULT_MODEL;
const key = apiKey.trim();
if (!key) return;
setCloud({ baseUrl: url, apiKey: key, model: mdl });
void refreshAvailability();
};
const onClear = () => {
setCloud(undefined);
setApiKey('');
setBaseUrl(DEFAULT_BASE_URL);
setModel(DEFAULT_MODEL);
void refreshAvailability();
};
return (
<>
<ThemedText type="subtitle">AI (optional)</ThemedText>
<ThemedText type="small" themeColor="textSecondary">
Wisp can answer questions about your lectures with cited answers. This is fully optional
without it, you still get on-device semantic search. It runs on-device when your browser
supports WebGPU, or through your own API key below.
</ThemedText>
<ThemedView type="backgroundElement" style={styles.card}>
<View style={styles.rowBetween}>
<ThemedText type="smallBold">Active engine</ThemedText>
<ThemedText type="small" style={{ color: engineKind === 'none' ? theme.textSecondary : ACCENT }}>
{engineLabel}
</ThemedText>
</View>
<ThemedText type="small" themeColor="textSecondary">
On-device generation needs a WebGPU-capable browser. If that isn&apos;t available, add an
API key below to use a cloud model instead.
</ThemedText>
</ThemedView>
<ThemedText type="smallBold">Bring your own key</ThemedText>
<ThemedText type="small" themeColor="textSecondary">
Use any OpenAI-compatible endpoint. Your key is stored locally on this device and is never
shared with Wisp. Only your question and the retrieved lecture snippets are sent to the
endpoint never your raw audio or full transcripts.
</ThemedText>
<Field label="Base URL">
<TextInput
value={baseUrl}
onChangeText={setBaseUrl}
autoCapitalize="none"
autoCorrect={false}
keyboardType="url"
placeholder={DEFAULT_BASE_URL}
placeholderTextColor={theme.textSecondary}
style={[styles.input, { color: theme.text, backgroundColor: theme.backgroundElement }]}
/>
</Field>
<Field label="Model">
<TextInput
value={model}
onChangeText={setModel}
autoCapitalize="none"
autoCorrect={false}
placeholder={DEFAULT_MODEL}
placeholderTextColor={theme.textSecondary}
style={[styles.input, { color: theme.text, backgroundColor: theme.backgroundElement }]}
/>
</Field>
<Field label="API key">
<TextInput
value={apiKey}
onChangeText={setApiKey}
autoCapitalize="none"
autoCorrect={false}
secureTextEntry
placeholder="sk-…"
placeholderTextColor={theme.textSecondary}
style={[styles.input, { color: theme.text, backgroundColor: theme.backgroundElement }]}
/>
</Field>
<View style={styles.btnRow}>
<Pressable
onPress={onSave}
disabled={apiKey.trim() === ''}
style={({ pressed }) => [
styles.saveBtn,
{ opacity: apiKey.trim() === '' ? 0.5 : pressed ? 0.85 : 1 },
]}>
<ThemedText style={styles.saveBtnText}>Save key</ThemedText>
</Pressable>
<Pressable
onPress={onClear}
disabled={!cloud}
style={({ pressed }) => [
styles.clearBtn,
{ borderColor: theme.textSecondary, opacity: !cloud ? 0.5 : pressed ? 0.85 : 1 },
]}>
<ThemedText type="smallBold" themeColor="textSecondary">
Clear
</ThemedText>
</Pressable>
</View>
</>
);
}
function Field({ label, children }: { label: string; children: React.ReactNode }) {
return (
<View style={styles.field}>
<ThemedText type="small" themeColor="textSecondary">
{label}
</ThemedText>
{children}
</View>
);
}
function cap(s: string) { function cap(s: string) {
return s.charAt(0).toUpperCase() + s.slice(1); return s.charAt(0).toUpperCase() + s.slice(1);
} }
@@ -63,4 +225,28 @@ const styles = StyleSheet.create({
card: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.one }, card: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.one },
rowBetween: { flexDirection: 'row', alignItems: 'center', justifyContent: 'space-between' }, rowBetween: { flexDirection: 'row', alignItems: 'center', justifyContent: 'space-between' },
spacer: { height: Spacing.three }, spacer: { height: Spacing.three },
field: { gap: Spacing.one },
input: {
borderRadius: Spacing.two,
paddingHorizontal: Spacing.three,
paddingVertical: Spacing.two,
fontSize: 15,
},
btnRow: { flexDirection: 'row', gap: Spacing.two, marginTop: Spacing.one },
saveBtn: {
flex: 1,
backgroundColor: ACCENT,
paddingVertical: Spacing.two,
borderRadius: Spacing.two,
alignItems: 'center',
},
saveBtnText: { color: '#fff', fontWeight: '700' },
clearBtn: {
paddingHorizontal: Spacing.four,
paddingVertical: Spacing.two,
borderRadius: Spacing.two,
borderWidth: 1,
alignItems: 'center',
justifyContent: 'center',
},
}); });
+95
View File
@@ -0,0 +1,95 @@
// Bring-your-own-key cloud generation engine (OpenAI-compatible chat
// completions). Used when the user has supplied a CloudConfig (baseUrl, apiKey,
// model). Works against OpenAI itself or any OpenAI-compatible endpoint.
//
// PRIVACY: this engine sends ONLY the system prompt + the user's question +
// the retrieved lecture snippets (the caller assembles these via prompt.ts).
// Raw audio and full transcripts are NEVER sent — that is a hard Phase 5 rule.
//
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
import type { CloudConfig, GenerationEngine, GenOptions } from './engine';
interface ChatMessage {
role: 'system' | 'user';
content: string;
}
interface ChatCompletionResponse {
choices?: { message?: { content?: string } }[];
}
export function createCloudEngine(cfg: CloudConfig): GenerationEngine {
// Normalize: drop a trailing slash so we can append the path cleanly.
const base = cfg.baseUrl.replace(/\/+$/, '');
return {
kind: 'cloud',
label: 'Cloud (your key)',
async isAvailable(): Promise<boolean> {
return !!cfg.apiKey;
},
// Cloud models need no local download; nothing to load.
async loadModel(): Promise<void> {
/* no-op */
},
isLoaded(): boolean {
return true;
},
async generate(prompt: string, opts?: GenOptions): Promise<string> {
if (!cfg.apiKey) {
throw new Error('Cloud generation requires an API key.');
}
const messages: ChatMessage[] = [];
if (opts?.system) messages.push({ role: 'system', content: opts.system });
messages.push({ role: 'user', content: prompt });
let res: Response;
try {
res = await fetch(`${base}/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${cfg.apiKey}`,
},
body: JSON.stringify({
model: cfg.model,
messages,
max_tokens: opts?.maxTokens ?? 512,
}),
signal: opts?.signal,
});
} catch (err) {
if (err instanceof DOMException && err.name === 'AbortError') throw err;
throw new Error(
`Cloud request failed: ${err instanceof Error ? err.message : String(err)}`,
);
}
if (!res.ok) {
const detail = await res.text().catch(() => '');
throw new Error(
`Cloud request failed (${res.status} ${res.statusText})${detail ? `: ${detail}` : ''}`,
);
}
let data: ChatCompletionResponse;
try {
data = (await res.json()) as ChatCompletionResponse;
} catch {
throw new Error('Cloud response was not valid JSON.');
}
const content = data.choices?.[0]?.message?.content;
if (typeof content !== 'string') {
throw new Error('Cloud response did not contain a message.');
}
return content;
},
};
}
+62
View File
@@ -0,0 +1,62 @@
// Optional generative layer (ROADMAP Phase 5). STRICTLY opt-in and gated: a
// local WebLLM model (WebGPU) or a bring-your-own-key cloud model. When neither
// is available the app falls back to the deterministic Phase 1/3 features — the
// LLM is never a hard dependency, and raw audio/transcripts are NEVER sent to a
// cloud: only the user's question + the retrieved snippets go out (BYO-key path).
export interface GenOptions {
system?: string;
maxTokens?: number;
signal?: AbortSignal;
/** Streamed token callback (optional). */
onToken?: (delta: string) => void;
}
export interface GenerationEngine {
/** Which backend this is. */
readonly kind: 'webllm' | 'cloud' | 'none';
/** Human label, e.g. 'On-device (Qwen2.5-1.5B)' or 'Cloud (your key)'. */
readonly label: string;
/** Whether this engine can run here right now (WebGPU present / key set). */
isAvailable(): Promise<boolean>;
/** Load/prepare the model. onProgress in [0,1] (for the WebLLM download). */
loadModel(onProgress?: (p: number) => void): Promise<void>;
isLoaded(): boolean;
/** Generate a completion for `prompt`. */
generate(prompt: string, opts?: GenOptions): Promise<string>;
}
/** Cloud (BYO-key) configuration — OpenAI-compatible chat completions. */
export interface CloudConfig {
/** e.g. https://api.openai.com/v1 (or any OpenAI-compatible base). */
baseUrl: string;
apiKey: string;
model: string;
}
/** Default local WebLLM model (small instruct, q4) — pulled from the CDN. */
export const WEBLLM_MODEL = 'Qwen2.5-1.5B-Instruct-q4f16_1-MLC';
// ---------------------------------------------------------------------------
// RAG ("ask your lectures") result shape — produced by rag.ts.
// ---------------------------------------------------------------------------
export interface RagCitation {
transcriptId: string;
segmentId?: string;
start: number;
text: string;
/** Retrieval score from Phase 1 search. */
score: number;
}
export interface RagAnswer {
/** The generated answer (grounded in the citations). */
answer: string;
/** The lecture moments the answer is based on (always shown verbatim). */
citations: RagCitation[];
/** False when no relevant lecture content was retrieved (we refuse to answer). */
grounded: boolean;
/** True when produced by an LLM; false when this is the search-only fallback. */
generated: boolean;
}
+32
View File
@@ -0,0 +1,32 @@
// NATIVE on-device generation stub. Running a local LLM on device (e.g. via
// llama.rn / MLC's native runtime) is a Phase 5 follow-up; until then the native
// webllm engine reports unavailable so callers transparently fall back to the
// search-only path (or a BYO-key cloud model). It NEVER produces a fake answer.
//
// This module is selected by Metro for native (.native.ts) and is never imported
// by any vitest test.
import type { GenerationEngine } from './engine';
const NOT_AVAILABLE = 'On-device generation is not available on native yet';
export const webllm: GenerationEngine = {
kind: 'webllm',
label: 'On-device (Qwen2.5-1.5B)',
async isAvailable(): Promise<boolean> {
return false;
},
async loadModel(): Promise<void> {
throw new Error(NOT_AVAILABLE);
},
isLoaded(): boolean {
return false;
},
async generate(): Promise<string> {
throw new Error(NOT_AVAILABLE);
},
};
+3
View File
@@ -0,0 +1,3 @@
// Base resolver for TypeScript. Metro picks engineImpl.web.ts / engineImpl.native.ts
// by platform extension at build time; tsc resolves this file (defaults to web).
export { webllm } from './engineImpl.web';
+139
View File
@@ -0,0 +1,139 @@
// WEB-ONLY on-device generation engine, backed by @mlc-ai/web-llm running a
// small instruct model (Qwen2.5-1.5B, q4) entirely in the browser via WebGPU.
//
// WHY WE LOAD IT FROM A CDN AT RUNTIME (not a static import):
// web-llm ships WASM and uses dynamic imports that Metro (Expo's web bundler)
// cannot statically bundle. As with transcription/engineImpl.web.ts, we never
// let Metro see the package: we load the ESM build from a CDN at runtime via a
// dynamic import hidden behind `new Function` (so Metro's static analyzer can't
// trip over it). The browser resolves it natively. This keeps the JS bundle
// small — the model itself is a ~1GB download, cached by the browser after the
// first run — and means web-llm is NEVER bundled.
//
// Requires WebGPU. On a device without a WebGPU adapter, isAvailable() is false
// and callers fall back to the search-only path (no fake answers).
//
// This module is web-only and is NEVER imported by any vitest test.
import { WEBLLM_MODEL } from './engine';
import type { GenerationEngine, GenOptions } from './engine';
// Pin the web-llm ESM build we load at runtime.
const WEBLLM_CDN = 'https://esm.run/@mlc-ai/web-llm';
// `new Function` hides the dynamic import() specifier from Metro's bundler so it
// never tries to resolve/transform web-llm or its WASM.
const runtimeImport = new Function('u', 'return import(u)') as (u: string) => Promise<WebLlmModule>;
// Minimal structural types for the bits of web-llm we use.
interface InitProgressReport {
progress: number;
text?: string;
}
interface ChatMessage {
role: 'system' | 'user' | 'assistant';
content: string;
}
interface CompletionChunk {
choices: { delta: { content?: string } }[];
}
interface CompletionResponse {
choices: { message: { content: string } }[];
}
interface ChatCompletions {
create(req: {
messages: ChatMessage[];
stream: false;
max_tokens: number;
}): Promise<CompletionResponse>;
create(req: {
messages: ChatMessage[];
stream: true;
max_tokens: number;
}): Promise<AsyncIterable<CompletionChunk>>;
}
interface MLCEngine {
chat: { completions: ChatCompletions };
}
interface WebLlmModule {
CreateMLCEngine(
model: string,
opts?: { initProgressCallback?: (p: InitProgressReport) => void },
): Promise<MLCEngine>;
}
let libPromise: Promise<WebLlmModule> | null = null;
function lib(): Promise<WebLlmModule> {
if (!libPromise) libPromise = runtimeImport(WEBLLM_CDN);
return libPromise;
}
let engineInstance: MLCEngine | null = null;
async function hasWebGpu(): Promise<boolean> {
try {
if (typeof navigator === 'undefined' || !('gpu' in navigator)) return false;
const gpu = (navigator as { gpu?: { requestAdapter(): Promise<unknown> } }).gpu;
const adapter = await gpu?.requestAdapter();
return adapter != null;
} catch {
return false;
}
}
export const webllm: GenerationEngine = {
kind: 'webllm',
label: 'On-device (Qwen2.5-1.5B)',
async isAvailable(): Promise<boolean> {
return hasWebGpu();
},
async loadModel(onProgress?: (p: number) => void): Promise<void> {
if (engineInstance) return; // idempotent
const { CreateMLCEngine } = await lib();
engineInstance = await CreateMLCEngine(WEBLLM_MODEL, {
initProgressCallback: (p) => onProgress?.(p.progress),
});
},
isLoaded(): boolean {
return engineInstance != null;
},
async generate(prompt: string, opts?: GenOptions): Promise<string> {
if (!engineInstance) {
throw new Error('On-device model is not loaded; call loadModel() first.');
}
const messages: ChatMessage[] = [];
if (opts?.system) messages.push({ role: 'system', content: opts.system });
messages.push({ role: 'user', content: prompt });
const maxTokens = opts?.maxTokens ?? 512;
if (opts?.onToken) {
// Stream: surface each delta and accumulate the full text to return.
const stream = await engineInstance.chat.completions.create({
messages,
stream: true,
max_tokens: maxTokens,
});
let full = '';
for await (const chunk of stream) {
const delta = chunk.choices[0]?.delta?.content ?? '';
if (delta) {
full += delta;
opts.onToken(delta);
}
}
return full;
}
const res = await engineInstance.chat.completions.create({
messages,
stream: false,
max_tokens: maxTokens,
});
return res.choices[0]?.message?.content ?? '';
},
};
+51
View File
@@ -0,0 +1,51 @@
// Public entry point for the optional generative ("ask your lectures") layer.
//
// `./engineImpl` is resolved by Metro to engineImpl.web.ts or
// engineImpl.native.ts by platform extension; the base engineImpl.ts re-export
// (web) is what TypeScript resolves for typechecking. Consumers call
// getGenerationEngine() and stay platform-agnostic.
//
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
import type { CloudConfig, GenerationEngine } from './engine';
import { createCloudEngine } from './cloud';
import { webllm } from './engineImpl';
/**
* An engine that does nothing useful — represents "no generation backend here".
* isAvailable() is always false; generate()/loadModel() throw. Callers should
* check isAvailable() and fall back to the search-only path (no fake answers).
*/
export const noneEngine: GenerationEngine = {
kind: 'none',
label: 'No model',
async isAvailable(): Promise<boolean> {
return false;
},
async loadModel(): Promise<void> {
throw new Error('No generation engine is available.');
},
isLoaded(): boolean {
return false;
},
async generate(): Promise<string> {
throw new Error('No generation engine is available.');
},
};
/**
* Pick the generation engine.
*
* - If a cloud config WITH an apiKey is given, use the BYO-key cloud engine.
* - Otherwise return the platform on-device (webllm) engine. On a device without
* WebGPU (web) or on native, that engine's isAvailable() resolves false, which
* is how the "none" state is represented in practice — callers MUST check
* isAvailable() before loading/generating, and fall back to search-only.
*/
export function getGenerationEngine(cloud?: CloudConfig): GenerationEngine {
if (cloud?.apiKey) return createCloudEngine(cloud);
return webllm;
}
export { createCloudEngine, webllm };
export * from './engine';
+45
View File
@@ -0,0 +1,45 @@
import { describe, it, expect } from 'vitest';
import { buildRagPrompt } from './prompt';
describe('buildRagPrompt', () => {
it('numbers excerpts 1-based with formatted times and includes the question', () => {
const { system, prompt } = buildRagPrompt('What is entropy?', [
{ text: 'Entropy measures disorder.', start: 65, transcriptId: 't1' },
{ text: 'It always increases in a closed system.', start: 130, transcriptId: 't1' },
]);
// Excerpts are numbered and carry a mm:ss timecode from `start`.
expect(prompt).toContain('[1] (1:05) Entropy measures disorder.');
expect(prompt).toContain('[2] (2:10) It always increases in a closed system.');
// Question is present.
expect(prompt).toContain('Question: What is entropy?');
// System prompt enforces grounding + citations.
expect(system).toMatch(/only/i);
expect(system).toMatch(/\[1\]|square brackets/i);
});
it('trims the question', () => {
const { prompt } = buildRagPrompt(' hello? ', [
{ text: 'hi', start: 0, transcriptId: 't1' },
]);
expect(prompt).toContain('Question: hello?');
expect(prompt).not.toContain('Question: hello?');
});
it('is deterministic for identical inputs', () => {
const snips = [{ text: 'a', start: 5, transcriptId: 't1' }];
const a = buildRagPrompt('q', snips);
const b = buildRagPrompt('q', snips);
expect(a).toEqual(b);
});
it('yields a refusal-style prompt with no snippets (no excerpts -> admit not found)', () => {
const { system, prompt } = buildRagPrompt('Anything?', []);
expect(prompt).toContain('(no lecture excerpts were found)');
expect(prompt).toContain('Question: Anything?');
// Still must not contain a fabricated [1] excerpt.
expect(prompt).not.toMatch(/^\[1\]/m);
// The system rule that drives the refusal is present.
expect(system).toMatch(/could not find it in the lectures/i);
});
});
+56
View File
@@ -0,0 +1,56 @@
// PURE, deterministic RAG prompt builder for the optional "ask your lectures"
// generative layer (Phase 5). No I/O, no platform deps — just string assembly,
// so it is fully unit-testable and produces byte-identical output for identical
// inputs.
//
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
import { formatClock } from '../format';
/** A retrieved lecture snippet to ground the answer in. */
export interface PromptSnippet {
text: string;
/** Snippet start, in seconds (rendered as mm:ss in the prompt). */
start: number;
transcriptId: string;
}
/**
* Build the system + user messages for a grounded, cited RAG answer.
*
* The system prompt forces the model to answer ONLY from the supplied excerpts,
* to admit when the excerpts don't cover the question (no hallucinated answers),
* to stay concise, and to cite excerpt numbers like [1], [2].
*
* The user prompt lists the excerpts as `[i] (mm:ss) text` (1-based), then the
* question. With zero snippets the excerpt block is replaced by an explicit
* "(no lecture excerpts were found)" marker, which — combined with the grounding
* rule — steers the model to a refusal rather than an invented answer.
*
* Deterministic: same inputs -> same output.
*/
export function buildRagPrompt(
question: string,
snippets: PromptSnippet[],
): { system: string; prompt: string } {
const system = [
'You are a study assistant that answers questions about a student\'s lecture recordings.',
'Answer ONLY using the numbered lecture excerpts provided below.',
'If the excerpts do not contain the answer, say you could not find it in the lectures — do not use outside knowledge and do not guess.',
'Be concise.',
'Cite the excerpts you used by their number in square brackets, like [1] or [2][3].',
].join(' ');
const q = question.trim();
const excerptBlock =
snippets.length === 0
? '(no lecture excerpts were found)'
: snippets
.map((s, i) => `[${i + 1}] (${formatClock(s.start)}) ${s.text.trim()}`)
.join('\n');
const prompt = `Lecture excerpts:\n${excerptBlock}\n\nQuestion: ${q}`;
return { system, prompt };
}
+91
View File
@@ -0,0 +1,91 @@
// RAG orchestrator for the optional "ask your lectures" feature (Phase 5).
//
// Flow: retrieve the most relevant lecture moments with the existing on-device
// semantic search, then (only if a generation engine is available) ask the model
// to write a grounded, cited answer from those moments. The retrieved moments are
// ALWAYS returned as citations so the UI can show them verbatim.
//
// Hard rules honoured here:
// - Opt-in + gated: if no engine is available we return the search hits as a
// "search-only" fallback (generated:false) — never a fake/ungrounded answer.
// - Privacy: we only ever pass the question + retrieved snippets to the engine
// (the cloud path is OpenAI-compatible BYO-key); raw audio/transcripts never
// leave the device.
// - Defensive: any generation failure degrades to the search-only fallback so
// the user still gets the moments.
//
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
import { searchLectures } from '../search/search';
import type { GenerationEngine, RagAnswer, RagCitation } from './engine';
import { buildRagPrompt } from './prompt';
/** How many lecture moments to retrieve and feed the model. */
const RAG_LIMIT = 6;
const EMPTY: RagAnswer = {
answer: '',
citations: [],
grounded: false,
generated: false,
};
/**
* Answer `question` from the user's lectures, grounded in retrieved moments.
*
* - Empty/whitespace question -> empty, ungrounded result.
* - No relevant moments retrieved -> empty, ungrounded result (we refuse to
* answer with nothing to cite).
* - Engine unavailable / generation fails -> the retrieved moments as a
* search-only fallback (grounded:true, generated:false).
* - Otherwise -> the model's grounded, cited answer (generated:true).
*/
export async function askLectures(
question: string,
engine: GenerationEngine,
opts?: { courseId?: string | null; signal?: AbortSignal },
): Promise<RagAnswer> {
const q = question.trim();
if (q.length === 0) return EMPTY;
const hits = await searchLectures(q, {
courseId: opts?.courseId,
limit: RAG_LIMIT,
});
if (hits.length === 0) return EMPTY;
// The retrieved moments are shown verbatim regardless of whether we generate.
const citations: RagCitation[] = hits.map((h) => ({
transcriptId: h.transcriptId,
segmentId: h.segmentId,
start: h.start,
text: h.text,
score: h.score,
}));
// Search-only fallback: no engine here right now (no WebGPU / no key set).
if (!(await engine.isAvailable())) {
return { answer: '', citations, grounded: true, generated: false };
}
try {
if (!engine.isLoaded()) await engine.loadModel();
const { system, prompt } = buildRagPrompt(
q,
hits.map((h) => ({
text: h.text,
start: h.start,
transcriptId: h.transcriptId,
})),
);
const answer = await engine.generate(prompt, {
system,
maxTokens: 512,
signal: opts?.signal,
});
return { answer, citations, grounded: true, generated: true };
} catch {
// Any failure (model load, generation, abort) -> still give the moments.
return { answer: '', citations, grounded: true, generated: false };
}
}
+159
View File
@@ -0,0 +1,159 @@
// Session/UI state for the optional generative ("ask your lectures") layer
// (Phase 5). Holds the BYO-key cloud config (if any), the on-device model load
// status, and which backend is selected, and exposes a single ask() that wires
// retrieval -> generation via askLectures.
//
// The feature is GATED BY AVAILABILITY, not a manual switch: `enabled` defaults
// true, but ask() still degrades to the search-only fallback when no engine is
// available (no WebGPU and no cloud key). Raw audio/transcripts never leave the
// device — only the question + retrieved snippets do (cloud path).
//
// Persistence: on web the cloud config is saved to localStorage under
// 'wisp:ai'; native keeps it in-memory (no localStorage). Everything here is
// defensive — refreshAvailability never throws.
import { create } from 'zustand';
import {
getGenerationEngine,
type CloudConfig,
type GenerationEngine,
type RagAnswer,
} from '@/lib/generation';
import { askLectures } from '@/lib/generation/rag';
const STORAGE_KEY = 'wisp:ai';
type ModelStatus = 'idle' | 'loading' | 'ready';
type EngineKind = 'webllm' | 'cloud' | 'none';
interface PersistedState {
enabled: boolean;
cloud?: CloudConfig;
}
/** Web-only localStorage handle (undefined on native / SSR). */
function ls(): Storage | undefined {
try {
return typeof localStorage !== 'undefined' ? localStorage : undefined;
} catch {
return undefined;
}
}
/** Read persisted enabled + cloud config from localStorage (web only). */
function loadPersisted(): PersistedState {
const store = ls();
if (!store) return { enabled: true };
try {
const raw = store.getItem(STORAGE_KEY);
if (!raw) return { enabled: true };
const parsed = JSON.parse(raw) as Partial<PersistedState>;
return {
enabled: parsed.enabled ?? true,
cloud: parsed.cloud,
};
} catch {
return { enabled: true };
}
}
/** Persist (or clear) the cloud config + enabled flag on web. No-op on native. */
function persist(state: PersistedState): void {
const store = ls();
if (!store) return;
try {
store.setItem(STORAGE_KEY, JSON.stringify(state));
} catch {
/* quota / private mode — ignore, just stay in-memory */
}
}
const initial = loadPersisted();
interface AiState {
/** Feature flag — defaults true; real gating is via availability. */
enabled: boolean;
/** BYO-key cloud config, if the user supplied one. */
cloud?: CloudConfig;
/** On-device model load status (WebLLM). */
modelStatus: ModelStatus;
/** WebLLM download progress in [0,1]. */
progress: number;
/** Which backend the current config resolves to. */
engineKind: EngineKind;
/** Set (or clear, with no arg) the cloud config; persists on web. */
setCloud: (cfg?: CloudConfig) => void;
/** Recompute engineKind and probe availability. Never throws. */
refreshAvailability: () => Promise<void>;
/** Resolve an engine, loading the on-device model if needed. */
ensureReady: () => Promise<GenerationEngine>;
/** Retrieve + (optionally) generate a grounded, cited answer. */
ask: (
question: string,
opts?: { courseId?: string | null; signal?: AbortSignal },
) => Promise<RagAnswer>;
}
export const useAi = create<AiState>((set, get) => ({
enabled: initial.enabled,
cloud: initial.cloud,
modelStatus: 'idle',
progress: 0,
// Resolve the initial backend synchronously from the persisted config.
engineKind: getGenerationEngine(initial.cloud).kind,
setCloud: (cfg) => {
persist({ enabled: get().enabled, cloud: cfg });
// A cloud config swap invalidates any in-progress on-device load state.
set({
cloud: cfg,
engineKind: getGenerationEngine(cfg).kind,
modelStatus: 'idle',
progress: 0,
});
},
refreshAvailability: async () => {
try {
const engine = getGenerationEngine(get().cloud);
set({ engineKind: engine.kind });
// Probe availability (WebGPU adapter / configured key). The result isn't
// stored beyond engineKind; ask()/ensureReady() re-check at call time.
await engine.isAvailable();
} catch {
// Stay defensive: never throw out of availability refresh.
set({ engineKind: 'none' });
}
},
ensureReady: async () => {
const engine = getGenerationEngine(get().cloud);
set({ engineKind: engine.kind });
// Only the on-device WebLLM engine has a model to download — and only when
// it can actually run here (WebGPU). On a machine without WebGPU we skip the
// (doomed, ~1GB) load and let askLectures degrade to search-only.
if (
engine.kind === 'webllm' &&
!engine.isLoaded() &&
(await engine.isAvailable())
) {
set({ modelStatus: 'loading', progress: 0 });
try {
await engine.loadModel((p) => set({ progress: p }));
set({ modelStatus: 'ready', progress: 1 });
} catch (err) {
set({ modelStatus: 'idle', progress: 0 });
throw err;
}
} else {
set({ modelStatus: 'ready' });
}
return engine;
},
ask: async (question, opts) => {
const engine = await get().ensureReady();
return askLectures(question, engine, opts);
},
}));