feat(phase2): in-app live recording into courses (web)
- useRecorder: MediaRecorder mic capture with elapsed timer, live input-level meter (AnalyserNode), pause/resume, and a screen Wake Lock; returns the recording as an ArrayBuffer that flows into the existing transcribe pipeline. - Record screen: big timer + level meter + start/pause/resume/stop; on stop the pre-capture sheet collects title/course/date, then it transcribes + saves + indexes (reusing Phase 0/1). Home gets Record + Import buttons. - Web-only for now; native realtime (whisper.rn), OPFS crash-recovery, tab-audio capture, and the per-course dashboard are follow-ups. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -15,6 +15,7 @@ export default function RootLayout() {
|
|||||||
<ThemeProvider value={scheme === 'dark' ? DarkTheme : DefaultTheme}>
|
<ThemeProvider value={scheme === 'dark' ? DarkTheme : DefaultTheme}>
|
||||||
<Stack>
|
<Stack>
|
||||||
<Stack.Screen name="index" options={{ title: 'Wisp' }} />
|
<Stack.Screen name="index" options={{ title: 'Wisp' }} />
|
||||||
|
<Stack.Screen name="record" options={{ title: 'Record' }} />
|
||||||
<Stack.Screen name="transcript/[id]" options={{ title: 'Transcript' }} />
|
<Stack.Screen name="transcript/[id]" options={{ title: 'Transcript' }} />
|
||||||
<Stack.Screen name="search" options={{ title: 'Search' }} />
|
<Stack.Screen name="search" options={{ title: 'Search' }} />
|
||||||
<Stack.Screen name="courses" options={{ title: 'Courses' }} />
|
<Stack.Screen name="courses" options={{ title: 'Courses' }} />
|
||||||
|
|||||||
+23
-9
@@ -90,15 +90,27 @@ export default function LibraryScreen() {
|
|||||||
</View>
|
</View>
|
||||||
</View>
|
</View>
|
||||||
|
|
||||||
<Pressable
|
<View style={styles.captureRow}>
|
||||||
onPress={onNew}
|
<Link href="/record" asChild>
|
||||||
disabled={busy}
|
<Pressable
|
||||||
style={({ pressed }) => [
|
disabled={busy}
|
||||||
styles.newButton,
|
style={({ pressed }) => [
|
||||||
{ backgroundColor: '#3c87f7', opacity: busy ? 0.5 : pressed ? 0.85 : 1 },
|
styles.captureBtn,
|
||||||
]}>
|
{ backgroundColor: '#e5484d', opacity: busy ? 0.5 : pressed ? 0.85 : 1 },
|
||||||
<ThemedText style={styles.newButtonText}>+ New transcription</ThemedText>
|
]}>
|
||||||
</Pressable>
|
<ThemedText style={styles.newButtonText}>🎙 Record</ThemedText>
|
||||||
|
</Pressable>
|
||||||
|
</Link>
|
||||||
|
<Pressable
|
||||||
|
onPress={onNew}
|
||||||
|
disabled={busy}
|
||||||
|
style={({ pressed }) => [
|
||||||
|
styles.captureBtn,
|
||||||
|
{ backgroundColor: '#3c87f7', opacity: busy ? 0.5 : pressed ? 0.85 : 1 },
|
||||||
|
]}>
|
||||||
|
<ThemedText style={styles.newButtonText}>+ Import file</ThemedText>
|
||||||
|
</Pressable>
|
||||||
|
</View>
|
||||||
|
|
||||||
{busy && <ActiveJob />}
|
{busy && <ActiveJob />}
|
||||||
{job.status === 'error' && (
|
{job.status === 'error' && (
|
||||||
@@ -241,6 +253,8 @@ const styles = StyleSheet.create({
|
|||||||
headerLinks: { flexDirection: 'row', gap: Spacing.three, alignItems: 'center' },
|
headerLinks: { flexDirection: 'row', gap: Spacing.three, alignItems: 'center' },
|
||||||
newButton: { paddingVertical: Spacing.three, borderRadius: Spacing.three, alignItems: 'center' },
|
newButton: { paddingVertical: Spacing.three, borderRadius: Spacing.three, alignItems: 'center' },
|
||||||
newButtonText: { color: '#fff', fontWeight: '700', fontSize: 16 },
|
newButtonText: { color: '#fff', fontWeight: '700', fontSize: 16 },
|
||||||
|
captureRow: { flexDirection: 'row', gap: Spacing.two },
|
||||||
|
captureBtn: { flex: 1, paddingVertical: Spacing.three, borderRadius: Spacing.three, alignItems: 'center' },
|
||||||
card: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.two },
|
card: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.two },
|
||||||
rowBetween: { flexDirection: 'row', alignItems: 'center', justifyContent: 'space-between', gap: Spacing.two },
|
rowBetween: { flexDirection: 'row', alignItems: 'center', justifyContent: 'space-between', gap: Spacing.two },
|
||||||
filterBar: { gap: Spacing.two, paddingVertical: Spacing.one, paddingRight: Spacing.three },
|
filterBar: { gap: Spacing.two, paddingVertical: Spacing.one, paddingRight: Spacing.three },
|
||||||
|
|||||||
@@ -0,0 +1,143 @@
|
|||||||
|
import { Stack, useRouter } from 'expo-router';
|
||||||
|
import { useState } from 'react';
|
||||||
|
import { Pressable, StyleSheet, View } from 'react-native';
|
||||||
|
|
||||||
|
import { PreCaptureSheet, type PreCaptureResult } from '@/components/PreCaptureSheet';
|
||||||
|
import { ThemedText } from '@/components/themed-text';
|
||||||
|
import { ThemedView } from '@/components/themed-view';
|
||||||
|
import { MaxContentWidth, Spacing } from '@/constants/theme';
|
||||||
|
import { useTheme } from '@/hooks/use-theme';
|
||||||
|
import { useRecorder, type RecordingResult } from '@/hooks/useRecorder';
|
||||||
|
import { formatClock } from '@/lib/format';
|
||||||
|
import { useTranscribe } from '@/stores/transcribeStore';
|
||||||
|
|
||||||
|
export default function RecordScreen() {
|
||||||
|
const theme = useTheme();
|
||||||
|
const router = useRouter();
|
||||||
|
const rec = useRecorder();
|
||||||
|
const [pending, setPending] = useState<RecordingResult | null>(null);
|
||||||
|
|
||||||
|
const onStop = async () => {
|
||||||
|
const r = await rec.stop();
|
||||||
|
if (r) setPending(r);
|
||||||
|
else router.back();
|
||||||
|
};
|
||||||
|
|
||||||
|
const onStart = async (meta: PreCaptureResult) => {
|
||||||
|
const r = pending;
|
||||||
|
setPending(null);
|
||||||
|
if (!r) return;
|
||||||
|
const id = await useTranscribe.getState().start({ data: r.data, name: r.name }, meta);
|
||||||
|
if (id) router.replace({ pathname: '/transcript/[id]', params: { id } });
|
||||||
|
else router.back();
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!rec.supported) {
|
||||||
|
return (
|
||||||
|
<ThemedView style={[styles.fill, styles.center]}>
|
||||||
|
<Stack.Screen options={{ title: 'Record' }} />
|
||||||
|
<ThemedText type="small" themeColor="textSecondary" style={styles.notice}>
|
||||||
|
In-app recording is available in the browser for now. On mobile, use your phone's
|
||||||
|
recorder and import the file (native live capture is coming).
|
||||||
|
</ThemedText>
|
||||||
|
</ThemedView>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const recording = rec.status === 'recording';
|
||||||
|
const paused = rec.status === 'paused';
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ThemedView style={[styles.fill, styles.center]}>
|
||||||
|
<Stack.Screen options={{ title: 'Record' }} />
|
||||||
|
<View style={styles.stage}>
|
||||||
|
<ThemedText style={styles.timer}>{formatClock(Math.floor(rec.elapsedMs / 1000))}</ThemedText>
|
||||||
|
<ThemedText type="small" themeColor="textSecondary">
|
||||||
|
{recording ? '● Recording — on your device' : paused ? 'Paused' : 'Ready to record'}
|
||||||
|
</ThemedText>
|
||||||
|
|
||||||
|
<View style={[styles.meterTrack, { backgroundColor: theme.backgroundElement }]}>
|
||||||
|
<View
|
||||||
|
style={[
|
||||||
|
styles.meterFill,
|
||||||
|
{ width: `${Math.round((recording ? rec.level : 0) * 100)}%`, backgroundColor: '#e5484d' },
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
</View>
|
||||||
|
|
||||||
|
{rec.error && (
|
||||||
|
<ThemedText type="small" style={styles.error}>{rec.error}</ThemedText>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<View style={styles.controls}>
|
||||||
|
{rec.status === 'idle' || rec.status === 'stopped' ? (
|
||||||
|
<BigButton label="Start recording" color="#e5484d" onPress={() => void rec.start()} />
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
{recording ? (
|
||||||
|
<BigButton label="Pause" color={theme.backgroundElement} textColor={theme.text} onPress={rec.pause} />
|
||||||
|
) : (
|
||||||
|
<BigButton label="Resume" color="#3c87f7" onPress={rec.resume} />
|
||||||
|
)}
|
||||||
|
<BigButton label="Stop & transcribe" color="#3c87f7" onPress={() => void onStop()} />
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</View>
|
||||||
|
|
||||||
|
<ThemedText type="small" themeColor="textSecondary" style={styles.hint}>
|
||||||
|
Keep this tab in front while recording. Everything stays on your device.
|
||||||
|
</ThemedText>
|
||||||
|
</View>
|
||||||
|
|
||||||
|
<PreCaptureSheet
|
||||||
|
visible={pending !== null}
|
||||||
|
defaultTitle={defaultName()}
|
||||||
|
onStart={(m) => void onStart(m)}
|
||||||
|
onCancel={() => {
|
||||||
|
setPending(null);
|
||||||
|
router.back();
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</ThemedView>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function BigButton({
|
||||||
|
label,
|
||||||
|
color,
|
||||||
|
textColor,
|
||||||
|
onPress,
|
||||||
|
}: {
|
||||||
|
label: string;
|
||||||
|
color: string;
|
||||||
|
textColor?: string;
|
||||||
|
onPress: () => void;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<Pressable onPress={onPress} style={({ pressed }) => [styles.btn, { backgroundColor: color, opacity: pressed ? 0.85 : 1 }]}>
|
||||||
|
<ThemedText style={[styles.btnText, textColor ? { color: textColor } : styles.btnTextLight]}>{label}</ThemedText>
|
||||||
|
</Pressable>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function defaultName(): string {
|
||||||
|
const d = new Date();
|
||||||
|
const pad = (n: number) => String(n).padStart(2, '0');
|
||||||
|
return `Lecture ${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const styles = StyleSheet.create({
|
||||||
|
fill: { flex: 1 },
|
||||||
|
center: { alignItems: 'center', justifyContent: 'center' },
|
||||||
|
stage: { width: '100%', maxWidth: MaxContentWidth, padding: Spacing.four, gap: Spacing.three, alignItems: 'center' },
|
||||||
|
timer: { fontSize: 56, fontWeight: '700', fontVariant: ['tabular-nums'] },
|
||||||
|
meterTrack: { width: '80%', height: 8, borderRadius: 4, overflow: 'hidden' },
|
||||||
|
meterFill: { height: 8, borderRadius: 4 },
|
||||||
|
controls: { flexDirection: 'row', gap: Spacing.two, marginTop: Spacing.three, flexWrap: 'wrap', justifyContent: 'center' },
|
||||||
|
btn: { paddingVertical: Spacing.three, paddingHorizontal: Spacing.four, borderRadius: Spacing.three, alignItems: 'center' },
|
||||||
|
btnText: { fontWeight: '700', fontSize: 16 },
|
||||||
|
btnTextLight: { color: '#fff' },
|
||||||
|
error: { color: '#e5484d' },
|
||||||
|
notice: { padding: Spacing.four, textAlign: 'center', maxWidth: 480 },
|
||||||
|
hint: { textAlign: 'center', marginTop: Spacing.two },
|
||||||
|
});
|
||||||
@@ -0,0 +1,185 @@
|
|||||||
|
// Web microphone recorder (MediaRecorder) for in-app lecture capture (ROADMAP
|
||||||
|
// Phase 2). Records to a Blob, exposes elapsed time + a live input level, holds
|
||||||
|
// a screen Wake Lock while recording, and supports pause/resume. On stop it
|
||||||
|
// returns the recording as an ArrayBuffer that flows straight into the existing
|
||||||
|
// transcribe pipeline (decode -> chunk -> Whisper -> save).
|
||||||
|
//
|
||||||
|
// Web-only: getUserMedia/MediaRecorder/AudioContext are browser APIs. On native
|
||||||
|
// `supported` is false and the controls no-op (native realtime capture via
|
||||||
|
// whisper.rn is a follow-up).
|
||||||
|
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||||
|
import { Platform } from 'react-native';
|
||||||
|
|
||||||
|
export type RecorderStatus = 'idle' | 'recording' | 'paused' | 'stopped';
|
||||||
|
|
||||||
|
export interface RecordingResult {
|
||||||
|
data: ArrayBuffer;
|
||||||
|
mime: string;
|
||||||
|
name: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Recorder {
|
||||||
|
supported: boolean;
|
||||||
|
status: RecorderStatus;
|
||||||
|
elapsedMs: number;
|
||||||
|
/** Live input level in [0,1], for a meter. */
|
||||||
|
level: number;
|
||||||
|
error?: string;
|
||||||
|
start: () => Promise<void>;
|
||||||
|
pause: () => void;
|
||||||
|
resume: () => void;
|
||||||
|
/** Stop and return the recording (null if nothing captured / unsupported). */
|
||||||
|
stop: () => Promise<RecordingResult | null>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSupported(): boolean {
|
||||||
|
return (
|
||||||
|
Platform.OS === 'web' &&
|
||||||
|
typeof navigator !== 'undefined' &&
|
||||||
|
!!navigator.mediaDevices?.getUserMedia &&
|
||||||
|
typeof MediaRecorder !== 'undefined'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useRecorder(): Recorder {
|
||||||
|
const supported = isSupported();
|
||||||
|
const [status, setStatus] = useState<RecorderStatus>('idle');
|
||||||
|
const [elapsedMs, setElapsedMs] = useState(0);
|
||||||
|
const [level, setLevel] = useState(0);
|
||||||
|
const [error, setError] = useState<string | undefined>(undefined);
|
||||||
|
|
||||||
|
const recorderRef = useRef<MediaRecorder | null>(null);
|
||||||
|
const chunksRef = useRef<Blob[]>([]);
|
||||||
|
const streamRef = useRef<MediaStream | null>(null);
|
||||||
|
const audioCtxRef = useRef<AudioContext | null>(null);
|
||||||
|
const analyserRef = useRef<AnalyserNode | null>(null);
|
||||||
|
const rafRef = useRef<number | null>(null);
|
||||||
|
const wakeRef = useRef<WakeLockSentinel | null>(null);
|
||||||
|
// Elapsed bookkeeping across pause/resume.
|
||||||
|
const segStartRef = useRef(0);
|
||||||
|
const accumRef = useRef(0);
|
||||||
|
|
||||||
|
const cleanup = useCallback(() => {
|
||||||
|
if (rafRef.current != null) cancelAnimationFrame(rafRef.current);
|
||||||
|
rafRef.current = null;
|
||||||
|
streamRef.current?.getTracks().forEach((t) => t.stop());
|
||||||
|
streamRef.current = null;
|
||||||
|
void audioCtxRef.current?.close().catch(() => {});
|
||||||
|
audioCtxRef.current = null;
|
||||||
|
analyserRef.current = null;
|
||||||
|
void wakeRef.current?.release().catch(() => {});
|
||||||
|
wakeRef.current = null;
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => cleanup, [cleanup]);
|
||||||
|
|
||||||
|
const tick = useCallback(() => {
|
||||||
|
// Elapsed: accumulated paused time + current running segment.
|
||||||
|
const running = status === 'recording' ? Date.now() - segStartRef.current : 0;
|
||||||
|
setElapsedMs(accumRef.current + running);
|
||||||
|
|
||||||
|
const an = analyserRef.current;
|
||||||
|
if (an) {
|
||||||
|
const buf = new Uint8Array(an.fftSize);
|
||||||
|
an.getByteTimeDomainData(buf);
|
||||||
|
let sum = 0;
|
||||||
|
for (let i = 0; i < buf.length; i++) {
|
||||||
|
const v = (buf[i]! - 128) / 128;
|
||||||
|
sum += v * v;
|
||||||
|
}
|
||||||
|
setLevel(Math.min(1, Math.sqrt(sum / buf.length) * 3));
|
||||||
|
}
|
||||||
|
rafRef.current = requestAnimationFrame(tick);
|
||||||
|
}, [status]);
|
||||||
|
|
||||||
|
// Keep the rAF loop bound to the latest `status` via tick's identity.
|
||||||
|
useEffect(() => {
|
||||||
|
if (status === 'recording' || status === 'paused') {
|
||||||
|
rafRef.current = requestAnimationFrame(tick);
|
||||||
|
return () => {
|
||||||
|
if (rafRef.current != null) cancelAnimationFrame(rafRef.current);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}, [status, tick]);
|
||||||
|
|
||||||
|
const start = useCallback(async () => {
|
||||||
|
if (!supported) {
|
||||||
|
setError('Recording is only available in the browser for now.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
setError(undefined);
|
||||||
|
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||||
|
streamRef.current = stream;
|
||||||
|
|
||||||
|
const mr = new MediaRecorder(stream);
|
||||||
|
chunksRef.current = [];
|
||||||
|
mr.ondataavailable = (e) => {
|
||||||
|
if (e.data && e.data.size > 0) chunksRef.current.push(e.data);
|
||||||
|
};
|
||||||
|
mr.start();
|
||||||
|
recorderRef.current = mr;
|
||||||
|
|
||||||
|
// Level metering off an AnalyserNode.
|
||||||
|
const ctx = new (window.AudioContext || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext)();
|
||||||
|
const src = ctx.createMediaStreamSource(stream);
|
||||||
|
const an = ctx.createAnalyser();
|
||||||
|
an.fftSize = 512;
|
||||||
|
src.connect(an);
|
||||||
|
audioCtxRef.current = ctx;
|
||||||
|
analyserRef.current = an;
|
||||||
|
|
||||||
|
// Keep the screen awake while recording (best-effort).
|
||||||
|
try {
|
||||||
|
wakeRef.current = (await navigator.wakeLock?.request('screen')) ?? null;
|
||||||
|
} catch {
|
||||||
|
/* wake lock optional */
|
||||||
|
}
|
||||||
|
|
||||||
|
accumRef.current = 0;
|
||||||
|
segStartRef.current = Date.now();
|
||||||
|
setElapsedMs(0);
|
||||||
|
setStatus('recording');
|
||||||
|
} catch (e) {
|
||||||
|
setError(e instanceof Error ? e.message : 'Could not start recording (mic permission?).');
|
||||||
|
cleanup();
|
||||||
|
setStatus('idle');
|
||||||
|
}
|
||||||
|
}, [supported, cleanup]);
|
||||||
|
|
||||||
|
const pause = useCallback(() => {
|
||||||
|
const mr = recorderRef.current;
|
||||||
|
if (!mr || status !== 'recording') return;
|
||||||
|
mr.pause();
|
||||||
|
accumRef.current += Date.now() - segStartRef.current;
|
||||||
|
setStatus('paused');
|
||||||
|
}, [status]);
|
||||||
|
|
||||||
|
const resume = useCallback(() => {
|
||||||
|
const mr = recorderRef.current;
|
||||||
|
if (!mr || status !== 'paused') return;
|
||||||
|
mr.resume();
|
||||||
|
segStartRef.current = Date.now();
|
||||||
|
setStatus('recording');
|
||||||
|
}, [status]);
|
||||||
|
|
||||||
|
const stop = useCallback(async (): Promise<RecordingResult | null> => {
|
||||||
|
const mr = recorderRef.current;
|
||||||
|
if (!mr) return null;
|
||||||
|
const mime = mr.mimeType || 'audio/webm';
|
||||||
|
const blob: Blob = await new Promise((resolve) => {
|
||||||
|
mr.onstop = () => resolve(new Blob(chunksRef.current, { type: mime }));
|
||||||
|
mr.stop();
|
||||||
|
});
|
||||||
|
recorderRef.current = null;
|
||||||
|
cleanup();
|
||||||
|
setStatus('stopped');
|
||||||
|
if (blob.size === 0) return null;
|
||||||
|
const data = await blob.arrayBuffer();
|
||||||
|
const ext = mime.includes('ogg') ? 'ogg' : mime.includes('mp4') ? 'mp4' : 'webm';
|
||||||
|
return { data, mime, name: `Recording.${ext}` };
|
||||||
|
}, [cleanup]);
|
||||||
|
|
||||||
|
return { supported, status, elapsedMs, level, error, start, pause, resume, stop };
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user