Compare commits
14 Commits
8db59d4bbe
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 1bf406d3a8 | |||
| ed1df8986f | |||
| 4bddc67e1a | |||
| fa12817050 | |||
| e3ed03a471 | |||
| 06519faea2 | |||
| 7aa8b05971 | |||
| e256508c44 | |||
| 108ac59cb1 | |||
| 7611ada001 | |||
| 97aee3e4b3 | |||
| 40858e0025 | |||
| 6d2f583136 | |||
| fc24c0875d |
+26
-21
@@ -7,12 +7,12 @@
|
||||
# `docker compose` simply builds and (re)creates the `wisp` container on the host.
|
||||
#
|
||||
# Triggers:
|
||||
# push to master -> test, then deploy-web
|
||||
# push to master -> test, then deploy-web, then build-apk
|
||||
# pull_request -> test only
|
||||
#
|
||||
# Required repo secrets (Settings -> Actions -> Secrets) — used by the APK job
|
||||
# once it is enabled (see TODO at the bottom): ANDROID_KEYSTORE_BASE64,
|
||||
# ANDROID_KEYSTORE_PASSWORD, ANDROID_KEY_ALIAS, ANDROID_KEY_PASSWORD.
|
||||
# Required repo secrets (Settings -> Actions -> Secrets) — used by the build-apk
|
||||
# job: ANDROID_KEYSTORE_BASE64, ANDROID_KEYSTORE_PASSWORD, ANDROID_KEY_ALIAS,
|
||||
# ANDROID_KEY_PASSWORD.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
name: CI
|
||||
@@ -68,15 +68,17 @@ jobs:
|
||||
# ---- build-apk: signed release APK, served at /wisp.apk ------------------
|
||||
# Socket-safe: the whole Android build runs inside `docker buildx build`
|
||||
# (context streamed to the host BuildKit — no host bind-mounts), signed via
|
||||
# BuildKit --secret from the ANDROID_* repo secrets. We then extract the APK
|
||||
# with docker create/cp and copy it into the running `wisp` web container so
|
||||
# it is downloadable at https://wisp.briggen.dev/wisp.apk.
|
||||
# BuildKit --secret from the ANDROID_* repo secrets. The `apk` target +
|
||||
# `--output type=local` write just the APK to ./out (no image load), which we
|
||||
# stream into /srv/wisp/wisp.apk so it's downloadable at
|
||||
# https://wisp.briggen.dev/wisp.apk. Speed comes from persistent BuildKit
|
||||
# cache mounts (ccache for C++, Gradle build cache) declared in the Dockerfile.
|
||||
build-apk:
|
||||
# ONLY on version tags (v*). The native build is heavy (NDK/whisper.cpp) and
|
||||
# slow on the shared prod runner; building it on every push would peg the box
|
||||
# (and degrade the live web apps). Cut a release with: git tag v1.2.3 && git push --tags
|
||||
needs: [test]
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
|
||||
# Auto-build the APK on every push to master (arm64-only keeps it ~minutes).
|
||||
# Runs after deploy-web so the web updates promptly, then the APK follows and
|
||||
# is published to /srv/wisp/wisp.apk (served at /wisp.apk via the compose mount).
|
||||
needs: [test, deploy-web]
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
@@ -89,23 +91,26 @@ jobs:
|
||||
printf '%s' "${{ secrets.ANDROID_KEYSTORE_PASSWORD }}" > "$RUNNER_TEMP/sec/ks_pass"
|
||||
printf '%s' "${{ secrets.ANDROID_KEY_ALIAS }}" > "$RUNNER_TEMP/sec/ks_alias"
|
||||
printf '%s' "${{ secrets.ANDROID_KEY_PASSWORD }}" > "$RUNNER_TEMP/sec/ks_keypass"
|
||||
- name: Build signed APK image
|
||||
- name: Build + extract the signed APK (export-only target, no image load)
|
||||
run: |
|
||||
# `--target apk --output type=local` writes ONLY app-release.apk to
|
||||
# ./out — no `--load` of the ~15GB build image (saves the multi-minute
|
||||
# export/unpack each run and stops that image filling the host disk).
|
||||
# The persistent ccache + Gradle build cache (BuildKit cache mounts in
|
||||
# the Dockerfile) make C++ recompiles near-instant on warm runs.
|
||||
docker buildx build \
|
||||
--secret id=ks_b64,src="$RUNNER_TEMP/sec/ks_b64" \
|
||||
--secret id=ks_pass,src="$RUNNER_TEMP/sec/ks_pass" \
|
||||
--secret id=ks_alias,src="$RUNNER_TEMP/sec/ks_alias" \
|
||||
--secret id=ks_keypass,src="$RUNNER_TEMP/sec/ks_keypass" \
|
||||
-f docker/android.Dockerfile -t wisp-android:ci --load .
|
||||
- name: Extract APK + publish to the host (served at /wisp.apk via the compose volume)
|
||||
--target apk --output type=local,dest=./out \
|
||||
-f docker/android.Dockerfile .
|
||||
ls -lh ./out/app-release.apk
|
||||
- name: Publish APK to the host (served at /wisp.apk via the compose volume)
|
||||
run: |
|
||||
cid=$(docker create wisp-android:ci)
|
||||
docker cp "$cid:/workspace/android/app/build/outputs/apk/release/app-release.apk" ./app-release.apk
|
||||
docker rm "$cid"
|
||||
ls -lh app-release.apk
|
||||
# Stream the APK into a host-mounted dir so it survives web container
|
||||
# recreation (the wisp container mounts /srv/wisp/wisp.apk read-only).
|
||||
docker run --rm -i -v /srv/wisp:/out alpine:3 sh -c 'mkdir -p /out && cat > /out/wisp.apk' < ./app-release.apk
|
||||
docker run --rm -i -v /srv/wisp:/out alpine:3 sh -c 'mkdir -p /out && cat > /out/wisp.apk' < ./out/app-release.apk
|
||||
- name: Cleanup staged secrets
|
||||
if: always()
|
||||
run: rm -rf "$RUNNER_TEMP/sec"
|
||||
@@ -114,4 +119,4 @@ jobs:
|
||||
continue-on-error: true
|
||||
with:
|
||||
name: wisp-release-apk
|
||||
path: app-release.apk
|
||||
path: ./out/app-release.apk
|
||||
|
||||
@@ -66,14 +66,53 @@ RUN yes | sdkmanager --licenses > /dev/null \
|
||||
"cmake;${ANDROID_CMAKE_VERSION}" > /dev/null \
|
||||
&& yes | sdkmanager --licenses > /dev/null
|
||||
|
||||
# Build accelerators, installed AFTER the (large, slow) SDK/NDK layer so that
|
||||
# layer stays cached:
|
||||
# - ccache : short-circuits recompiling unchanged C++ across builds. The bulk
|
||||
# of the build is third-party native modules (whisper.cpp via
|
||||
# whisper.rn, reanimated/worklets, gesture-handler); see the NDK
|
||||
# toolchain patch below for how it's wired in globally.
|
||||
# - nodejs : scripts/ci-android-sign.sh runs a small Node patch script (this
|
||||
# image ships Bun, not Node). Lives here so it's cached, not
|
||||
# reinstalled on every source change.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ccache nodejs \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Route EVERY NDK CMake C/C++ compile + link through ccache. AGP invokes each
|
||||
# externalNativeBuild project with this NDK toolchain file, so setting the
|
||||
# compiler launcher here applies ccache to all native modules at once — no need
|
||||
# to patch each module's CMakeLists (third-party ones don't honor ccache on
|
||||
# their own). Guarded on ccache being present; a no-op otherwise.
|
||||
RUN printf '\n%s\n' \
|
||||
'# --- WISP: route NDK CMake compiles/links through ccache ---' \
|
||||
'find_program(WISP_CCACHE_PROGRAM ccache)' \
|
||||
'if(WISP_CCACHE_PROGRAM)' \
|
||||
' set(CMAKE_C_COMPILER_LAUNCHER "${WISP_CCACHE_PROGRAM}")' \
|
||||
' set(CMAKE_CXX_COMPILER_LAUNCHER "${WISP_CCACHE_PROGRAM}")' \
|
||||
'endif()' \
|
||||
>> "${ANDROID_HOME}/ndk/${ANDROID_NDK_VERSION}/build/cmake/android.toolchain.cmake"
|
||||
|
||||
# ccache tuning. CCACHE_DIR is mounted as a persistent BuildKit cache on the
|
||||
# gradle RUN below, so hits survive across builds. COMPILERCHECK=content keeps
|
||||
# hits valid across image rebuilds; the sloppiness flags raise the hit rate for
|
||||
# RN's headers without risking incorrect cache hits for release builds.
|
||||
ENV CCACHE_DIR=/root/.ccache \
|
||||
CCACHE_MAXSIZE=5G \
|
||||
CCACHE_COMPILERCHECK=content \
|
||||
CCACHE_SLOPPINESS=time_macros,include_file_mtime,include_file_ctime,file_macro \
|
||||
CCACHE_BASEDIR=/workspace
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
FROM toolchain AS build
|
||||
|
||||
# Install JS deps first (cached unless package.json/bun.lock change).
|
||||
# Install JS deps first (cached unless package.json/bun.lock change). The bun
|
||||
# cache mount keeps re-installs fast even when the lockfile DOES change.
|
||||
COPY package.json bun.lock ./
|
||||
RUN bun install --frozen-lockfile
|
||||
RUN --mount=type=cache,target=/root/.bun/install/cache \
|
||||
bun install --frozen-lockfile
|
||||
|
||||
# Bring in the rest of the source (.dockerignore excludes node_modules, android/, ios/, dist...).
|
||||
COPY . .
|
||||
@@ -81,16 +120,22 @@ COPY . .
|
||||
# Generate the native android/ project (autolinks whisper.rn, reanimated, etc.).
|
||||
RUN bunx expo prebuild --platform android --no-install
|
||||
|
||||
# Node is required by scripts/ci-android-sign.sh (this image ships Bun, not Node).
|
||||
# Installed AFTER prebuild so the cached toolchain + prebuild layers are reused.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends nodejs && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Decode the keystore + wire release signing, then assemble the signed APK.
|
||||
# Secrets are mounted as files under /run/secrets and exported only for this RUN.
|
||||
#
|
||||
# Two persistent BuildKit cache mounts make rebuilds fast (they survive the
|
||||
# `COPY . .` layer invalidation that happens on every source change):
|
||||
# - /root/.ccache : compiled C++ objects (the big win — whisper.cpp etc.).
|
||||
# - /opt/gradle : GRADLE_USER_HOME — the Gradle build cache (--build-cache),
|
||||
# resolved dependencies, and the downloaded wrapper dist.
|
||||
# --build-cache + --parallel let unchanged Kotlin/Java/resource/dex tasks be
|
||||
# served from cache and independent module tasks run concurrently.
|
||||
RUN --mount=type=secret,id=ks_b64 \
|
||||
--mount=type=secret,id=ks_pass \
|
||||
--mount=type=secret,id=ks_alias \
|
||||
--mount=type=secret,id=ks_keypass \
|
||||
--mount=type=cache,target=/root/.ccache \
|
||||
--mount=type=cache,target=/opt/gradle \
|
||||
ANDROID_KEYSTORE_BASE64="$(cat /run/secrets/ks_b64)" \
|
||||
ANDROID_KEYSTORE_PASSWORD="$(cat /run/secrets/ks_pass)" \
|
||||
ANDROID_KEY_ALIAS="$(cat /run/secrets/ks_alias)" \
|
||||
@@ -99,7 +144,20 @@ RUN --mount=type=secret,id=ks_b64 \
|
||||
&& cd android \
|
||||
&& ./gradlew assembleRelease \
|
||||
-PreactNativeArchitectures=arm64-v8a \
|
||||
--no-daemon --max-workers=2 -x lint --stacktrace
|
||||
--build-cache --parallel \
|
||||
--no-daemon --max-workers=2 -x lint --stacktrace \
|
||||
&& ccache --show-stats --verbose 2>/dev/null | grep -iE 'cacheable|hit|miss|cache size' || true
|
||||
|
||||
# The signed APK (handy for `docker create` + `docker cp` extraction in CI).
|
||||
# /workspace/android/app/build/outputs/apk/release/app-release.apk
|
||||
# The signed APK lands at:
|
||||
# /workspace/android/app/build/outputs/apk/release/app-release.apk
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apk : a tiny export-only stage holding JUST the signed APK.
|
||||
#
|
||||
# CI builds this stage with `--target apk --output type=local,dest=...`, which
|
||||
# writes only app-release.apk to the host — NO `--load` of the ~15GB build image
|
||||
# (saves the multi-minute image export/unpack on every push AND avoids that
|
||||
# image piling up on the server's disk). The COPY pulls from `build`, so the
|
||||
# whole build still runs; only the output is slimmed.
|
||||
FROM scratch AS apk
|
||||
COPY --from=build /workspace/android/app/build/outputs/apk/release/app-release.apk /app-release.apk
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
// https://docs.expo.dev/guides/using-eslint/
|
||||
const { defineConfig } = require('eslint/config');
|
||||
const expoConfig = require("eslint-config-expo/flat");
|
||||
|
||||
module.exports = defineConfig([
|
||||
expoConfig,
|
||||
{
|
||||
ignores: ["dist/*"],
|
||||
}
|
||||
]);
|
||||
@@ -16,6 +16,7 @@
|
||||
"expo-image": "~56.0.11",
|
||||
"expo-linking": "~56.0.14",
|
||||
"expo-router": "~56.2.10",
|
||||
"expo-sharing": "~56",
|
||||
"expo-splash-screen": "~56.0.10",
|
||||
"expo-sqlite": "^56.0.5",
|
||||
"expo-status-bar": "~56.0.4",
|
||||
@@ -25,6 +26,7 @@
|
||||
"react": "19.2.3",
|
||||
"react-dom": "19.2.3",
|
||||
"react-native": "0.85.3",
|
||||
"react-native-audio-api": "0.12.2",
|
||||
"react-native-gesture-handler": "~2.31.1",
|
||||
"react-native-reanimated": "4.3.1",
|
||||
"react-native-safe-area-context": "~5.7.0",
|
||||
@@ -38,6 +40,8 @@
|
||||
"devDependencies": {
|
||||
"@types/node": "^25.9.3",
|
||||
"@types/react": "~19.2.2",
|
||||
"eslint": "^9.0.0",
|
||||
"eslint-config-expo": "~56.0.4",
|
||||
"fake-indexeddb": "^6.2.5",
|
||||
"fast-check": "^4.8.0",
|
||||
"typescript": "~6.0.3",
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
// Bottom tab navigation for the primary destinations (mobile-first). Secondary
|
||||
// screens (Record, Transcript, Courses, Quiz, Bibliography) are pushed on the
|
||||
// root Stack from within these tabs. Route-group parens keep URLs unchanged
|
||||
// (/, /search, /study, /ask, /settings).
|
||||
import { Tabs } from 'expo-router';
|
||||
import { Text, type ColorValue } from 'react-native';
|
||||
|
||||
import { useTheme } from '@/hooks/use-theme';
|
||||
|
||||
const ACCENT = '#3c87f7';
|
||||
|
||||
function TabIcon({ glyph, color }: { glyph: string; color: ColorValue }) {
|
||||
return <Text style={{ fontSize: 20, color }}>{glyph}</Text>;
|
||||
}
|
||||
|
||||
export default function TabsLayout() {
|
||||
const theme = useTheme();
|
||||
return (
|
||||
<Tabs
|
||||
screenOptions={{
|
||||
tabBarActiveTintColor: ACCENT,
|
||||
tabBarInactiveTintColor: theme.textSecondary,
|
||||
tabBarStyle: { backgroundColor: theme.background, borderTopColor: theme.backgroundElement },
|
||||
headerStyle: { backgroundColor: theme.background },
|
||||
headerTitleStyle: { color: theme.text },
|
||||
headerTintColor: theme.text,
|
||||
headerShadowVisible: false,
|
||||
}}>
|
||||
<Tabs.Screen
|
||||
name="index"
|
||||
options={{ title: 'Wisp', tabBarLabel: 'Library', tabBarIcon: ({ color }) => <TabIcon glyph="📚" color={color} /> }}
|
||||
/>
|
||||
<Tabs.Screen
|
||||
name="search"
|
||||
options={{ title: 'Search', tabBarIcon: ({ color }) => <TabIcon glyph="🔎" color={color} /> }}
|
||||
/>
|
||||
<Tabs.Screen
|
||||
name="study"
|
||||
options={{ title: 'Study', tabBarIcon: ({ color }) => <TabIcon glyph="🎴" color={color} /> }}
|
||||
/>
|
||||
<Tabs.Screen
|
||||
name="ask"
|
||||
options={{ title: 'Ask', tabBarIcon: ({ color }) => <TabIcon glyph="✨" color={color} /> }}
|
||||
/>
|
||||
<Tabs.Screen
|
||||
name="settings"
|
||||
options={{ title: 'Settings', tabBarIcon: ({ color }) => <TabIcon glyph="⚙️" color={color} /> }}
|
||||
/>
|
||||
</Tabs>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
import { useFocusEffect, useRouter } from 'expo-router';
|
||||
import { useCallback, useState } from 'react';
|
||||
import {
|
||||
ActivityIndicator,
|
||||
Pressable,
|
||||
ScrollView,
|
||||
StyleSheet,
|
||||
TextInput,
|
||||
View,
|
||||
} from 'react-native';
|
||||
|
||||
import { ThemedText } from '@/components/themed-text';
|
||||
import { ThemedView } from '@/components/themed-view';
|
||||
import { MaxContentWidth, Spacing } from '@/constants/theme';
|
||||
import { useTheme } from '@/hooks/use-theme';
|
||||
import type { RagAnswer, RagCitation } from '@/lib/generation/engine';
|
||||
import { formatClock } from '@/lib/format';
|
||||
import { useCourses } from '@/stores/coursesStore';
|
||||
import { useAi } from '@/stores/aiStore';
|
||||
import { useEmbedding } from '@/stores/embeddingStore';
|
||||
|
||||
const ACCENT = '#3c87f7';
|
||||
|
||||
// Course filter is a course id, or `null` for "all courses".
|
||||
type CourseSel = string | null;
|
||||
|
||||
export default function AskScreen() {
|
||||
const theme = useTheme();
|
||||
const router = useRouter();
|
||||
|
||||
const courses = useCourses((s) => s.items);
|
||||
const refreshCourses = useCourses((s) => s.refresh);
|
||||
|
||||
// Whether any lectures have been indexed at all (used to nudge the user to
|
||||
// build the search index when an empty/ungrounded answer comes back).
|
||||
const pending = useEmbedding((s) => s.pending);
|
||||
const refreshPending = useEmbedding((s) => s.refreshPending);
|
||||
|
||||
// AI engine state (model download / readiness) is observed from the store so
|
||||
// the loading bar reflects an in-flight on-device model download.
|
||||
const modelStatus = useAi((s) => s.modelStatus);
|
||||
const progress = useAi((s) => s.progress);
|
||||
|
||||
const [question, setQuestion] = useState('');
|
||||
const [courseId, setCourseId] = useState<CourseSel>(null);
|
||||
const [answering, setAnswering] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [result, setResult] = useState<RagAnswer | null>(null);
|
||||
|
||||
useFocusEffect(
|
||||
useCallback(() => {
|
||||
void refreshCourses();
|
||||
void refreshPending();
|
||||
}, [refreshCourses, refreshPending]),
|
||||
);
|
||||
|
||||
const onAsk = useCallback(async () => {
|
||||
const q = question.trim();
|
||||
if (!q || answering) return;
|
||||
setAnswering(true);
|
||||
setError(null);
|
||||
setResult(null);
|
||||
try {
|
||||
const ans = await useAi.getState().ask(q, { courseId });
|
||||
setResult(ans);
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : 'Something went wrong.');
|
||||
} finally {
|
||||
setAnswering(false);
|
||||
}
|
||||
}, [question, courseId, answering]);
|
||||
|
||||
const openCitation = (c: RagCitation) =>
|
||||
router.push({
|
||||
pathname: '/transcript/[id]',
|
||||
params: { id: c.transcriptId, t: String(Math.floor(c.start)) },
|
||||
});
|
||||
|
||||
// The model is downloading/preparing on-device (WebLLM). Show a progress bar.
|
||||
const loadingModel = modelStatus === 'loading';
|
||||
|
||||
return (
|
||||
<ThemedView style={styles.fill}>
|
||||
<ScrollView contentContainerStyle={styles.content} keyboardShouldPersistTaps="handled">
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
Ask a question and get an answer grounded in your lectures — every claim links back to the
|
||||
moment it came from.
|
||||
</ThemedText>
|
||||
|
||||
{courses.length > 0 && (
|
||||
<ScrollView
|
||||
horizontal
|
||||
showsHorizontalScrollIndicator={false}
|
||||
contentContainerStyle={styles.filterBar}>
|
||||
<FilterChip label="All courses" active={courseId === null} onPress={() => setCourseId(null)} />
|
||||
{courses.map((c) => (
|
||||
<FilterChip
|
||||
key={c.id}
|
||||
label={c.name}
|
||||
active={courseId === c.id}
|
||||
onPress={() => setCourseId(c.id)}
|
||||
/>
|
||||
))}
|
||||
</ScrollView>
|
||||
)}
|
||||
|
||||
<TextInput
|
||||
value={question}
|
||||
onChangeText={setQuestion}
|
||||
onSubmitEditing={() => void onAsk()}
|
||||
returnKeyType="search"
|
||||
autoFocus
|
||||
multiline
|
||||
placeholder="Ask your lectures anything…"
|
||||
placeholderTextColor={theme.textSecondary}
|
||||
style={[styles.input, { color: theme.text, backgroundColor: theme.backgroundElement }]}
|
||||
/>
|
||||
|
||||
<Pressable
|
||||
onPress={() => void onAsk()}
|
||||
disabled={answering || question.trim() === ''}
|
||||
style={({ pressed }) => [
|
||||
styles.askBtn,
|
||||
{ opacity: answering || question.trim() === '' ? 0.5 : pressed ? 0.85 : 1 },
|
||||
]}>
|
||||
<ThemedText style={styles.askBtnText}>Ask</ThemedText>
|
||||
</Pressable>
|
||||
|
||||
{loadingModel && (
|
||||
<ThemedView type="backgroundElement" style={styles.card}>
|
||||
<ThemedText type="smallBold">Preparing on-device AI… {Math.round(progress * 100)}%</ThemedText>
|
||||
<ProgressBar value={progress} />
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
The model downloads once, then runs locally on your device.
|
||||
</ThemedText>
|
||||
</ThemedView>
|
||||
)}
|
||||
|
||||
{answering && !loadingModel && (
|
||||
<View style={styles.center}>
|
||||
<ActivityIndicator />
|
||||
<ThemedText type="small" themeColor="textSecondary" style={styles.centerText}>
|
||||
Thinking…
|
||||
</ThemedText>
|
||||
</View>
|
||||
)}
|
||||
|
||||
{error && (
|
||||
<ThemedView type="backgroundElement" style={styles.card}>
|
||||
<ThemedText type="smallBold">Couldn't answer that</ThemedText>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
{error}
|
||||
</ThemedText>
|
||||
</ThemedView>
|
||||
)}
|
||||
|
||||
{result && !answering && (
|
||||
<Answer
|
||||
result={result}
|
||||
indexEmpty={pending > 0}
|
||||
onOpenCitation={openCitation}
|
||||
/>
|
||||
)}
|
||||
</ScrollView>
|
||||
</ThemedView>
|
||||
);
|
||||
}
|
||||
|
||||
function Answer({
|
||||
result,
|
||||
indexEmpty,
|
||||
onOpenCitation,
|
||||
}: {
|
||||
result: RagAnswer;
|
||||
indexEmpty: boolean;
|
||||
onOpenCitation: (c: RagCitation) => void;
|
||||
}) {
|
||||
// Nothing relevant was retrieved — we refuse to invent an answer.
|
||||
if (!result.grounded) {
|
||||
return (
|
||||
<ThemedView type="backgroundElement" style={styles.card}>
|
||||
<ThemedText type="smallBold">
|
||||
I couldn't find anything about that in your lectures.
|
||||
</ThemedText>
|
||||
{indexEmpty && (
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
Tip: build the search index on the Search screen so your lectures become searchable.
|
||||
</ThemedText>
|
||||
)}
|
||||
</ThemedView>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<View style={styles.answerWrap}>
|
||||
{result.generated ? (
|
||||
<ThemedView type="backgroundElement" style={styles.answerCard}>
|
||||
<ThemedText type="default">{result.answer}</ThemedText>
|
||||
</ThemedView>
|
||||
) : (
|
||||
// Grounded but no LLM available: deterministic search-only fallback.
|
||||
<ThemedView type="backgroundElement" style={styles.card}>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
On-device AI needs WebGPU, or add an API key in Settings — here are the matching moments:
|
||||
</ThemedText>
|
||||
</ThemedView>
|
||||
)}
|
||||
|
||||
<ThemedText type="smallBold" style={styles.sourcesHeading}>
|
||||
Sources
|
||||
</ThemedText>
|
||||
{result.citations.map((c, i) => (
|
||||
<CitationChip
|
||||
key={`${c.transcriptId}:${c.segmentId ?? i}`}
|
||||
citation={c}
|
||||
onPress={() => onOpenCitation(c)}
|
||||
/>
|
||||
))}
|
||||
|
||||
<ThemedText type="small" themeColor="textSecondary" style={styles.disclaimer}>
|
||||
AI answers can be wrong — every claim links to the lecture; verify against the source.
|
||||
</ThemedText>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
function CitationChip({
|
||||
citation,
|
||||
onPress,
|
||||
}: {
|
||||
citation: RagCitation;
|
||||
onPress: () => void;
|
||||
}) {
|
||||
return (
|
||||
<Pressable onPress={onPress} style={({ pressed }) => [pressed && styles.pressed]}>
|
||||
<ThemedView type="backgroundElement" style={styles.card}>
|
||||
<ThemedText type="small" numberOfLines={3}>
|
||||
{citation.text}
|
||||
</ThemedText>
|
||||
<ThemedText type="small" style={styles.citationTime}>
|
||||
({formatClock(citation.start)})
|
||||
</ThemedText>
|
||||
</ThemedView>
|
||||
</Pressable>
|
||||
);
|
||||
}
|
||||
|
||||
function FilterChip({
|
||||
label,
|
||||
active,
|
||||
onPress,
|
||||
}: {
|
||||
label: string;
|
||||
active: boolean;
|
||||
onPress: () => void;
|
||||
}) {
|
||||
const theme = useTheme();
|
||||
return (
|
||||
<Pressable
|
||||
onPress={onPress}
|
||||
style={[styles.filterChip, { backgroundColor: active ? ACCENT : theme.backgroundElement }]}>
|
||||
<ThemedText type="small" style={active ? styles.chipActive : undefined}>
|
||||
{label}
|
||||
</ThemedText>
|
||||
</Pressable>
|
||||
);
|
||||
}
|
||||
|
||||
function ProgressBar({ value }: { value: number }) {
|
||||
return (
|
||||
<View style={styles.track}>
|
||||
<View style={[styles.bar, { width: `${Math.max(2, Math.min(100, value * 100))}%` }]} />
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
fill: { flex: 1 },
|
||||
content: {
|
||||
padding: Spacing.three,
|
||||
gap: Spacing.three,
|
||||
maxWidth: MaxContentWidth,
|
||||
width: '100%',
|
||||
alignSelf: 'center',
|
||||
},
|
||||
filterBar: { gap: Spacing.two, paddingVertical: Spacing.one, paddingRight: Spacing.three },
|
||||
filterChip: { paddingHorizontal: Spacing.three, paddingVertical: Spacing.one, borderRadius: 999 },
|
||||
chipActive: { color: '#fff', fontWeight: '700' },
|
||||
input: {
|
||||
borderRadius: Spacing.two,
|
||||
paddingHorizontal: Spacing.three,
|
||||
paddingVertical: Spacing.two,
|
||||
fontSize: 15,
|
||||
minHeight: 48,
|
||||
},
|
||||
askBtn: {
|
||||
backgroundColor: ACCENT,
|
||||
paddingVertical: Spacing.three,
|
||||
borderRadius: Spacing.three,
|
||||
alignItems: 'center',
|
||||
},
|
||||
askBtnText: { color: '#fff', fontWeight: '700', fontSize: 16 },
|
||||
card: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.two },
|
||||
answerWrap: { gap: Spacing.three },
|
||||
answerCard: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.two },
|
||||
sourcesHeading: { marginTop: Spacing.one },
|
||||
citationTime: { color: ACCENT, fontWeight: '700' },
|
||||
disclaimer: { marginTop: Spacing.one, fontStyle: 'italic' },
|
||||
center: { alignItems: 'center', gap: Spacing.two, paddingVertical: Spacing.four },
|
||||
centerText: { textAlign: 'center' },
|
||||
track: { height: 6, borderRadius: 3, backgroundColor: '#88888833', overflow: 'hidden' },
|
||||
bar: { height: 6, borderRadius: 3, backgroundColor: ACCENT },
|
||||
pressed: { opacity: 0.7 },
|
||||
});
|
||||
@@ -16,11 +16,10 @@ import { MaxContentWidth, Spacing } from '@/constants/theme';
|
||||
import { useTheme } from '@/hooks/use-theme';
|
||||
import type { TranscriptMeta } from '@/lib/db';
|
||||
import { formatClock } from '@/lib/format';
|
||||
import { MODELS } from '@/lib/models/catalog';
|
||||
import { pickAudio, type PickedAudio } from '@/lib/pickAudio';
|
||||
import { useCourses } from '@/stores/coursesStore';
|
||||
import { useTranscribe } from '@/stores/transcribeStore';
|
||||
import { useTranscripts, type CourseFilter } from '@/stores/transcriptsStore';
|
||||
import { useTranscripts } from '@/stores/transcriptsStore';
|
||||
|
||||
export default function LibraryScreen() {
|
||||
const theme = useTheme();
|
||||
@@ -44,7 +43,7 @@ export default function LibraryScreen() {
|
||||
const onNew = useCallback(async () => {
|
||||
if (busy) return;
|
||||
const p = await pickAudio();
|
||||
if (p) setPicked(p); // opens the pre-capture sheet
|
||||
if (p) setPicked(p);
|
||||
}, [busy]);
|
||||
|
||||
const onStart = useCallback(
|
||||
@@ -64,36 +63,38 @@ export default function LibraryScreen() {
|
||||
return (
|
||||
<ThemedView style={styles.fill}>
|
||||
<ScrollView contentContainerStyle={styles.content}>
|
||||
<View style={styles.headerRow}>
|
||||
<View style={styles.flex}>
|
||||
<ThemedText type="subtitle">Wisp</ThemedText>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
Private transcription — runs on your device, nothing uploaded.
|
||||
</ThemedText>
|
||||
</View>
|
||||
<View style={styles.headerLinks}>
|
||||
<Link href="/courses" asChild>
|
||||
<Pressable hitSlop={8}>
|
||||
<ThemedText type="link" themeColor="textSecondary">Courses</ThemedText>
|
||||
</Pressable>
|
||||
</Link>
|
||||
<Link href="/settings" asChild>
|
||||
<Pressable hitSlop={8}>
|
||||
<ThemedText type="link" themeColor="textSecondary">⚙ Settings</ThemedText>
|
||||
</Pressable>
|
||||
</Link>
|
||||
</View>
|
||||
<View style={styles.subHeader}>
|
||||
<ThemedText type="small" themeColor="textSecondary" style={styles.flex}>
|
||||
On your device — nothing uploaded.
|
||||
</ThemedText>
|
||||
<Link href="/courses" asChild>
|
||||
<Pressable hitSlop={8}>
|
||||
<ThemedText type="link" themeColor="textSecondary">Courses ›</ThemedText>
|
||||
</Pressable>
|
||||
</Link>
|
||||
</View>
|
||||
|
||||
<Pressable
|
||||
onPress={onNew}
|
||||
disabled={busy}
|
||||
style={({ pressed }) => [
|
||||
styles.newButton,
|
||||
{ backgroundColor: '#3c87f7', opacity: busy ? 0.5 : pressed ? 0.85 : 1 },
|
||||
]}>
|
||||
<ThemedText style={styles.newButtonText}>+ New transcription</ThemedText>
|
||||
</Pressable>
|
||||
<View style={styles.captureRow}>
|
||||
<Link href="/record" asChild>
|
||||
<Pressable
|
||||
disabled={busy}
|
||||
style={({ pressed }) => [
|
||||
styles.captureBtn,
|
||||
{ backgroundColor: '#e5484d', opacity: busy ? 0.5 : pressed ? 0.85 : 1 },
|
||||
]}>
|
||||
<ThemedText style={styles.captureText}>🎙 Record</ThemedText>
|
||||
</Pressable>
|
||||
</Link>
|
||||
<Pressable
|
||||
onPress={onNew}
|
||||
disabled={busy}
|
||||
style={({ pressed }) => [
|
||||
styles.captureBtn,
|
||||
{ backgroundColor: '#3c87f7', opacity: busy ? 0.5 : pressed ? 0.85 : 1 },
|
||||
]}>
|
||||
<ThemedText style={styles.captureText}>+ Import</ThemedText>
|
||||
</Pressable>
|
||||
</View>
|
||||
|
||||
{busy && <ActiveJob />}
|
||||
{job.status === 'error' && (
|
||||
@@ -103,18 +104,20 @@ export default function LibraryScreen() {
|
||||
</ThemedView>
|
||||
)}
|
||||
|
||||
<ScrollView horizontal showsHorizontalScrollIndicator={false} contentContainerStyle={styles.filterBar}>
|
||||
<FilterChip label="All" active={courseFilter === 'all'} onPress={() => void setCourseFilter('all')} />
|
||||
<FilterChip label="Unsorted" active={courseFilter === null} onPress={() => void setCourseFilter(null)} />
|
||||
{courses.map((c) => (
|
||||
<FilterChip key={c.id} label={c.name} active={courseFilter === c.id} onPress={() => void setCourseFilter(c.id)} />
|
||||
))}
|
||||
</ScrollView>
|
||||
{courses.length > 0 && (
|
||||
<ScrollView horizontal showsHorizontalScrollIndicator={false} contentContainerStyle={styles.filterBar}>
|
||||
<FilterChip label="All" active={courseFilter === 'all'} onPress={() => void setCourseFilter('all')} />
|
||||
<FilterChip label="Unsorted" active={courseFilter === null} onPress={() => void setCourseFilter(null)} />
|
||||
{courses.map((c) => (
|
||||
<FilterChip key={c.id} label={c.name} active={courseFilter === c.id} onPress={() => void setCourseFilter(c.id)} />
|
||||
))}
|
||||
</ScrollView>
|
||||
)}
|
||||
|
||||
<TextInput
|
||||
value={query}
|
||||
onChangeText={(t) => void setQuery(t)}
|
||||
placeholder="Search transcripts…"
|
||||
placeholder="Filter by title…"
|
||||
placeholderTextColor={theme.textSecondary}
|
||||
style={[styles.search, { color: theme.text, backgroundColor: theme.backgroundElement }]}
|
||||
/>
|
||||
@@ -123,7 +126,7 @@ export default function LibraryScreen() {
|
||||
<ActivityIndicator style={styles.pad} />
|
||||
) : items.length === 0 ? (
|
||||
<ThemedText type="small" themeColor="textSecondary" style={styles.pad}>
|
||||
{query ? 'No matches.' : 'No transcripts here yet. Pick an audio or video file to begin.'}
|
||||
{query ? 'No matches.' : 'No lectures yet. Record one or import an audio/video file to begin.'}
|
||||
</ThemedText>
|
||||
) : (
|
||||
items.map((t) => (
|
||||
@@ -160,19 +163,31 @@ function FilterChip({ label, active, onPress }: { label: string; active: boolean
|
||||
}
|
||||
|
||||
function ActiveJob() {
|
||||
const { stage, progress, partial, cancel } = useTranscribe();
|
||||
const { stage, progress, partial, chunkIndex, chunkCount, cancel } = useTranscribe();
|
||||
const pct = Math.round(progress * 100);
|
||||
const transcribing = stage === 'transcribing';
|
||||
// While transcribing, show "chunk i/N" so a long file's progress is legible
|
||||
// and it's obvious work is happening between the (discrete) per-chunk updates.
|
||||
const label = stage === 'loading' ? `Loading model… ${pct}%` : `Transcribing… ${pct}%`;
|
||||
const sub =
|
||||
transcribing && chunkCount
|
||||
? `part ${Math.min((chunkIndex ?? 0) + (pct < 100 ? 1 : 0), chunkCount)} of ${chunkCount}`
|
||||
: undefined;
|
||||
return (
|
||||
<ThemedView type="backgroundElement" style={styles.card}>
|
||||
<View style={styles.rowBetween}>
|
||||
<ThemedText type="smallBold">
|
||||
{stage === 'loading' ? 'Loading model…' : 'Transcribing…'} {pct}%
|
||||
</ThemedText>
|
||||
<View style={styles.jobTitleRow}>
|
||||
<ActivityIndicator size="small" />
|
||||
<ThemedText type="smallBold">{label}</ThemedText>
|
||||
</View>
|
||||
<Pressable onPress={cancel} hitSlop={8}>
|
||||
<ThemedText type="small" themeColor="textSecondary">Cancel</ThemedText>
|
||||
</Pressable>
|
||||
</View>
|
||||
<ProgressBar value={progress} />
|
||||
{sub && (
|
||||
<ThemedText type="small" themeColor="textSecondary">{sub}</ThemedText>
|
||||
)}
|
||||
{partial.length > 0 && (
|
||||
<ThemedText type="small" themeColor="textSecondary" numberOfLines={3}>
|
||||
{partial.map((s) => s.text).join(' ')}
|
||||
@@ -206,9 +221,7 @@ function TranscriptRow({
|
||||
<Pressable onPress={onOpen} style={({ pressed }) => [pressed && styles.pressed]}>
|
||||
<ThemedView type="backgroundElement" style={styles.card}>
|
||||
<View style={styles.rowBetween}>
|
||||
<ThemedText type="smallBold" numberOfLines={1} style={styles.flex}>
|
||||
{item.title}
|
||||
</ThemedText>
|
||||
<ThemedText type="smallBold" numberOfLines={1} style={styles.flex}>{item.title}</ThemedText>
|
||||
<Pressable onPress={onDelete} hitSlop={8}>
|
||||
<ThemedText type="small" themeColor="textSecondary">✕</ThemedText>
|
||||
</Pressable>
|
||||
@@ -230,18 +243,20 @@ const styles = StyleSheet.create({
|
||||
maxWidth: MaxContentWidth,
|
||||
width: '100%',
|
||||
alignSelf: 'center',
|
||||
paddingBottom: Spacing.six,
|
||||
},
|
||||
flex: { flex: 1 },
|
||||
headerRow: { flexDirection: 'row', alignItems: 'flex-start', gap: Spacing.two },
|
||||
headerLinks: { flexDirection: 'row', gap: Spacing.three, alignItems: 'center' },
|
||||
newButton: { paddingVertical: Spacing.three, borderRadius: Spacing.three, alignItems: 'center' },
|
||||
newButtonText: { color: '#fff', fontWeight: '700', fontSize: 16 },
|
||||
subHeader: { flexDirection: 'row', alignItems: 'center', gap: Spacing.two },
|
||||
captureRow: { flexDirection: 'row', gap: Spacing.two },
|
||||
captureBtn: { flex: 1, paddingVertical: Spacing.three, borderRadius: Spacing.three, alignItems: 'center' },
|
||||
captureText: { color: '#fff', fontWeight: '700', fontSize: 16 },
|
||||
card: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.two },
|
||||
rowBetween: { flexDirection: 'row', alignItems: 'center', justifyContent: 'space-between', gap: Spacing.two },
|
||||
jobTitleRow: { flexDirection: 'row', alignItems: 'center', gap: Spacing.two, flex: 1 },
|
||||
filterBar: { gap: Spacing.two, paddingVertical: Spacing.one, paddingRight: Spacing.three },
|
||||
filterChip: { paddingHorizontal: Spacing.three, paddingVertical: Spacing.one, borderRadius: 999 },
|
||||
filterChip: { paddingHorizontal: Spacing.three, paddingVertical: Spacing.two, borderRadius: 999 },
|
||||
chipActive: { color: '#fff', fontWeight: '700' },
|
||||
search: { borderRadius: Spacing.two, paddingHorizontal: Spacing.three, paddingVertical: Spacing.two, fontSize: 15 },
|
||||
search: { borderRadius: Spacing.two, paddingHorizontal: Spacing.three, paddingVertical: Spacing.three, fontSize: 15 },
|
||||
track: { height: 6, borderRadius: 3, backgroundColor: '#88888833', overflow: 'hidden' },
|
||||
bar: { height: 6, borderRadius: 3, backgroundColor: '#3c87f7' },
|
||||
pad: { paddingVertical: Spacing.four, textAlign: 'center' },
|
||||
@@ -0,0 +1,231 @@
|
||||
import { useFocusEffect, useRouter } from 'expo-router';
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { ActivityIndicator, Pressable, ScrollView, StyleSheet, TextInput, View } from 'react-native';
|
||||
|
||||
import { ThemedText } from '@/components/themed-text';
|
||||
import { ThemedView } from '@/components/themed-view';
|
||||
import { MaxContentWidth, Spacing } from '@/constants/theme';
|
||||
import { useTheme } from '@/hooks/use-theme';
|
||||
import { formatClock } from '@/lib/format';
|
||||
import { searchLectures } from '@/lib/search/search';
|
||||
import type { SearchHit } from '@/lib/search/types';
|
||||
import { useCourses } from '@/stores/coursesStore';
|
||||
import { useEmbedding } from '@/stores/embeddingStore';
|
||||
|
||||
const ACCENT = '#3c87f7';
|
||||
|
||||
export default function SearchScreen() {
|
||||
const theme = useTheme();
|
||||
const router = useRouter();
|
||||
|
||||
const courses = useCourses((s) => s.items);
|
||||
const refreshCourses = useCourses((s) => s.refresh);
|
||||
|
||||
const { status, progress, pending, refreshPending, buildIndex } = useEmbedding();
|
||||
|
||||
const [query, setQuery] = useState('');
|
||||
const [hits, setHits] = useState<SearchHit[]>([]);
|
||||
const [searching, setSearching] = useState(false);
|
||||
// Distinguish "haven't searched yet" from "searched, got nothing".
|
||||
const [searched, setSearched] = useState(false);
|
||||
|
||||
// Refresh courses + pending count whenever the screen gains focus.
|
||||
useFocusEffect(
|
||||
useCallback(() => {
|
||||
void refreshCourses();
|
||||
void refreshPending();
|
||||
}, [refreshCourses, refreshPending]),
|
||||
);
|
||||
|
||||
// Debounced search on query change. The latest run wins (stale guard via seq).
|
||||
const seq = useRef(0);
|
||||
useEffect(() => {
|
||||
const q = query.trim();
|
||||
if (!q) {
|
||||
setHits([]);
|
||||
setSearched(false);
|
||||
setSearching(false);
|
||||
return;
|
||||
}
|
||||
const mySeq = ++seq.current;
|
||||
setSearching(true);
|
||||
const handle = setTimeout(() => {
|
||||
void searchLectures(q)
|
||||
.then((res) => {
|
||||
if (seq.current !== mySeq) return; // a newer query superseded us
|
||||
setHits(res);
|
||||
setSearched(true);
|
||||
})
|
||||
.catch(() => {
|
||||
if (seq.current !== mySeq) return;
|
||||
setHits([]);
|
||||
setSearched(true);
|
||||
})
|
||||
.finally(() => {
|
||||
if (seq.current !== mySeq) return;
|
||||
setSearching(false);
|
||||
});
|
||||
}, 250);
|
||||
return () => clearTimeout(handle);
|
||||
}, [query]);
|
||||
|
||||
const runNow = useCallback(() => {
|
||||
const q = query.trim();
|
||||
if (!q) return;
|
||||
const mySeq = ++seq.current;
|
||||
setSearching(true);
|
||||
void searchLectures(q)
|
||||
.then((res) => {
|
||||
if (seq.current !== mySeq) return;
|
||||
setHits(res);
|
||||
setSearched(true);
|
||||
})
|
||||
.catch(() => {
|
||||
if (seq.current !== mySeq) return;
|
||||
setHits([]);
|
||||
setSearched(true);
|
||||
})
|
||||
.finally(() => {
|
||||
if (seq.current !== mySeq) return;
|
||||
setSearching(false);
|
||||
});
|
||||
}, [query]);
|
||||
|
||||
const courseName = (cid: string | null) =>
|
||||
cid ? courses.find((c) => c.id === cid)?.name ?? 'Course' : 'Unsorted';
|
||||
|
||||
const openHit = (hit: SearchHit) =>
|
||||
router.push({
|
||||
pathname: '/transcript/[id]',
|
||||
params: { id: hit.transcriptId, t: String(Math.floor(hit.start)) },
|
||||
});
|
||||
|
||||
const indexReady = status === 'ready' && pending === 0;
|
||||
|
||||
return (
|
||||
<ThemedView style={styles.fill}>
|
||||
<ScrollView contentContainerStyle={styles.content} keyboardShouldPersistTaps="handled">
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
Semantic search across your lectures — runs entirely on your device.
|
||||
</ThemedText>
|
||||
|
||||
{pending > 0 && (
|
||||
<ThemedView type="backgroundElement" style={styles.banner}>
|
||||
<ThemedText type="smallBold">
|
||||
Build search index ({pending} {pending === 1 ? 'lecture' : 'lectures'} pending)
|
||||
</ThemedText>
|
||||
{status === 'indexing' ? (
|
||||
<>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
Indexing… {Math.round(progress * 100)}%
|
||||
</ThemedText>
|
||||
<ProgressBar value={progress} />
|
||||
</>
|
||||
) : (
|
||||
<Pressable
|
||||
onPress={() => void buildIndex()}
|
||||
style={({ pressed }) => [styles.bannerBtn, { opacity: pressed ? 0.85 : 1 }]}>
|
||||
<ThemedText style={styles.bannerBtnText}>Build index</ThemedText>
|
||||
</Pressable>
|
||||
)}
|
||||
</ThemedView>
|
||||
)}
|
||||
|
||||
<TextInput
|
||||
value={query}
|
||||
onChangeText={setQuery}
|
||||
onSubmitEditing={runNow}
|
||||
returnKeyType="search"
|
||||
autoFocus
|
||||
placeholder="Search your lectures…"
|
||||
placeholderTextColor={theme.textSecondary}
|
||||
style={[styles.search, { color: theme.text, backgroundColor: theme.backgroundElement }]}
|
||||
/>
|
||||
|
||||
{searching ? (
|
||||
<ActivityIndicator style={styles.pad} />
|
||||
) : query.trim() === '' ? (
|
||||
<ThemedText type="small" themeColor="textSecondary" style={styles.pad}>
|
||||
{indexReady
|
||||
? 'Type a question or topic to search across your lectures.'
|
||||
: 'Build the index to search.'}
|
||||
</ThemedText>
|
||||
) : searched && hits.length === 0 ? (
|
||||
<ThemedText type="small" themeColor="textSecondary" style={styles.pad}>
|
||||
No matches.
|
||||
</ThemedText>
|
||||
) : (
|
||||
hits.map((hit) => (
|
||||
<ResultRow
|
||||
key={`${hit.transcriptId}:${hit.segmentId}`}
|
||||
hit={hit}
|
||||
courseName={courseName(hit.courseId)}
|
||||
onPress={() => openHit(hit)}
|
||||
/>
|
||||
))
|
||||
)}
|
||||
</ScrollView>
|
||||
</ThemedView>
|
||||
);
|
||||
}
|
||||
|
||||
function ResultRow({
|
||||
hit,
|
||||
courseName,
|
||||
onPress,
|
||||
}: {
|
||||
hit: SearchHit;
|
||||
courseName: string;
|
||||
onPress: () => void;
|
||||
}) {
|
||||
return (
|
||||
<Pressable onPress={onPress} style={({ pressed }) => [pressed && styles.pressed]}>
|
||||
<ThemedView type="backgroundElement" style={styles.card}>
|
||||
<ThemedText type="small" numberOfLines={2}>
|
||||
{hit.text}
|
||||
</ThemedText>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
{courseName} · {formatClock(hit.start)}
|
||||
</ThemedText>
|
||||
</ThemedView>
|
||||
</Pressable>
|
||||
);
|
||||
}
|
||||
|
||||
function ProgressBar({ value }: { value: number }) {
|
||||
return (
|
||||
<View style={styles.track}>
|
||||
<View style={[styles.bar, { width: `${Math.max(2, Math.min(100, value * 100))}%` }]} />
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
fill: { flex: 1 },
|
||||
content: {
|
||||
padding: Spacing.three,
|
||||
gap: Spacing.three,
|
||||
maxWidth: MaxContentWidth,
|
||||
width: '100%',
|
||||
alignSelf: 'center',
|
||||
},
|
||||
banner: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.two },
|
||||
bannerBtn: {
|
||||
backgroundColor: ACCENT,
|
||||
paddingVertical: Spacing.two,
|
||||
borderRadius: Spacing.two,
|
||||
alignItems: 'center',
|
||||
},
|
||||
bannerBtnText: { color: '#fff', fontWeight: '700' },
|
||||
search: {
|
||||
borderRadius: Spacing.two,
|
||||
paddingHorizontal: Spacing.three,
|
||||
paddingVertical: Spacing.two,
|
||||
fontSize: 15,
|
||||
},
|
||||
card: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.two },
|
||||
track: { height: 6, borderRadius: 3, backgroundColor: '#88888833', overflow: 'hidden' },
|
||||
bar: { height: 6, borderRadius: 3, backgroundColor: ACCENT },
|
||||
pad: { paddingVertical: Spacing.four, textAlign: 'center' },
|
||||
pressed: { opacity: 0.7 },
|
||||
});
|
||||
@@ -0,0 +1,272 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import { ScrollView, StyleSheet, Pressable, TextInput, View } from 'react-native';
|
||||
|
||||
import { ThemedText } from '@/components/themed-text';
|
||||
import { ThemedView } from '@/components/themed-view';
|
||||
import { MaxContentWidth, Spacing } from '@/constants/theme';
|
||||
import { useTheme } from '@/hooks/use-theme';
|
||||
import { listModels } from '@/lib/models/catalog';
|
||||
import { getEngine } from '@/lib/transcription';
|
||||
import type { Backend } from '@/lib/types';
|
||||
import { useAi } from '@/stores/aiStore';
|
||||
import { useTranscribe } from '@/stores/transcribeStore';
|
||||
|
||||
const ACCENT = '#3c87f7';
|
||||
const DEFAULT_BASE_URL = 'https://api.openai.com/v1';
|
||||
const DEFAULT_MODEL = 'gpt-4o-mini';
|
||||
|
||||
export default function SettingsScreen() {
|
||||
const theme = useTheme();
|
||||
const modelId = useTranscribe((s) => s.modelId);
|
||||
const setModel = useTranscribe((s) => s.setModel);
|
||||
const models = listModels();
|
||||
|
||||
// Download size depends on the resolved backend: the no-WebGPU WEB path pulls
|
||||
// full-precision (fp32) weights — roughly 2x the quantized figure in the
|
||||
// catalog — while WebGPU (fp16) and native (ggml) are close to it. Resolve the
|
||||
// backend once so the sizes we advertise aren't ~half the real download.
|
||||
const [sizeMult, setSizeMult] = useState(1);
|
||||
useEffect(() => {
|
||||
let alive = true;
|
||||
getEngine()
|
||||
.capabilities()
|
||||
.then((c: { backend: Backend }) => {
|
||||
if (alive) setSizeMult(c.backend === 'wasm' ? 2 : c.backend === 'webgpu' ? 1.4 : 1);
|
||||
})
|
||||
.catch(() => {});
|
||||
return () => {
|
||||
alive = false;
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<ThemedView style={styles.fill}>
|
||||
<ScrollView contentContainerStyle={styles.content}>
|
||||
<ThemedText type="subtitle">Model</ThemedText>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
Smaller models are faster and run well on any CPU; larger ones are more accurate but want a GPU.
|
||||
The model downloads once, then works fully offline.
|
||||
</ThemedText>
|
||||
|
||||
{models.map((m) => {
|
||||
const selected = m.id === modelId;
|
||||
return (
|
||||
<Pressable key={m.id} onPress={() => setModel(m.id)}>
|
||||
<ThemedView
|
||||
type={selected ? 'backgroundSelected' : 'backgroundElement'}
|
||||
style={[styles.card, selected && { borderColor: ACCENT, borderWidth: 1 }]}>
|
||||
<View style={styles.rowBetween}>
|
||||
<ThemedText type="smallBold">{m.label}</ThemedText>
|
||||
{selected && <ThemedText type="small" style={{ color: ACCENT }}>✓ selected</ThemedText>}
|
||||
</View>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
{cap(m.tier)} · ~{Math.round(m.approxMB * sizeMult)} MB · {m.multilingual ? 'multilingual' : 'English-only'}
|
||||
</ThemedText>
|
||||
</ThemedView>
|
||||
</Pressable>
|
||||
);
|
||||
})}
|
||||
|
||||
<View style={styles.spacer} />
|
||||
<AiSection />
|
||||
|
||||
<View style={styles.spacer} />
|
||||
<ThemedText type="subtitle">Privacy</ThemedText>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
Wisp transcribes entirely on your device using OpenAI's Whisper model. Your audio is never
|
||||
uploaded to any server — there is no account, no per-minute fee, and it keeps working with the
|
||||
network off. The only download is the model file itself.
|
||||
</ThemedText>
|
||||
</ScrollView>
|
||||
</ThemedView>
|
||||
);
|
||||
}
|
||||
|
||||
function AiSection() {
|
||||
const theme = useTheme();
|
||||
|
||||
const cloud = useAi((s) => s.cloud);
|
||||
const engineKind = useAi((s) => s.engineKind);
|
||||
const setCloud = useAi((s) => s.setCloud);
|
||||
const refreshAvailability = useAi((s) => s.refreshAvailability);
|
||||
|
||||
// Form fields seeded from any saved cloud config.
|
||||
const [baseUrl, setBaseUrl] = useState(cloud?.baseUrl ?? DEFAULT_BASE_URL);
|
||||
const [model, setModel] = useState(cloud?.model ?? DEFAULT_MODEL);
|
||||
const [apiKey, setApiKey] = useState(cloud?.apiKey ?? '');
|
||||
|
||||
// Probe availability (WebGPU / key set) when the screen mounts.
|
||||
useEffect(() => {
|
||||
void refreshAvailability();
|
||||
}, [refreshAvailability]);
|
||||
|
||||
// Keep the form in sync if the saved config changes elsewhere.
|
||||
useEffect(() => {
|
||||
setBaseUrl(cloud?.baseUrl ?? DEFAULT_BASE_URL);
|
||||
setModel(cloud?.model ?? DEFAULT_MODEL);
|
||||
setApiKey(cloud?.apiKey ?? '');
|
||||
}, [cloud]);
|
||||
|
||||
const engineLabel =
|
||||
engineKind === 'cloud'
|
||||
? 'Cloud (your key)'
|
||||
: engineKind === 'webllm'
|
||||
? 'On-device (WebGPU)'
|
||||
: 'Not available';
|
||||
|
||||
const onSave = () => {
|
||||
const url = baseUrl.trim() || DEFAULT_BASE_URL;
|
||||
const mdl = model.trim() || DEFAULT_MODEL;
|
||||
const key = apiKey.trim();
|
||||
if (!key) return;
|
||||
setCloud({ baseUrl: url, apiKey: key, model: mdl });
|
||||
void refreshAvailability();
|
||||
};
|
||||
|
||||
const onClear = () => {
|
||||
setCloud(undefined);
|
||||
setApiKey('');
|
||||
setBaseUrl(DEFAULT_BASE_URL);
|
||||
setModel(DEFAULT_MODEL);
|
||||
void refreshAvailability();
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<ThemedText type="subtitle">AI (optional)</ThemedText>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
Wisp can answer questions about your lectures with cited answers. This is fully optional —
|
||||
without it, you still get on-device semantic search. It runs on-device when your browser
|
||||
supports WebGPU, or through your own API key below.
|
||||
</ThemedText>
|
||||
|
||||
<ThemedView type="backgroundElement" style={styles.card}>
|
||||
<View style={styles.rowBetween}>
|
||||
<ThemedText type="smallBold">Active engine</ThemedText>
|
||||
<ThemedText type="small" style={{ color: engineKind === 'none' ? theme.textSecondary : ACCENT }}>
|
||||
{engineLabel}
|
||||
</ThemedText>
|
||||
</View>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
On-device generation needs a WebGPU-capable browser. If that isn't available, add an
|
||||
API key below to use a cloud model instead.
|
||||
</ThemedText>
|
||||
</ThemedView>
|
||||
|
||||
<ThemedText type="smallBold">Bring your own key</ThemedText>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
Use any OpenAI-compatible endpoint. Your key is stored locally on this device and is never
|
||||
shared with Wisp. Only your question and the retrieved lecture snippets are sent to the
|
||||
endpoint — never your raw audio or full transcripts.
|
||||
</ThemedText>
|
||||
|
||||
<Field label="Base URL">
|
||||
<TextInput
|
||||
value={baseUrl}
|
||||
onChangeText={setBaseUrl}
|
||||
autoCapitalize="none"
|
||||
autoCorrect={false}
|
||||
keyboardType="url"
|
||||
placeholder={DEFAULT_BASE_URL}
|
||||
placeholderTextColor={theme.textSecondary}
|
||||
style={[styles.input, { color: theme.text, backgroundColor: theme.backgroundElement }]}
|
||||
/>
|
||||
</Field>
|
||||
|
||||
<Field label="Model">
|
||||
<TextInput
|
||||
value={model}
|
||||
onChangeText={setModel}
|
||||
autoCapitalize="none"
|
||||
autoCorrect={false}
|
||||
placeholder={DEFAULT_MODEL}
|
||||
placeholderTextColor={theme.textSecondary}
|
||||
style={[styles.input, { color: theme.text, backgroundColor: theme.backgroundElement }]}
|
||||
/>
|
||||
</Field>
|
||||
|
||||
<Field label="API key">
|
||||
<TextInput
|
||||
value={apiKey}
|
||||
onChangeText={setApiKey}
|
||||
autoCapitalize="none"
|
||||
autoCorrect={false}
|
||||
secureTextEntry
|
||||
placeholder="sk-…"
|
||||
placeholderTextColor={theme.textSecondary}
|
||||
style={[styles.input, { color: theme.text, backgroundColor: theme.backgroundElement }]}
|
||||
/>
|
||||
</Field>
|
||||
|
||||
<View style={styles.btnRow}>
|
||||
<Pressable
|
||||
onPress={onSave}
|
||||
disabled={apiKey.trim() === ''}
|
||||
style={({ pressed }) => [
|
||||
styles.saveBtn,
|
||||
{ opacity: apiKey.trim() === '' ? 0.5 : pressed ? 0.85 : 1 },
|
||||
]}>
|
||||
<ThemedText style={styles.saveBtnText}>Save key</ThemedText>
|
||||
</Pressable>
|
||||
<Pressable
|
||||
onPress={onClear}
|
||||
disabled={!cloud}
|
||||
style={({ pressed }) => [
|
||||
styles.clearBtn,
|
||||
{ borderColor: theme.textSecondary, opacity: !cloud ? 0.5 : pressed ? 0.85 : 1 },
|
||||
]}>
|
||||
<ThemedText type="smallBold" themeColor="textSecondary">
|
||||
Clear
|
||||
</ThemedText>
|
||||
</Pressable>
|
||||
</View>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function Field({ label, children }: { label: string; children: React.ReactNode }) {
|
||||
return (
|
||||
<View style={styles.field}>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
{label}
|
||||
</ThemedText>
|
||||
{children}
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
function cap(s: string) {
|
||||
return s.charAt(0).toUpperCase() + s.slice(1);
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
fill: { flex: 1 },
|
||||
content: { padding: Spacing.three, gap: Spacing.two, maxWidth: MaxContentWidth, width: '100%', alignSelf: 'center' },
|
||||
card: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.one },
|
||||
rowBetween: { flexDirection: 'row', alignItems: 'center', justifyContent: 'space-between' },
|
||||
spacer: { height: Spacing.three },
|
||||
field: { gap: Spacing.one },
|
||||
input: {
|
||||
borderRadius: Spacing.two,
|
||||
paddingHorizontal: Spacing.three,
|
||||
paddingVertical: Spacing.two,
|
||||
fontSize: 15,
|
||||
},
|
||||
btnRow: { flexDirection: 'row', gap: Spacing.two, marginTop: Spacing.one },
|
||||
saveBtn: {
|
||||
flex: 1,
|
||||
backgroundColor: ACCENT,
|
||||
paddingVertical: Spacing.two,
|
||||
borderRadius: Spacing.two,
|
||||
alignItems: 'center',
|
||||
},
|
||||
saveBtnText: { color: '#fff', fontWeight: '700' },
|
||||
clearBtn: {
|
||||
paddingHorizontal: Spacing.four,
|
||||
paddingVertical: Spacing.two,
|
||||
borderRadius: Spacing.two,
|
||||
borderWidth: 1,
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,200 @@
|
||||
import { useFocusEffect, useRouter } from 'expo-router';
|
||||
import { useCallback, useState } from 'react';
|
||||
import { ActivityIndicator, Pressable, ScrollView, StyleSheet, View } from 'react-native';
|
||||
|
||||
import { ThemedText } from '@/components/themed-text';
|
||||
import { ThemedView } from '@/components/themed-view';
|
||||
import { MaxContentWidth, Spacing } from '@/constants/theme';
|
||||
import { useTheme } from '@/hooks/use-theme';
|
||||
import { getRepo, type Flashcard } from '@/lib/db';
|
||||
import { downloadText } from '@/lib/download';
|
||||
import { review } from '@/lib/learn';
|
||||
import { useCourses } from '@/stores/coursesStore';
|
||||
|
||||
const ACCENT = '#3c87f7';
|
||||
|
||||
// Course scope for the due queue: 'all' = no filter, null = Unsorted, string = course id.
|
||||
type Scope = 'all' | string | null;
|
||||
|
||||
const GRADES: { label: string; grade: 0 | 1 | 2 | 3 }[] = [
|
||||
{ label: 'Again', grade: 0 },
|
||||
{ label: 'Hard', grade: 1 },
|
||||
{ label: 'Good', grade: 2 },
|
||||
{ label: 'Easy', grade: 3 },
|
||||
];
|
||||
|
||||
export default function StudyScreen() {
|
||||
const theme = useTheme();
|
||||
const router = useRouter();
|
||||
|
||||
const courses = useCourses((s) => s.items);
|
||||
const refreshCourses = useCourses((s) => s.refresh);
|
||||
|
||||
const [scope, setScope] = useState<Scope>('all');
|
||||
const [queue, setQueue] = useState<Flashcard[] | null>(null);
|
||||
const [index, setIndex] = useState(0);
|
||||
const [revealed, setRevealed] = useState(false);
|
||||
|
||||
const load = useCallback(async (s: Scope) => {
|
||||
setQueue(null);
|
||||
setIndex(0);
|
||||
setRevealed(false);
|
||||
const cards = await getRepo().listDueFlashcards(
|
||||
s === 'all' ? {} : { courseId: s },
|
||||
);
|
||||
setQueue(cards);
|
||||
}, []);
|
||||
|
||||
// Reload the due queue and courses every time the screen gains focus.
|
||||
useFocusEffect(
|
||||
useCallback(() => {
|
||||
void refreshCourses();
|
||||
void load(scope);
|
||||
}, [refreshCourses, load, scope]),
|
||||
);
|
||||
|
||||
const pickScope = (s: Scope) => {
|
||||
setScope(s);
|
||||
void load(s);
|
||||
};
|
||||
|
||||
const total = queue?.length ?? 0;
|
||||
const card = queue && index < total ? queue[index] : undefined;
|
||||
|
||||
const grade = async (g: 0 | 1 | 2 | 3) => {
|
||||
if (!card) return;
|
||||
await getRepo().updateFlashcardSrs(card.id, review(card.srs, g, Date.now()));
|
||||
setRevealed(false);
|
||||
setIndex((i) => i + 1);
|
||||
};
|
||||
|
||||
const exportAnki = async () => {
|
||||
const all = await getRepo().listFlashcards();
|
||||
const rows = all.map((c) => `${csvField(c.front)},${csvField(c.back)}`);
|
||||
const csv = ['front,back', ...rows].join('\r\n');
|
||||
downloadText('wisp-flashcards.csv', 'text/csv', csv);
|
||||
};
|
||||
|
||||
const courseName = (cid?: string | null) =>
|
||||
cid ? courses.find((c) => c.id === cid)?.name ?? 'Course' : 'Unsorted';
|
||||
|
||||
return (
|
||||
<ThemedView style={styles.fill}>
|
||||
<ScrollView contentContainerStyle={styles.content}>
|
||||
<View style={styles.topActions}>
|
||||
<Pressable onPress={() => void exportAnki()} hitSlop={8}>
|
||||
<ThemedText type="link" themeColor="textSecondary">Export Anki (.csv)</ThemedText>
|
||||
</Pressable>
|
||||
</View>
|
||||
<ScrollView horizontal showsHorizontalScrollIndicator={false} contentContainerStyle={styles.filterBar}>
|
||||
<FilterChip label="All" active={scope === 'all'} onPress={() => pickScope('all')} />
|
||||
<FilterChip label="Unsorted" active={scope === null} onPress={() => pickScope(null)} />
|
||||
{courses.map((c) => (
|
||||
<FilterChip key={c.id} label={c.name} active={scope === c.id} onPress={() => pickScope(c.id)} />
|
||||
))}
|
||||
</ScrollView>
|
||||
|
||||
{queue === null ? (
|
||||
<ActivityIndicator style={styles.pad} />
|
||||
) : card === undefined ? (
|
||||
<ThemedText type="small" themeColor="textSecondary" style={styles.pad}>
|
||||
{total === 0
|
||||
? 'No cards due — generate some from a lecture.'
|
||||
: 'All done — no more cards due right now.'}
|
||||
</ThemedText>
|
||||
) : (
|
||||
<>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
{index + 1} of {total}
|
||||
</ThemedText>
|
||||
|
||||
<ThemedView type="backgroundElement" style={styles.card}>
|
||||
<ThemedText type="small" themeColor="textSecondary">{courseName(card.courseId)}</ThemedText>
|
||||
<ThemedText type="subtitle" style={styles.front}>{card.front}</ThemedText>
|
||||
|
||||
{revealed ? (
|
||||
<>
|
||||
<View style={[styles.divider, { backgroundColor: theme.backgroundSelected }]} />
|
||||
<ThemedText type="default">{card.back}</ThemedText>
|
||||
{card.start !== undefined && (
|
||||
<Pressable
|
||||
onPress={() =>
|
||||
router.push({
|
||||
pathname: '/transcript/[id]',
|
||||
params: { id: card.transcriptId, t: String(Math.floor(card.start ?? 0)) },
|
||||
})
|
||||
}
|
||||
hitSlop={6}>
|
||||
<ThemedText type="linkPrimary">Jump to lecture</ThemedText>
|
||||
</Pressable>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
<Pressable
|
||||
onPress={() => setRevealed(true)}
|
||||
style={({ pressed }) => [styles.showBtn, { backgroundColor: ACCENT, opacity: pressed ? 0.85 : 1 }]}>
|
||||
<ThemedText style={styles.showBtnText}>Show answer</ThemedText>
|
||||
</Pressable>
|
||||
)}
|
||||
</ThemedView>
|
||||
|
||||
{revealed && (
|
||||
<View style={styles.gradeRow}>
|
||||
{GRADES.map((g) => (
|
||||
<Pressable
|
||||
key={g.grade}
|
||||
onPress={() => void grade(g.grade)}
|
||||
style={({ pressed }) => [
|
||||
styles.gradeBtn,
|
||||
{ backgroundColor: theme.backgroundElement, opacity: pressed ? 0.7 : 1 },
|
||||
]}>
|
||||
<ThemedText type="smallBold">{g.label}</ThemedText>
|
||||
</Pressable>
|
||||
))}
|
||||
</View>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</ScrollView>
|
||||
</ThemedView>
|
||||
);
|
||||
}
|
||||
|
||||
function FilterChip({ label, active, onPress }: { label: string; active: boolean; onPress: () => void }) {
|
||||
const theme = useTheme();
|
||||
return (
|
||||
<Pressable
|
||||
onPress={onPress}
|
||||
style={[styles.filterChip, { backgroundColor: active ? ACCENT : theme.backgroundElement }]}>
|
||||
<ThemedText type="small" style={active ? styles.chipActive : undefined}>{label}</ThemedText>
|
||||
</Pressable>
|
||||
);
|
||||
}
|
||||
|
||||
/** RFC4180-quote a CSV field: wrap in quotes, double any embedded quotes. */
|
||||
function csvField(value: string): string {
|
||||
return `"${value.replace(/"/g, '""')}"`;
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
fill: { flex: 1 },
|
||||
content: {
|
||||
padding: Spacing.three,
|
||||
gap: Spacing.three,
|
||||
maxWidth: MaxContentWidth,
|
||||
width: '100%',
|
||||
alignSelf: 'center',
|
||||
},
|
||||
filterBar: { gap: Spacing.two, paddingVertical: Spacing.one, paddingRight: Spacing.three },
|
||||
filterChip: { paddingHorizontal: Spacing.three, paddingVertical: Spacing.one, borderRadius: 999 },
|
||||
topActions: { flexDirection: 'row', justifyContent: 'flex-end' },
|
||||
chipActive: { color: '#fff', fontWeight: '700' },
|
||||
card: { padding: Spacing.four, borderRadius: Spacing.three, gap: Spacing.three },
|
||||
front: { fontSize: 24, lineHeight: 32 },
|
||||
divider: { height: StyleSheet.hairlineWidth, marginVertical: Spacing.one },
|
||||
showBtn: { paddingVertical: Spacing.three, borderRadius: Spacing.three, alignItems: 'center' },
|
||||
showBtnText: { color: '#fff', fontWeight: '700', fontSize: 16 },
|
||||
gradeRow: { flexDirection: 'row', gap: Spacing.two },
|
||||
gradeBtn: { flex: 1, paddingVertical: Spacing.three, borderRadius: Spacing.two, alignItems: 'center' },
|
||||
pad: { paddingVertical: Spacing.four, textAlign: 'center' },
|
||||
});
|
||||
+4
-2
@@ -14,10 +14,12 @@ export default function RootLayout() {
|
||||
return (
|
||||
<ThemeProvider value={scheme === 'dark' ? DarkTheme : DefaultTheme}>
|
||||
<Stack>
|
||||
<Stack.Screen name="index" options={{ title: 'Wisp' }} />
|
||||
<Stack.Screen name="(tabs)" options={{ headerShown: false }} />
|
||||
<Stack.Screen name="record" options={{ title: 'Record' }} />
|
||||
<Stack.Screen name="transcript/[id]" options={{ title: 'Transcript' }} />
|
||||
<Stack.Screen name="courses" options={{ title: 'Courses' }} />
|
||||
<Stack.Screen name="settings" options={{ title: 'Settings' }} />
|
||||
<Stack.Screen name="quiz" options={{ title: 'Quiz' }} />
|
||||
<Stack.Screen name="bibliography" options={{ title: 'Bibliography' }} />
|
||||
</Stack>
|
||||
<StatusBar style="auto" />
|
||||
</ThemeProvider>
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
import * as Linking from 'expo-linking';
|
||||
import { Stack, useLocalSearchParams } from 'expo-router';
|
||||
import { useCallback, useState } from 'react';
|
||||
import { ActivityIndicator, Pressable, ScrollView, StyleSheet, View } from 'react-native';
|
||||
|
||||
import { ThemedText } from '@/components/themed-text';
|
||||
import { ThemedView } from '@/components/themed-view';
|
||||
import { MaxContentWidth, Spacing } from '@/constants/theme';
|
||||
import { useTheme } from '@/hooks/use-theme';
|
||||
import { getRepo } from '@/lib/db';
|
||||
import { downloadText } from '@/lib/download';
|
||||
import {
|
||||
detectCitations,
|
||||
lookupLinksFor,
|
||||
resolveCitation,
|
||||
toBibtex,
|
||||
toRis,
|
||||
type Citation,
|
||||
type LookupLinks,
|
||||
type RefMetadata,
|
||||
} from '@/lib/enrich';
|
||||
|
||||
const ACCENT = '#3c87f7';
|
||||
|
||||
/** A detected citation paired with its (best-effort) resolved metadata. */
|
||||
interface BibEntry {
|
||||
citation: Citation;
|
||||
meta?: RefMetadata;
|
||||
}
|
||||
|
||||
export default function BibliographyScreen() {
|
||||
const theme = useTheme();
|
||||
const { courseId } = useLocalSearchParams<{ courseId?: string }>();
|
||||
|
||||
const [entries, setEntries] = useState<BibEntry[] | null>(null);
|
||||
const [building, setBuilding] = useState(false);
|
||||
|
||||
// Collect citations across the course's (or all) transcripts, then resolve
|
||||
// each via the metadata APIs. Detection is local; lookups send only ids.
|
||||
const build = useCallback(async () => {
|
||||
setBuilding(true);
|
||||
setEntries(null);
|
||||
try {
|
||||
const repo = getRepo();
|
||||
const metas = courseId ? await repo.listByCourse(courseId) : await repo.list();
|
||||
|
||||
// Detect citations across every transcript body. Dedupe by kind+value so
|
||||
// a reference cited in several lectures shows once.
|
||||
const byKey = new Map<string, Citation>();
|
||||
for (const m of metas) {
|
||||
const full = await repo.get(m.id);
|
||||
if (!full) continue;
|
||||
for (const c of detectCitations(full.segments)) {
|
||||
const key = `${c.kind}:${c.value}`;
|
||||
if (!byKey.has(key)) byKey.set(key, c);
|
||||
}
|
||||
}
|
||||
const citations = [...byKey.values()];
|
||||
// Seed the list immediately (links work without lookups), then resolve.
|
||||
setEntries(citations.map((citation) => ({ citation })));
|
||||
|
||||
const resolved = await Promise.all(
|
||||
citations.map(async (citation) => {
|
||||
try {
|
||||
return { citation, meta: await resolveCitation(citation) };
|
||||
} catch {
|
||||
return { citation };
|
||||
}
|
||||
}),
|
||||
);
|
||||
setEntries(resolved);
|
||||
} finally {
|
||||
setBuilding(false);
|
||||
}
|
||||
}, [courseId]);
|
||||
|
||||
// Resolved metadata feeds the exporters; fall back to the raw citation text
|
||||
// for entries that couldn't be looked up so nothing is silently dropped.
|
||||
const exportable = (entries ?? [])
|
||||
.map((e) => e.meta ?? fallbackMeta(e.citation))
|
||||
.filter((m): m is RefMetadata => m !== undefined);
|
||||
|
||||
const exportBibtex = () => {
|
||||
if (exportable.length === 0) return;
|
||||
downloadText('bibliography.bib', 'application/x-bibtex', toBibtex(exportable));
|
||||
};
|
||||
|
||||
const exportRis = () => {
|
||||
if (exportable.length === 0) return;
|
||||
downloadText('bibliography.ris', 'application/x-research-info-systems', toRis(exportable));
|
||||
};
|
||||
|
||||
const openLink = (url: string) => {
|
||||
void Linking.openURL(url);
|
||||
};
|
||||
|
||||
return (
|
||||
<ThemedView style={styles.fill}>
|
||||
<Stack.Screen options={{ title: 'Bibliography' }} />
|
||||
<ScrollView contentContainerStyle={styles.content}>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
{courseId
|
||||
? 'References detected across this course’s lectures.'
|
||||
: 'References detected across all lectures.'}
|
||||
</ThemedText>
|
||||
|
||||
<View style={styles.row}>
|
||||
<Pressable
|
||||
onPress={() => void build()}
|
||||
disabled={building}
|
||||
style={({ pressed }) => [
|
||||
styles.btn,
|
||||
{ backgroundColor: ACCENT, opacity: building ? 0.6 : pressed ? 0.85 : 1 },
|
||||
]}>
|
||||
<ThemedText style={styles.btnText}>
|
||||
{building ? 'Scanning…' : entries ? 'Re-scan' : 'Build bibliography'}
|
||||
</ThemedText>
|
||||
</Pressable>
|
||||
{entries && entries.length > 0 && (
|
||||
<>
|
||||
<Pressable
|
||||
onPress={exportBibtex}
|
||||
style={({ pressed }) => [
|
||||
styles.btn,
|
||||
{ backgroundColor: theme.backgroundElement, opacity: pressed ? 0.7 : 1 },
|
||||
]}>
|
||||
<ThemedText type="smallBold">Export BibTeX</ThemedText>
|
||||
</Pressable>
|
||||
<Pressable
|
||||
onPress={exportRis}
|
||||
style={({ pressed }) => [
|
||||
styles.btn,
|
||||
{ backgroundColor: theme.backgroundElement, opacity: pressed ? 0.7 : 1 },
|
||||
]}>
|
||||
<ThemedText type="smallBold">Export RIS</ThemedText>
|
||||
</Pressable>
|
||||
</>
|
||||
)}
|
||||
</View>
|
||||
|
||||
{building && entries === null && <ActivityIndicator style={styles.pad} />}
|
||||
|
||||
{entries && entries.length === 0 && (
|
||||
<ThemedText type="small" themeColor="textSecondary" style={styles.pad}>
|
||||
No references found.
|
||||
</ThemedText>
|
||||
)}
|
||||
|
||||
{entries?.map((e, i) => {
|
||||
const links = lookupLinksFor(e.citation, e.meta);
|
||||
return (
|
||||
<ThemedView key={`${e.citation.kind}:${e.citation.value}:${i}`} type="backgroundElement" style={styles.card}>
|
||||
{e.meta?.title ? (
|
||||
<ThemedText type="smallBold">{e.meta.title}</ThemedText>
|
||||
) : (
|
||||
<ThemedText type="smallBold">{e.citation.raw}</ThemedText>
|
||||
)}
|
||||
{(e.meta?.authors?.length || e.meta?.year) && (
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
{[e.meta?.authors?.join(', '), e.meta?.year].filter(Boolean).join(' · ')}
|
||||
</ThemedText>
|
||||
)}
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
{e.meta?.source ?? e.citation.kind}
|
||||
</ThemedText>
|
||||
<View style={styles.linkChips}>
|
||||
{linkEntries(links).map(([label, url]) => (
|
||||
<Pressable
|
||||
key={label}
|
||||
onPress={() => openLink(url)}
|
||||
style={({ pressed }) => [
|
||||
styles.linkChip,
|
||||
{ backgroundColor: theme.backgroundSelected, opacity: pressed ? 0.7 : 1 },
|
||||
]}>
|
||||
<ThemedText type="small">{label}</ThemedText>
|
||||
</Pressable>
|
||||
))}
|
||||
</View>
|
||||
</ThemedView>
|
||||
);
|
||||
})}
|
||||
</ScrollView>
|
||||
</ThemedView>
|
||||
);
|
||||
}
|
||||
|
||||
/** Minimal metadata so an unresolved citation still exports something useful. */
|
||||
function fallbackMeta(c: Citation): RefMetadata {
|
||||
return { title: c.raw, source: c.kind };
|
||||
}
|
||||
|
||||
/** Ordered, labeled, defined link entries from a LookupLinks bag. */
|
||||
function linkEntries(links: LookupLinks): [string, string][] {
|
||||
const order: [keyof LookupLinks, string][] = [
|
||||
['doi', 'DOI'],
|
||||
['arxiv', 'arXiv'],
|
||||
['openAlex', 'OpenAlex'],
|
||||
['openLibrary', 'Open Library'],
|
||||
['googleBooks', 'Google Books'],
|
||||
['wikipedia', 'Wikipedia'],
|
||||
['annasArchive', "Anna's Archive"],
|
||||
['libgen', 'LibGen'],
|
||||
];
|
||||
return order
|
||||
.map(([key, label]) => [label, links[key]] as const)
|
||||
.filter((pair): pair is [string, string] => typeof pair[1] === 'string' && pair[1].length > 0)
|
||||
.map(([label, url]) => [label, url]);
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
fill: { flex: 1 },
|
||||
content: { padding: Spacing.three, gap: Spacing.two, maxWidth: MaxContentWidth, width: '100%', alignSelf: 'center' },
|
||||
row: { flexDirection: 'row', flexWrap: 'wrap', gap: Spacing.two, alignItems: 'center' },
|
||||
btn: { paddingHorizontal: Spacing.three, paddingVertical: Spacing.two, borderRadius: Spacing.two, alignItems: 'center' },
|
||||
btnText: { color: '#fff', fontWeight: '700', fontSize: 14 },
|
||||
card: { padding: Spacing.three, borderRadius: Spacing.three, gap: 2 },
|
||||
linkChips: { flexDirection: 'row', flexWrap: 'wrap', gap: Spacing.one, marginTop: Spacing.one },
|
||||
linkChip: { paddingHorizontal: Spacing.two, paddingVertical: 2, borderRadius: 999 },
|
||||
pad: { paddingVertical: Spacing.four, textAlign: 'center' },
|
||||
});
|
||||
+52
-5
@@ -1,4 +1,4 @@
|
||||
import { Stack, useFocusEffect } from 'expo-router';
|
||||
import { Link, Stack, useFocusEffect } from 'expo-router';
|
||||
import { useCallback, useState } from 'react';
|
||||
import { Pressable, ScrollView, StyleSheet, TextInput, View } from 'react-native';
|
||||
|
||||
@@ -6,19 +6,42 @@ import { ThemedText } from '@/components/themed-text';
|
||||
import { ThemedView } from '@/components/themed-view';
|
||||
import { MaxContentWidth, Spacing } from '@/constants/theme';
|
||||
import { useTheme } from '@/hooks/use-theme';
|
||||
import { getRepo } from '@/lib/db';
|
||||
import { useCourses } from '@/stores/coursesStore';
|
||||
|
||||
/** Per-course rollup shown on each row. */
|
||||
interface CourseStat {
|
||||
count: number;
|
||||
hours: number;
|
||||
}
|
||||
|
||||
export default function CoursesScreen() {
|
||||
const theme = useTheme();
|
||||
const { items, refresh, createCourse, rename, remove } = useCourses();
|
||||
const [name, setName] = useState('');
|
||||
const [editing, setEditing] = useState<string | null>(null);
|
||||
const [editName, setEditName] = useState('');
|
||||
const [stats, setStats] = useState<Record<string, CourseStat>>({});
|
||||
|
||||
// Load the courses list, then roll up lecture count + total hours per course
|
||||
// in a single pass (one listByCourse() per course; fetched once on focus).
|
||||
const reload = useCallback(async () => {
|
||||
await refresh();
|
||||
const courses = useCourses.getState().items;
|
||||
const entries = await Promise.all(
|
||||
courses.map(async (c) => {
|
||||
const metas = await getRepo().listByCourse(c.id);
|
||||
const hours = metas.reduce((sum, m) => sum + m.durationSec, 0) / 3600;
|
||||
return [c.id, { count: metas.length, hours }] as const;
|
||||
}),
|
||||
);
|
||||
setStats(Object.fromEntries(entries));
|
||||
}, [refresh]);
|
||||
|
||||
useFocusEffect(
|
||||
useCallback(() => {
|
||||
void refresh();
|
||||
}, [refresh]),
|
||||
void reload();
|
||||
}, [reload]),
|
||||
);
|
||||
|
||||
const add = async () => {
|
||||
@@ -26,6 +49,7 @@ export default function CoursesScreen() {
|
||||
if (!n) return;
|
||||
await createCourse({ name: n });
|
||||
setName('');
|
||||
await reload();
|
||||
};
|
||||
|
||||
return (
|
||||
@@ -74,8 +98,18 @@ export default function CoursesScreen() {
|
||||
</View>
|
||||
) : (
|
||||
<View style={styles.rowBetween}>
|
||||
<ThemedText type="smallBold" style={styles.flex} numberOfLines={1}>{c.name}</ThemedText>
|
||||
<View style={styles.flex}>
|
||||
<ThemedText type="smallBold" numberOfLines={1}>{c.name}</ThemedText>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
{statLine(stats[c.id])}
|
||||
</ThemedText>
|
||||
</View>
|
||||
<View style={styles.actions}>
|
||||
<Link href={{ pathname: '/bibliography', params: { courseId: c.id } }} asChild>
|
||||
<Pressable hitSlop={8}>
|
||||
<ThemedText type="small" themeColor="textSecondary">Bibliography</ThemedText>
|
||||
</Pressable>
|
||||
</Link>
|
||||
<Pressable
|
||||
onPress={() => {
|
||||
setEditing(c.id);
|
||||
@@ -84,7 +118,12 @@ export default function CoursesScreen() {
|
||||
hitSlop={8}>
|
||||
<ThemedText type="small" themeColor="textSecondary">Rename</ThemedText>
|
||||
</Pressable>
|
||||
<Pressable onPress={() => void remove(c.id)} hitSlop={8}>
|
||||
<Pressable
|
||||
onPress={async () => {
|
||||
await remove(c.id);
|
||||
await reload();
|
||||
}}
|
||||
hitSlop={8}>
|
||||
<ThemedText type="small" themeColor="textSecondary">Delete</ThemedText>
|
||||
</Pressable>
|
||||
</View>
|
||||
@@ -97,6 +136,14 @@ export default function CoursesScreen() {
|
||||
);
|
||||
}
|
||||
|
||||
/** "3 lectures · 4.2 h" — falls back to a zero state while stats load. */
|
||||
function statLine(stat: CourseStat | undefined): string {
|
||||
const count = stat?.count ?? 0;
|
||||
const hours = stat?.hours ?? 0;
|
||||
const lectures = `${count} ${count === 1 ? 'lecture' : 'lectures'}`;
|
||||
return `${lectures} · ${hours.toFixed(1)} h`;
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
fill: { flex: 1 },
|
||||
content: { padding: Spacing.three, gap: Spacing.two, maxWidth: MaxContentWidth, width: '100%', alignSelf: 'center' },
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
import { Stack, useLocalSearchParams, useRouter } from 'expo-router';
|
||||
import { useEffect, useMemo, useState } from 'react';
|
||||
import { ActivityIndicator, Pressable, ScrollView, StyleSheet, View } from 'react-native';
|
||||
|
||||
import { ThemedText } from '@/components/themed-text';
|
||||
import { ThemedView } from '@/components/themed-view';
|
||||
import { MaxContentWidth, Spacing } from '@/constants/theme';
|
||||
import { useTheme } from '@/hooks/use-theme';
|
||||
import { getRepo, type Transcript } from '@/lib/db';
|
||||
import { formatClock } from '@/lib/format';
|
||||
import { generateQuiz, glossary, type QuizQuestion } from '@/lib/learn';
|
||||
|
||||
const ACCENT = '#3c87f7';
|
||||
const CORRECT = '#30a46c';
|
||||
const WRONG = '#e5484d';
|
||||
|
||||
export default function QuizScreen() {
|
||||
const theme = useTheme();
|
||||
const router = useRouter();
|
||||
const { id } = useLocalSearchParams<{ id: string }>();
|
||||
|
||||
const [transcript, setTranscript] = useState<Transcript | null | undefined>(undefined);
|
||||
|
||||
useEffect(() => {
|
||||
let alive = true;
|
||||
void getRepo()
|
||||
.get(id)
|
||||
.then((t) => {
|
||||
if (alive) setTranscript(t ?? null);
|
||||
});
|
||||
return () => {
|
||||
alive = false;
|
||||
};
|
||||
}, [id]);
|
||||
|
||||
const questions = useMemo<QuizQuestion[]>(
|
||||
() => (transcript ? generateQuiz(glossary(transcript.segments)) : []),
|
||||
[transcript],
|
||||
);
|
||||
|
||||
const [index, setIndex] = useState(0);
|
||||
const [picked, setPicked] = useState<number | null>(null);
|
||||
const [score, setScore] = useState(0);
|
||||
|
||||
const total = questions.length;
|
||||
const q = index < total ? questions[index] : undefined;
|
||||
const finished = total > 0 && index >= total;
|
||||
|
||||
const onPick = (optionIndex: number) => {
|
||||
if (picked !== null || !q) return;
|
||||
setPicked(optionIndex);
|
||||
if (optionIndex === q.answerIndex) setScore((s) => s + 1);
|
||||
};
|
||||
|
||||
const next = () => {
|
||||
setPicked(null);
|
||||
setIndex((i) => i + 1);
|
||||
};
|
||||
|
||||
const restart = () => {
|
||||
setIndex(0);
|
||||
setPicked(null);
|
||||
setScore(0);
|
||||
};
|
||||
|
||||
if (transcript === undefined) {
|
||||
return (
|
||||
<Centered>
|
||||
<ActivityIndicator />
|
||||
</Centered>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<ThemedView style={styles.fill}>
|
||||
<Stack.Screen options={{ title: 'Quiz' }} />
|
||||
<ScrollView contentContainerStyle={styles.content}>
|
||||
{transcript === null ? (
|
||||
<ThemedText type="small" themeColor="textSecondary" style={styles.pad}>
|
||||
Transcript not found.
|
||||
</ThemedText>
|
||||
) : total === 0 ? (
|
||||
<ThemedText type="small" themeColor="textSecondary" style={styles.pad}>
|
||||
Not enough glossary terms to build a quiz for this lecture.
|
||||
</ThemedText>
|
||||
) : finished ? (
|
||||
<ThemedView type="backgroundElement" style={styles.card}>
|
||||
<ThemedText type="subtitle">Quiz complete</ThemedText>
|
||||
<ThemedText type="default">
|
||||
You scored {score} of {total}.
|
||||
</ThemedText>
|
||||
<Pressable
|
||||
onPress={restart}
|
||||
style={({ pressed }) => [styles.primaryBtn, { backgroundColor: ACCENT, opacity: pressed ? 0.85 : 1 }]}>
|
||||
<ThemedText style={styles.primaryBtnText}>Try again</ThemedText>
|
||||
</Pressable>
|
||||
<Pressable
|
||||
onPress={() => router.push({ pathname: '/transcript/[id]', params: { id } })}
|
||||
style={({ pressed }) => [styles.secondaryBtn, { backgroundColor: theme.backgroundSelected, opacity: pressed ? 0.85 : 1 }]}>
|
||||
<ThemedText type="smallBold">Back to lecture</ThemedText>
|
||||
</Pressable>
|
||||
</ThemedView>
|
||||
) : q ? (
|
||||
<>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
Question {index + 1} of {total}
|
||||
</ThemedText>
|
||||
<ThemedText type="subtitle" style={styles.question}>{q.question}</ThemedText>
|
||||
|
||||
<View style={styles.options}>
|
||||
{q.options.map((opt, i) => {
|
||||
const isAnswer = i === q.answerIndex;
|
||||
const isPicked = i === picked;
|
||||
const revealed = picked !== null;
|
||||
const border =
|
||||
revealed && isAnswer ? CORRECT : revealed && isPicked ? WRONG : theme.backgroundElement;
|
||||
return (
|
||||
<Pressable
|
||||
key={i}
|
||||
onPress={() => onPick(i)}
|
||||
disabled={revealed}
|
||||
style={({ pressed }) => [
|
||||
styles.option,
|
||||
{ backgroundColor: theme.backgroundElement, borderColor: border, opacity: pressed && !revealed ? 0.8 : 1 },
|
||||
]}>
|
||||
<ThemedText type="small">{opt}</ThemedText>
|
||||
</Pressable>
|
||||
);
|
||||
})}
|
||||
</View>
|
||||
|
||||
{picked !== null && (
|
||||
<ThemedView type="backgroundElement" style={styles.feedback}>
|
||||
<ThemedText type="smallBold" themeColor="textSecondary">
|
||||
{picked === q.answerIndex ? 'Correct' : 'Incorrect'}
|
||||
</ThemedText>
|
||||
{q.start !== undefined && (
|
||||
<Pressable
|
||||
onPress={() =>
|
||||
router.push({
|
||||
pathname: '/transcript/[id]',
|
||||
params: { id, t: String(Math.floor(q.start ?? 0)) },
|
||||
})
|
||||
}
|
||||
hitSlop={6}>
|
||||
<ThemedText type="linkPrimary">Jump to lecture ({formatClock(q.start)})</ThemedText>
|
||||
</Pressable>
|
||||
)}
|
||||
<Pressable
|
||||
onPress={next}
|
||||
style={({ pressed }) => [styles.primaryBtn, { backgroundColor: ACCENT, opacity: pressed ? 0.85 : 1 }]}>
|
||||
<ThemedText style={styles.primaryBtnText}>
|
||||
{index + 1 >= total ? 'See score' : 'Next'}
|
||||
</ThemedText>
|
||||
</Pressable>
|
||||
</ThemedView>
|
||||
)}
|
||||
</>
|
||||
) : null}
|
||||
</ScrollView>
|
||||
</ThemedView>
|
||||
);
|
||||
}
|
||||
|
||||
function Centered({ children }: { children: React.ReactNode }) {
|
||||
return <ThemedView style={[styles.fill, styles.centered]}>{children}</ThemedView>;
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
fill: { flex: 1 },
|
||||
centered: { alignItems: 'center', justifyContent: 'center' },
|
||||
content: {
|
||||
padding: Spacing.three,
|
||||
gap: Spacing.three,
|
||||
maxWidth: MaxContentWidth,
|
||||
width: '100%',
|
||||
alignSelf: 'center',
|
||||
},
|
||||
card: { padding: Spacing.four, borderRadius: Spacing.three, gap: Spacing.three },
|
||||
question: { fontSize: 24, lineHeight: 32 },
|
||||
options: { gap: Spacing.two },
|
||||
option: { padding: Spacing.three, borderRadius: Spacing.three, borderWidth: 2 },
|
||||
feedback: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.two },
|
||||
primaryBtn: { paddingVertical: Spacing.three, borderRadius: Spacing.three, alignItems: 'center' },
|
||||
primaryBtnText: { color: '#fff', fontWeight: '700', fontSize: 16 },
|
||||
secondaryBtn: { paddingVertical: Spacing.three, borderRadius: Spacing.three, alignItems: 'center' },
|
||||
pad: { paddingVertical: Spacing.four, textAlign: 'center' },
|
||||
});
|
||||
@@ -0,0 +1,143 @@
|
||||
import { Stack, useRouter } from 'expo-router';
|
||||
import { useState } from 'react';
|
||||
import { Pressable, StyleSheet, View } from 'react-native';
|
||||
|
||||
import { PreCaptureSheet, type PreCaptureResult } from '@/components/PreCaptureSheet';
|
||||
import { ThemedText } from '@/components/themed-text';
|
||||
import { ThemedView } from '@/components/themed-view';
|
||||
import { MaxContentWidth, Spacing } from '@/constants/theme';
|
||||
import { useTheme } from '@/hooks/use-theme';
|
||||
import { useRecorder, type RecordingResult } from '@/hooks/useRecorder';
|
||||
import { formatClock } from '@/lib/format';
|
||||
import { useTranscribe } from '@/stores/transcribeStore';
|
||||
|
||||
export default function RecordScreen() {
|
||||
const theme = useTheme();
|
||||
const router = useRouter();
|
||||
const rec = useRecorder();
|
||||
const [pending, setPending] = useState<RecordingResult | null>(null);
|
||||
|
||||
const onStop = async () => {
|
||||
const r = await rec.stop();
|
||||
if (r) setPending(r);
|
||||
else router.back();
|
||||
};
|
||||
|
||||
const onStart = async (meta: PreCaptureResult) => {
|
||||
const r = pending;
|
||||
setPending(null);
|
||||
if (!r) return;
|
||||
const id = await useTranscribe.getState().start({ data: r.data, name: r.name }, meta);
|
||||
if (id) router.replace({ pathname: '/transcript/[id]', params: { id } });
|
||||
else router.back();
|
||||
};
|
||||
|
||||
if (!rec.supported) {
|
||||
return (
|
||||
<ThemedView style={[styles.fill, styles.center]}>
|
||||
<Stack.Screen options={{ title: 'Record' }} />
|
||||
<ThemedText type="small" themeColor="textSecondary" style={styles.notice}>
|
||||
In-app recording is available in the browser for now. On mobile, use your phone's
|
||||
recorder and import the file (native live capture is coming).
|
||||
</ThemedText>
|
||||
</ThemedView>
|
||||
);
|
||||
}
|
||||
|
||||
const recording = rec.status === 'recording';
|
||||
const paused = rec.status === 'paused';
|
||||
|
||||
return (
|
||||
<ThemedView style={[styles.fill, styles.center]}>
|
||||
<Stack.Screen options={{ title: 'Record' }} />
|
||||
<View style={styles.stage}>
|
||||
<ThemedText style={styles.timer}>{formatClock(Math.floor(rec.elapsedMs / 1000))}</ThemedText>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
{recording ? '● Recording — on your device' : paused ? 'Paused' : 'Ready to record'}
|
||||
</ThemedText>
|
||||
|
||||
<View style={[styles.meterTrack, { backgroundColor: theme.backgroundElement }]}>
|
||||
<View
|
||||
style={[
|
||||
styles.meterFill,
|
||||
{ width: `${Math.round((recording ? rec.level : 0) * 100)}%`, backgroundColor: '#e5484d' },
|
||||
]}
|
||||
/>
|
||||
</View>
|
||||
|
||||
{rec.error && (
|
||||
<ThemedText type="small" style={styles.error}>{rec.error}</ThemedText>
|
||||
)}
|
||||
|
||||
<View style={styles.controls}>
|
||||
{rec.status === 'idle' || rec.status === 'stopped' ? (
|
||||
<BigButton label="Start recording" color="#e5484d" onPress={() => void rec.start()} />
|
||||
) : (
|
||||
<>
|
||||
{recording ? (
|
||||
<BigButton label="Pause" color={theme.backgroundElement} textColor={theme.text} onPress={rec.pause} />
|
||||
) : (
|
||||
<BigButton label="Resume" color="#3c87f7" onPress={rec.resume} />
|
||||
)}
|
||||
<BigButton label="Stop & transcribe" color="#3c87f7" onPress={() => void onStop()} />
|
||||
</>
|
||||
)}
|
||||
</View>
|
||||
|
||||
<ThemedText type="small" themeColor="textSecondary" style={styles.hint}>
|
||||
Keep this tab in front while recording. Everything stays on your device.
|
||||
</ThemedText>
|
||||
</View>
|
||||
|
||||
<PreCaptureSheet
|
||||
visible={pending !== null}
|
||||
defaultTitle={defaultName()}
|
||||
onStart={(m) => void onStart(m)}
|
||||
onCancel={() => {
|
||||
setPending(null);
|
||||
router.back();
|
||||
}}
|
||||
/>
|
||||
</ThemedView>
|
||||
);
|
||||
}
|
||||
|
||||
function BigButton({
|
||||
label,
|
||||
color,
|
||||
textColor,
|
||||
onPress,
|
||||
}: {
|
||||
label: string;
|
||||
color: string;
|
||||
textColor?: string;
|
||||
onPress: () => void;
|
||||
}) {
|
||||
return (
|
||||
<Pressable onPress={onPress} style={({ pressed }) => [styles.btn, { backgroundColor: color, opacity: pressed ? 0.85 : 1 }]}>
|
||||
<ThemedText style={[styles.btnText, textColor ? { color: textColor } : styles.btnTextLight]}>{label}</ThemedText>
|
||||
</Pressable>
|
||||
);
|
||||
}
|
||||
|
||||
function defaultName(): string {
|
||||
const d = new Date();
|
||||
const pad = (n: number) => String(n).padStart(2, '0');
|
||||
return `Lecture ${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
fill: { flex: 1 },
|
||||
center: { alignItems: 'center', justifyContent: 'center' },
|
||||
stage: { width: '100%', maxWidth: MaxContentWidth, padding: Spacing.four, gap: Spacing.three, alignItems: 'center' },
|
||||
timer: { fontSize: 56, fontWeight: '700', fontVariant: ['tabular-nums'] },
|
||||
meterTrack: { width: '80%', height: 8, borderRadius: 4, overflow: 'hidden' },
|
||||
meterFill: { height: 8, borderRadius: 4 },
|
||||
controls: { flexDirection: 'row', gap: Spacing.two, marginTop: Spacing.three, flexWrap: 'wrap', justifyContent: 'center' },
|
||||
btn: { paddingVertical: Spacing.three, paddingHorizontal: Spacing.four, borderRadius: Spacing.three, alignItems: 'center' },
|
||||
btnText: { fontWeight: '700', fontSize: 16 },
|
||||
btnTextLight: { color: '#fff' },
|
||||
error: { color: '#e5484d' },
|
||||
notice: { padding: Spacing.four, textAlign: 'center', maxWidth: 480 },
|
||||
hint: { textAlign: 'center', marginTop: Spacing.two },
|
||||
});
|
||||
@@ -1,66 +0,0 @@
|
||||
import { ScrollView, StyleSheet, Pressable, View } from 'react-native';
|
||||
|
||||
import { ThemedText } from '@/components/themed-text';
|
||||
import { ThemedView } from '@/components/themed-view';
|
||||
import { MaxContentWidth, Spacing } from '@/constants/theme';
|
||||
import { useTheme } from '@/hooks/use-theme';
|
||||
import { listModels } from '@/lib/models/catalog';
|
||||
import { useTranscribe } from '@/stores/transcribeStore';
|
||||
|
||||
export default function SettingsScreen() {
|
||||
const theme = useTheme();
|
||||
const modelId = useTranscribe((s) => s.modelId);
|
||||
const setModel = useTranscribe((s) => s.setModel);
|
||||
const models = listModels();
|
||||
|
||||
return (
|
||||
<ThemedView style={styles.fill}>
|
||||
<ScrollView contentContainerStyle={styles.content}>
|
||||
<ThemedText type="subtitle">Model</ThemedText>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
Smaller models are faster and run well on any CPU; larger ones are more accurate but want a GPU.
|
||||
The model downloads once, then works fully offline.
|
||||
</ThemedText>
|
||||
|
||||
{models.map((m) => {
|
||||
const selected = m.id === modelId;
|
||||
return (
|
||||
<Pressable key={m.id} onPress={() => setModel(m.id)}>
|
||||
<ThemedView
|
||||
type={selected ? 'backgroundSelected' : 'backgroundElement'}
|
||||
style={[styles.card, selected && { borderColor: '#3c87f7', borderWidth: 1 }]}>
|
||||
<View style={styles.rowBetween}>
|
||||
<ThemedText type="smallBold">{m.label}</ThemedText>
|
||||
{selected && <ThemedText type="small" style={{ color: '#3c87f7' }}>✓ selected</ThemedText>}
|
||||
</View>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
{cap(m.tier)} · ~{m.approxMB} MB · {m.multilingual ? 'multilingual' : 'English-only'}
|
||||
</ThemedText>
|
||||
</ThemedView>
|
||||
</Pressable>
|
||||
);
|
||||
})}
|
||||
|
||||
<View style={styles.spacer} />
|
||||
<ThemedText type="subtitle">Privacy</ThemedText>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
Wisp transcribes entirely on your device using OpenAI's Whisper model. Your audio is never
|
||||
uploaded to any server — there is no account, no per-minute fee, and it keeps working with the
|
||||
network off. The only download is the model file itself.
|
||||
</ThemedText>
|
||||
</ScrollView>
|
||||
</ThemedView>
|
||||
);
|
||||
}
|
||||
|
||||
function cap(s: string) {
|
||||
return s.charAt(0).toUpperCase() + s.slice(1);
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
fill: { flex: 1 },
|
||||
content: { padding: Spacing.three, gap: Spacing.two, maxWidth: MaxContentWidth, width: '100%', alignSelf: 'center' },
|
||||
card: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.one },
|
||||
rowBetween: { flexDirection: 'row', alignItems: 'center', justifyContent: 'space-between' },
|
||||
spacer: { height: Spacing.three },
|
||||
});
|
||||
+453
-10
@@ -1,4 +1,5 @@
|
||||
import { Stack, useLocalSearchParams } from 'expo-router';
|
||||
import * as Linking from 'expo-linking';
|
||||
import { Link, Stack, useLocalSearchParams } from 'expo-router';
|
||||
import { useEffect, useMemo, useRef, useState } from 'react';
|
||||
import {
|
||||
ActivityIndicator,
|
||||
@@ -16,22 +17,73 @@ import { MaxContentWidth, Spacing } from '@/constants/theme';
|
||||
import { useTheme } from '@/hooks/use-theme';
|
||||
import { getRepo, type Transcript } from '@/lib/db';
|
||||
import { downloadText } from '@/lib/download';
|
||||
import { errorMessage } from '@/lib/errorMessage';
|
||||
import {
|
||||
detectCitations,
|
||||
detectEvents,
|
||||
lookupLinksFor,
|
||||
resolveCitation,
|
||||
toIcs,
|
||||
type CandidateEvent,
|
||||
type Citation,
|
||||
type LookupLinks,
|
||||
type RefMetadata,
|
||||
} from '@/lib/enrich';
|
||||
import { EXPORT_META, formatTranscript, type ExportFormat } from '@/lib/export';
|
||||
import { formatClock } from '@/lib/format';
|
||||
import {
|
||||
cardsFromGlossary,
|
||||
glossary,
|
||||
summarize,
|
||||
type GlossaryEntry,
|
||||
type SourcedSentence,
|
||||
} from '@/lib/learn';
|
||||
import type { Segment } from '@/lib/types';
|
||||
import { useTranscribe } from '@/stores/transcribeStore';
|
||||
|
||||
const ACCENT = '#3c87f7';
|
||||
|
||||
export default function TranscriptScreen() {
|
||||
const theme = useTheme();
|
||||
const { id } = useLocalSearchParams<{ id: string }>();
|
||||
const { id, t } = useLocalSearchParams<{ id: string; t?: string }>();
|
||||
// Deep-link target time (seconds) from a search hit, if any.
|
||||
const jumpTo = useMemo(() => {
|
||||
const n = Number(t);
|
||||
return Number.isFinite(n) && n >= 0 ? n : null;
|
||||
}, [t]);
|
||||
|
||||
const [transcript, setTranscript] = useState<Transcript | null | undefined>(undefined);
|
||||
const [loadError, setLoadError] = useState<string | null>(null);
|
||||
const [title, setTitle] = useState('');
|
||||
const [segments, setSegments] = useState<Segment[]>([]);
|
||||
const [dirty, setDirty] = useState(false);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [currentTime, setCurrentTime] = useState(0);
|
||||
|
||||
// Study aids (deterministic, computed on demand — no storage, no model).
|
||||
const [aids, setAids] = useState<{
|
||||
summary: SourcedSentence[];
|
||||
glossary: GlossaryEntry[];
|
||||
} | null>(null);
|
||||
const [creatingCards, setCreatingCards] = useState(false);
|
||||
const [cardsAdded, setCardsAdded] = useState<number | null>(null);
|
||||
|
||||
// Dates & references (Phase 4) — detected on demand, nothing persisted.
|
||||
// `enrichment === null` means "not scanned yet".
|
||||
const [enrichment, setEnrichment] = useState<{
|
||||
events: CandidateEvent[];
|
||||
citations: Citation[];
|
||||
} | null>(null);
|
||||
// Per-citation resolved metadata + in-flight flags, keyed by list index.
|
||||
const [refMeta, setRefMeta] = useState<Record<number, RefMetadata>>({});
|
||||
const [refLoading, setRefLoading] = useState<Record<number, boolean>>({});
|
||||
|
||||
const scrollRef = useRef<ScrollView | null>(null);
|
||||
// Y offset of each rendered segment row, captured via onLayout, for scroll-to.
|
||||
const segmentYs = useRef<Record<number, number>>({});
|
||||
// Ensure we only auto-jump once per (id, t) deep-link.
|
||||
const jumpedRef = useRef(false);
|
||||
|
||||
// Prefer the just-transcribed in-session audio; otherwise load the persisted
|
||||
// source media so playback/scrub works after a reload (ROADMAP Phase 0).
|
||||
const sessionAudioUrl = useTranscribe((s) => (s.lastTranscriptId === id ? s.audioUrl : undefined));
|
||||
@@ -49,6 +101,9 @@ export default function TranscriptScreen() {
|
||||
if (!alive) return;
|
||||
url = u;
|
||||
setPersistedUrl(u);
|
||||
})
|
||||
.catch(() => {
|
||||
// Missing/failed media is non-fatal — playback just stays unavailable.
|
||||
});
|
||||
return () => {
|
||||
alive = false;
|
||||
@@ -65,6 +120,12 @@ export default function TranscriptScreen() {
|
||||
setTranscript(t ?? null);
|
||||
setTitle(t?.title ?? '');
|
||||
setSegments(t?.segments ?? []);
|
||||
})
|
||||
.catch((e) => {
|
||||
// Without this, a rejected load left the screen on its spinner forever.
|
||||
if (!alive) return;
|
||||
setLoadError(errorMessage(e));
|
||||
setTranscript(null);
|
||||
});
|
||||
return () => {
|
||||
alive = false;
|
||||
@@ -90,13 +151,48 @@ export default function TranscriptScreen() {
|
||||
[segments, currentTime],
|
||||
);
|
||||
|
||||
const seek = (t: number) => {
|
||||
const seek = (time: number) => {
|
||||
const el = audioRef.current;
|
||||
if (!el) return;
|
||||
el.currentTime = t;
|
||||
el.currentTime = time;
|
||||
void el.play();
|
||||
};
|
||||
|
||||
// Reset the one-shot jump guard whenever the deep-link target changes.
|
||||
useEffect(() => {
|
||||
jumpedRef.current = false;
|
||||
}, [id, jumpTo]);
|
||||
|
||||
// Deep-link jump: once the transcript is loaded and we have a finite `t`,
|
||||
// seek the audio (if present), highlight the matching segment, and scroll it
|
||||
// into view. Works even without audio (scroll/highlight by time only).
|
||||
useEffect(() => {
|
||||
if (jumpTo === null || jumpedRef.current) return;
|
||||
if (!transcript || segments.length === 0) return;
|
||||
|
||||
const target = segments.findIndex((s) => jumpTo >= s.start && jumpTo < s.end);
|
||||
const idx = target >= 0 ? target : nearestIndex(segments, jumpTo);
|
||||
if (idx < 0) return;
|
||||
|
||||
jumpedRef.current = true;
|
||||
// Drive the highlight by time even if audio isn't available.
|
||||
setCurrentTime(jumpTo);
|
||||
if (audioRef.current) seek(jumpTo);
|
||||
|
||||
// Scroll once the row's Y offset has been measured (next frames).
|
||||
const scrollToIdx = () => {
|
||||
const y = segmentYs.current[idx];
|
||||
if (y === undefined) return false;
|
||||
scrollRef.current?.scrollTo({ y: Math.max(0, y - Spacing.four), animated: true });
|
||||
return true;
|
||||
};
|
||||
if (!scrollToIdx()) {
|
||||
// Layout may not be measured yet; retry on the next ticks.
|
||||
const timers = [requestAnimationFrame(() => { if (!scrollToIdx()) setTimeout(scrollToIdx, 120); })];
|
||||
return () => timers.forEach(cancelAnimationFrame);
|
||||
}
|
||||
}, [jumpTo, transcript, segments, audioUrl]);
|
||||
|
||||
const editSegment = (i: number, text: string) => {
|
||||
setSegments((prev) => prev.map((s, idx) => (idx === i ? { ...s, text } : s)));
|
||||
setDirty(true);
|
||||
@@ -119,22 +215,93 @@ export default function TranscriptScreen() {
|
||||
downloadText(`${safeName}.${meta.ext}`, meta.mime, content);
|
||||
};
|
||||
|
||||
// Compute summary + glossary on demand (pure helpers, nothing persisted).
|
||||
const generateAids = () => {
|
||||
setAids({ summary: summarize(segments), glossary: glossary(segments) });
|
||||
setCardsAdded(null);
|
||||
};
|
||||
|
||||
// Seeds derived from the current glossary — drives the "Create N flashcards" label.
|
||||
const cardSeeds = useMemo(
|
||||
() => (aids ? cardsFromGlossary(aids.glossary) : []),
|
||||
[aids],
|
||||
);
|
||||
|
||||
const createCards = async () => {
|
||||
if (cardSeeds.length === 0 || creatingCards) return;
|
||||
setCreatingCards(true);
|
||||
try {
|
||||
const created = await getRepo().createFlashcards(
|
||||
cardSeeds.map((s) => ({
|
||||
transcriptId: id,
|
||||
courseId: transcript?.courseId ?? null,
|
||||
segmentId: s.segmentId,
|
||||
start: s.start,
|
||||
front: s.front,
|
||||
back: s.back,
|
||||
})),
|
||||
);
|
||||
setCardsAdded(created.length);
|
||||
} finally {
|
||||
setCreatingCards(false);
|
||||
}
|
||||
};
|
||||
|
||||
// Scan segments for deadlines + citations. Dates are anchored to the lecture
|
||||
// date when known, else the transcript's creation time. Pure & local.
|
||||
const scanEnrichment = () => {
|
||||
const anchorMs = transcript?.lectureDate ?? transcript?.createdAt ?? Date.now();
|
||||
setEnrichment({
|
||||
events: detectEvents(segments, anchorMs),
|
||||
citations: detectCitations(segments),
|
||||
});
|
||||
setRefMeta({});
|
||||
setRefLoading({});
|
||||
};
|
||||
|
||||
// Download a calendar file for one or more events.
|
||||
const exportIcs = (events: CandidateEvent[]) => {
|
||||
if (events.length === 0) return;
|
||||
const ics = toIcs(
|
||||
events.map((e) => ({ title: e.title, dateMs: e.dateMs, allDay: e.allDay })),
|
||||
);
|
||||
downloadText('event.ics', 'text/calendar', ics);
|
||||
};
|
||||
|
||||
// Network lookup for a single citation (sends only the id string, never text).
|
||||
const lookupRef = async (i: number, c: Citation) => {
|
||||
if (refLoading[i]) return;
|
||||
setRefLoading((prev) => ({ ...prev, [i]: true }));
|
||||
try {
|
||||
const meta = await resolveCitation(c);
|
||||
if (meta) setRefMeta((prev) => ({ ...prev, [i]: meta }));
|
||||
} finally {
|
||||
setRefLoading((prev) => ({ ...prev, [i]: false }));
|
||||
}
|
||||
};
|
||||
|
||||
const openLink = (url: string) => {
|
||||
void Linking.openURL(url);
|
||||
};
|
||||
|
||||
if (transcript === undefined) return <Centered><ActivityIndicator /></Centered>;
|
||||
if (transcript === null)
|
||||
return (
|
||||
<Centered>
|
||||
<ThemedText type="small" themeColor="textSecondary">Transcript not found.</ThemedText>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
{loadError ? `Couldn't load this transcript: ${loadError}` : 'Transcript not found.'}
|
||||
</ThemedText>
|
||||
</Centered>
|
||||
);
|
||||
|
||||
return (
|
||||
<ThemedView style={styles.fill}>
|
||||
<Stack.Screen options={{ title: title || 'Transcript' }} />
|
||||
<ScrollView contentContainerStyle={styles.content}>
|
||||
<ScrollView ref={scrollRef} contentContainerStyle={styles.content}>
|
||||
<TextInput
|
||||
value={title}
|
||||
onChangeText={(t) => {
|
||||
setTitle(t);
|
||||
onChangeText={(next) => {
|
||||
setTitle(next);
|
||||
setDirty(true);
|
||||
}}
|
||||
style={[styles.title, { color: theme.text }]}
|
||||
@@ -159,8 +326,226 @@ export default function TranscriptScreen() {
|
||||
))}
|
||||
</View>
|
||||
|
||||
{/* Study aids — deterministic helpers, computed on demand. */}
|
||||
<ThemedView type="backgroundElement" style={styles.aidsCard}>
|
||||
<ThemedText type="smallBold">Study aids</ThemedText>
|
||||
<View style={styles.aidsRow}>
|
||||
<Pressable
|
||||
onPress={generateAids}
|
||||
disabled={segments.length === 0}
|
||||
style={({ pressed }) => [
|
||||
styles.aidBtn,
|
||||
{ backgroundColor: ACCENT, opacity: segments.length === 0 ? 0.5 : pressed ? 0.85 : 1 },
|
||||
]}>
|
||||
<ThemedText style={styles.aidBtnText}>
|
||||
{aids ? 'Regenerate summary & glossary' : 'Generate summary & glossary'}
|
||||
</ThemedText>
|
||||
</Pressable>
|
||||
<Link href={{ pathname: '/quiz', params: { id } }} asChild>
|
||||
<Pressable
|
||||
style={({ pressed }) => [
|
||||
styles.aidBtn,
|
||||
{ backgroundColor: theme.backgroundSelected, opacity: pressed ? 0.85 : 1 },
|
||||
]}>
|
||||
<ThemedText style={styles.aidBtnTextAlt}>Quiz this lecture</ThemedText>
|
||||
</Pressable>
|
||||
</Link>
|
||||
</View>
|
||||
|
||||
{aids && (
|
||||
<>
|
||||
{aids.summary.length > 0 ? (
|
||||
<View style={styles.aidSection}>
|
||||
<ThemedText type="smallBold" themeColor="textSecondary">Summary</ThemedText>
|
||||
{aids.summary.map((s, i) => (
|
||||
<Pressable key={`sum-${i}`} onPress={() => seek(s.start)} hitSlop={4}>
|
||||
<View style={styles.aidLine}>
|
||||
<ThemedText type="code" themeColor="textSecondary" style={styles.ts}>
|
||||
{formatClock(s.start)}
|
||||
</ThemedText>
|
||||
<ThemedText type="small" style={styles.flex}>{s.text}</ThemedText>
|
||||
</View>
|
||||
</Pressable>
|
||||
))}
|
||||
</View>
|
||||
) : (
|
||||
<ThemedText type="small" themeColor="textSecondary">No summary could be derived.</ThemedText>
|
||||
)}
|
||||
|
||||
{aids.glossary.length > 0 ? (
|
||||
<View style={styles.aidSection}>
|
||||
<ThemedText type="smallBold" themeColor="textSecondary">Glossary</ThemedText>
|
||||
{aids.glossary.map((g, i) => (
|
||||
<Pressable key={`glo-${i}`} onPress={() => seek(g.start)} hitSlop={4}>
|
||||
<View style={styles.aidLine}>
|
||||
<ThemedText type="code" themeColor="textSecondary" style={styles.ts}>
|
||||
{formatClock(g.start)}
|
||||
</ThemedText>
|
||||
<ThemedText type="small" style={styles.flex}>
|
||||
<ThemedText type="smallBold">{g.term}</ThemedText>
|
||||
{` — ${g.definition}`}
|
||||
</ThemedText>
|
||||
</View>
|
||||
</Pressable>
|
||||
))}
|
||||
</View>
|
||||
) : (
|
||||
<ThemedText type="small" themeColor="textSecondary">No glossary terms found.</ThemedText>
|
||||
)}
|
||||
|
||||
{cardSeeds.length > 0 && (
|
||||
<Pressable
|
||||
onPress={() => void createCards()}
|
||||
disabled={creatingCards}
|
||||
style={({ pressed }) => [
|
||||
styles.aidBtn,
|
||||
{ backgroundColor: ACCENT, opacity: creatingCards ? 0.6 : pressed ? 0.85 : 1 },
|
||||
]}>
|
||||
<ThemedText style={styles.aidBtnText}>
|
||||
{creatingCards ? 'Adding…' : `Create ${cardSeeds.length} flashcards`}
|
||||
</ThemedText>
|
||||
</Pressable>
|
||||
)}
|
||||
|
||||
{cardsAdded !== null && (
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
Added {cardsAdded} {cardsAdded === 1 ? 'card' : 'cards'}.
|
||||
</ThemedText>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</ThemedView>
|
||||
|
||||
{/* Dates & references — detected on demand (pure/local detection). */}
|
||||
<ThemedView type="backgroundElement" style={styles.aidsCard}>
|
||||
<ThemedText type="smallBold">Dates & references</ThemedText>
|
||||
<Pressable
|
||||
onPress={scanEnrichment}
|
||||
disabled={segments.length === 0}
|
||||
style={({ pressed }) => [
|
||||
styles.aidBtn,
|
||||
{ backgroundColor: ACCENT, opacity: segments.length === 0 ? 0.5 : pressed ? 0.85 : 1 },
|
||||
]}>
|
||||
<ThemedText style={styles.aidBtnText}>
|
||||
{enrichment ? 'Re-scan for dates & references' : 'Scan for dates & references'}
|
||||
</ThemedText>
|
||||
</Pressable>
|
||||
|
||||
{enrichment && (
|
||||
<>
|
||||
{/* --- Dates --- */}
|
||||
<View style={styles.aidSection}>
|
||||
<View style={styles.refHeader}>
|
||||
<ThemedText type="smallBold" themeColor="textSecondary">Dates</ThemedText>
|
||||
{enrichment.events.length > 0 && (
|
||||
<Pressable onPress={() => exportIcs(enrichment.events)} hitSlop={6}>
|
||||
<ThemedText type="linkPrimary">Add all ({enrichment.events.length})</ThemedText>
|
||||
</Pressable>
|
||||
)}
|
||||
</View>
|
||||
{enrichment.events.length === 0 ? (
|
||||
<ThemedText type="small" themeColor="textSecondary">No dates found.</ThemedText>
|
||||
) : (
|
||||
enrichment.events.map((e, i) => (
|
||||
<View key={`evt-${i}`} style={styles.refRow}>
|
||||
<Pressable
|
||||
onPress={() => e.start !== undefined && seek(e.start)}
|
||||
disabled={e.start === undefined}
|
||||
hitSlop={4}
|
||||
style={styles.flex}>
|
||||
<ThemedText type="small">{e.title}</ThemedText>
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
{formatEventDate(e)}
|
||||
{e.start !== undefined ? ` · ${formatClock(e.start)}` : ''}
|
||||
</ThemedText>
|
||||
</Pressable>
|
||||
<Pressable onPress={() => exportIcs([e])} hitSlop={6}>
|
||||
<ThemedText type="link" themeColor="textSecondary">Add to calendar</ThemedText>
|
||||
</Pressable>
|
||||
</View>
|
||||
))
|
||||
)}
|
||||
</View>
|
||||
|
||||
{/* --- References --- */}
|
||||
<View style={styles.aidSection}>
|
||||
<ThemedText type="smallBold" themeColor="textSecondary">References</ThemedText>
|
||||
{enrichment.citations.length === 0 ? (
|
||||
<ThemedText type="small" themeColor="textSecondary">No references found.</ThemedText>
|
||||
) : (
|
||||
enrichment.citations.map((c, i) => {
|
||||
const meta = refMeta[i];
|
||||
const links = lookupLinksFor(c, meta);
|
||||
return (
|
||||
<View key={`cit-${i}`} style={styles.citCard}>
|
||||
<View style={styles.refRow}>
|
||||
<Pressable
|
||||
onPress={() => c.start !== undefined && seek(c.start)}
|
||||
disabled={c.start === undefined}
|
||||
hitSlop={4}
|
||||
style={styles.flex}>
|
||||
<ThemedText type="small">{c.raw}</ThemedText>
|
||||
{c.start !== undefined && (
|
||||
<ThemedText type="code" themeColor="textSecondary">
|
||||
{formatClock(c.start)}
|
||||
</ThemedText>
|
||||
)}
|
||||
</Pressable>
|
||||
<Pressable
|
||||
onPress={() => void lookupRef(i, c)}
|
||||
disabled={refLoading[i]}
|
||||
hitSlop={6}>
|
||||
{refLoading[i] ? (
|
||||
<ActivityIndicator size="small" />
|
||||
) : (
|
||||
<ThemedText type="link" themeColor="textSecondary">
|
||||
{meta ? 'Looked up' : 'Look up'}
|
||||
</ThemedText>
|
||||
)}
|
||||
</Pressable>
|
||||
</View>
|
||||
|
||||
{meta && (
|
||||
<View style={styles.metaBox}>
|
||||
{meta.title && <ThemedText type="smallBold">{meta.title}</ThemedText>}
|
||||
{(meta.authors?.length || meta.year) && (
|
||||
<ThemedText type="small" themeColor="textSecondary">
|
||||
{[meta.authors?.join(', '), meta.year].filter(Boolean).join(' · ')}
|
||||
</ThemedText>
|
||||
)}
|
||||
<ThemedText type="small" themeColor="textSecondary">{meta.source}</ThemedText>
|
||||
</View>
|
||||
)}
|
||||
|
||||
<View style={styles.linkChips}>
|
||||
{linkEntries(links).map(([label, url]) => (
|
||||
<Pressable
|
||||
key={label}
|
||||
onPress={() => openLink(url)}
|
||||
style={({ pressed }) => [
|
||||
styles.linkChip,
|
||||
{ backgroundColor: theme.backgroundSelected, opacity: pressed ? 0.7 : 1 },
|
||||
]}>
|
||||
<ThemedText type="small">{label}</ThemedText>
|
||||
</Pressable>
|
||||
))}
|
||||
</View>
|
||||
</View>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</View>
|
||||
</>
|
||||
)}
|
||||
</ThemedView>
|
||||
|
||||
{segments.map((s, i) => (
|
||||
<View key={i} style={[styles.segRow, i === activeIndex && { backgroundColor: theme.backgroundSelected }]}>
|
||||
<View
|
||||
key={i}
|
||||
onLayout={(e) => {
|
||||
segmentYs.current[i] = e.nativeEvent.layout.y;
|
||||
}}
|
||||
style={[styles.segRow, i === activeIndex && { backgroundColor: theme.backgroundSelected }]}>
|
||||
<Pressable onPress={() => seek(s.start)} hitSlop={6}>
|
||||
<ThemedText type="code" themeColor="textSecondary" style={styles.ts}>
|
||||
{formatClock(s.start)}
|
||||
@@ -168,7 +553,7 @@ export default function TranscriptScreen() {
|
||||
</Pressable>
|
||||
<TextInput
|
||||
value={s.text}
|
||||
onChangeText={(t) => editSegment(i, t)}
|
||||
onChangeText={(next) => editSegment(i, next)}
|
||||
multiline
|
||||
style={[styles.segText, { color: theme.text }]}
|
||||
/>
|
||||
@@ -192,10 +577,54 @@ export default function TranscriptScreen() {
|
||||
);
|
||||
}
|
||||
|
||||
/** Index of the segment whose start is closest to `time` (fallback for jumps
|
||||
* that don't land inside any segment's [start, end) window). */
|
||||
function nearestIndex(segments: Segment[], time: number): number {
|
||||
if (segments.length === 0) return -1;
|
||||
let best = 0;
|
||||
let bestDist = Infinity;
|
||||
for (let i = 0; i < segments.length; i++) {
|
||||
const seg = segments[i];
|
||||
if (!seg) continue;
|
||||
const dist = Math.abs(seg.start - time);
|
||||
if (dist < bestDist) {
|
||||
bestDist = dist;
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
function Centered({ children }: { children: React.ReactNode }) {
|
||||
return <ThemedView style={[styles.fill, styles.centered]}>{children}</ThemedView>;
|
||||
}
|
||||
|
||||
/** Localized date for a candidate event — date-only when it's an all-day item. */
|
||||
function formatEventDate(e: CandidateEvent): string {
|
||||
const d = new Date(e.dateMs);
|
||||
return e.allDay
|
||||
? d.toLocaleDateString()
|
||||
: d.toLocaleString([], { dateStyle: 'medium', timeStyle: 'short' });
|
||||
}
|
||||
|
||||
/** Ordered, labeled, defined link entries from a LookupLinks bag. */
|
||||
function linkEntries(links: LookupLinks): [string, string][] {
|
||||
const order: [keyof LookupLinks, string][] = [
|
||||
['doi', 'DOI'],
|
||||
['arxiv', 'arXiv'],
|
||||
['openAlex', 'OpenAlex'],
|
||||
['openLibrary', 'Open Library'],
|
||||
['googleBooks', 'Google Books'],
|
||||
['wikipedia', 'Wikipedia'],
|
||||
['annasArchive', "Anna's Archive"],
|
||||
['libgen', 'LibGen'],
|
||||
];
|
||||
return order
|
||||
.map(([key, label]) => [label, links[key]] as const)
|
||||
.filter((pair): pair is [string, string] => typeof pair[1] === 'string' && pair[1].length > 0)
|
||||
.map(([label, url]) => [label, url]);
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
fill: { flex: 1 },
|
||||
centered: { alignItems: 'center', justifyContent: 'center' },
|
||||
@@ -203,6 +632,20 @@ const styles = StyleSheet.create({
|
||||
title: { fontSize: 22, fontWeight: '700', paddingVertical: Spacing.two },
|
||||
exportRow: { flexDirection: 'row', flexWrap: 'wrap', gap: Spacing.two, marginBottom: Spacing.two },
|
||||
chip: { paddingHorizontal: Spacing.three, paddingVertical: Spacing.one, borderRadius: 999 },
|
||||
aidsCard: { padding: Spacing.three, borderRadius: Spacing.three, gap: Spacing.two, marginBottom: Spacing.two },
|
||||
aidsRow: { flexDirection: 'row', flexWrap: 'wrap', gap: Spacing.two },
|
||||
aidBtn: { paddingHorizontal: Spacing.three, paddingVertical: Spacing.two, borderRadius: Spacing.two, alignItems: 'center' },
|
||||
aidBtnText: { color: '#fff', fontWeight: '700', fontSize: 14 },
|
||||
aidBtnTextAlt: { fontWeight: '700', fontSize: 14 },
|
||||
aidSection: { gap: Spacing.one, marginTop: Spacing.one },
|
||||
aidLine: { flexDirection: 'row', gap: Spacing.two, alignItems: 'flex-start', paddingVertical: 2 },
|
||||
flex: { flex: 1 },
|
||||
refHeader: { flexDirection: 'row', alignItems: 'center', justifyContent: 'space-between', gap: Spacing.two },
|
||||
refRow: { flexDirection: 'row', gap: Spacing.two, alignItems: 'flex-start', paddingVertical: 2 },
|
||||
citCard: { gap: Spacing.one, paddingVertical: Spacing.one },
|
||||
metaBox: { gap: 2, paddingLeft: Spacing.two },
|
||||
linkChips: { flexDirection: 'row', flexWrap: 'wrap', gap: Spacing.one },
|
||||
linkChip: { paddingHorizontal: Spacing.two, paddingVertical: 2, borderRadius: 999 },
|
||||
segRow: { flexDirection: 'row', gap: Spacing.two, paddingVertical: Spacing.one, paddingHorizontal: Spacing.two, borderRadius: Spacing.two, alignItems: 'flex-start' },
|
||||
ts: { paddingTop: 4, minWidth: 52 },
|
||||
segText: { flex: 1, fontSize: 16, lineHeight: 24, padding: 0 },
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
// Web microphone recorder (MediaRecorder) for in-app lecture capture (ROADMAP
|
||||
// Phase 2). Records to a Blob, exposes elapsed time + a live input level, holds
|
||||
// a screen Wake Lock while recording, and supports pause/resume. On stop it
|
||||
// returns the recording as an ArrayBuffer that flows straight into the existing
|
||||
// transcribe pipeline (decode -> chunk -> Whisper -> save).
|
||||
//
|
||||
// Web-only: getUserMedia/MediaRecorder/AudioContext are browser APIs. On native
|
||||
// `supported` is false and the controls no-op (native realtime capture via
|
||||
// whisper.rn is a follow-up).
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { Platform } from 'react-native';
|
||||
|
||||
export type RecorderStatus = 'idle' | 'recording' | 'paused' | 'stopped';
|
||||
|
||||
export interface RecordingResult {
|
||||
data: ArrayBuffer;
|
||||
mime: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface Recorder {
|
||||
supported: boolean;
|
||||
status: RecorderStatus;
|
||||
elapsedMs: number;
|
||||
/** Live input level in [0,1], for a meter. */
|
||||
level: number;
|
||||
error?: string;
|
||||
start: () => Promise<void>;
|
||||
pause: () => void;
|
||||
resume: () => void;
|
||||
/** Stop and return the recording (null if nothing captured / unsupported). */
|
||||
stop: () => Promise<RecordingResult | null>;
|
||||
}
|
||||
|
||||
function isSupported(): boolean {
|
||||
return (
|
||||
Platform.OS === 'web' &&
|
||||
typeof navigator !== 'undefined' &&
|
||||
!!navigator.mediaDevices?.getUserMedia &&
|
||||
typeof MediaRecorder !== 'undefined'
|
||||
);
|
||||
}
|
||||
|
||||
export function useRecorder(): Recorder {
|
||||
const supported = isSupported();
|
||||
const [status, setStatus] = useState<RecorderStatus>('idle');
|
||||
const [elapsedMs, setElapsedMs] = useState(0);
|
||||
const [level, setLevel] = useState(0);
|
||||
const [error, setError] = useState<string | undefined>(undefined);
|
||||
|
||||
const recorderRef = useRef<MediaRecorder | null>(null);
|
||||
const chunksRef = useRef<Blob[]>([]);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const audioCtxRef = useRef<AudioContext | null>(null);
|
||||
const analyserRef = useRef<AnalyserNode | null>(null);
|
||||
const rafRef = useRef<number | null>(null);
|
||||
const wakeRef = useRef<WakeLockSentinel | null>(null);
|
||||
// Elapsed bookkeeping across pause/resume.
|
||||
const segStartRef = useRef(0);
|
||||
const accumRef = useRef(0);
|
||||
|
||||
const cleanup = useCallback(() => {
|
||||
if (rafRef.current != null) cancelAnimationFrame(rafRef.current);
|
||||
rafRef.current = null;
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop());
|
||||
streamRef.current = null;
|
||||
void audioCtxRef.current?.close().catch(() => {});
|
||||
audioCtxRef.current = null;
|
||||
analyserRef.current = null;
|
||||
void wakeRef.current?.release().catch(() => {});
|
||||
wakeRef.current = null;
|
||||
}, []);
|
||||
|
||||
useEffect(() => cleanup, [cleanup]);
|
||||
|
||||
const tick = useCallback(() => {
|
||||
// Elapsed: accumulated paused time + current running segment.
|
||||
const running = status === 'recording' ? Date.now() - segStartRef.current : 0;
|
||||
setElapsedMs(accumRef.current + running);
|
||||
|
||||
const an = analyserRef.current;
|
||||
if (an) {
|
||||
const buf = new Uint8Array(an.fftSize);
|
||||
an.getByteTimeDomainData(buf);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < buf.length; i++) {
|
||||
const v = (buf[i]! - 128) / 128;
|
||||
sum += v * v;
|
||||
}
|
||||
setLevel(Math.min(1, Math.sqrt(sum / buf.length) * 3));
|
||||
}
|
||||
rafRef.current = requestAnimationFrame(tick);
|
||||
}, [status]);
|
||||
|
||||
// Keep the rAF loop bound to the latest `status` via tick's identity.
|
||||
useEffect(() => {
|
||||
if (status === 'recording' || status === 'paused') {
|
||||
rafRef.current = requestAnimationFrame(tick);
|
||||
return () => {
|
||||
if (rafRef.current != null) cancelAnimationFrame(rafRef.current);
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}, [status, tick]);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (!supported) {
|
||||
setError('Recording is only available in the browser for now.');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
setError(undefined);
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
streamRef.current = stream;
|
||||
|
||||
const mr = new MediaRecorder(stream);
|
||||
chunksRef.current = [];
|
||||
mr.ondataavailable = (e) => {
|
||||
if (e.data && e.data.size > 0) chunksRef.current.push(e.data);
|
||||
};
|
||||
mr.start();
|
||||
recorderRef.current = mr;
|
||||
|
||||
// Level metering off an AnalyserNode.
|
||||
const ctx = new (window.AudioContext || (window as unknown as { webkitAudioContext: typeof AudioContext }).webkitAudioContext)();
|
||||
const src = ctx.createMediaStreamSource(stream);
|
||||
const an = ctx.createAnalyser();
|
||||
an.fftSize = 512;
|
||||
src.connect(an);
|
||||
audioCtxRef.current = ctx;
|
||||
analyserRef.current = an;
|
||||
|
||||
// Keep the screen awake while recording (best-effort).
|
||||
try {
|
||||
wakeRef.current = (await navigator.wakeLock?.request('screen')) ?? null;
|
||||
} catch {
|
||||
/* wake lock optional */
|
||||
}
|
||||
|
||||
accumRef.current = 0;
|
||||
segStartRef.current = Date.now();
|
||||
setElapsedMs(0);
|
||||
setStatus('recording');
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : 'Could not start recording (mic permission?).');
|
||||
cleanup();
|
||||
setStatus('idle');
|
||||
}
|
||||
}, [supported, cleanup]);
|
||||
|
||||
const pause = useCallback(() => {
|
||||
const mr = recorderRef.current;
|
||||
if (!mr || status !== 'recording') return;
|
||||
mr.pause();
|
||||
accumRef.current += Date.now() - segStartRef.current;
|
||||
setStatus('paused');
|
||||
}, [status]);
|
||||
|
||||
const resume = useCallback(() => {
|
||||
const mr = recorderRef.current;
|
||||
if (!mr || status !== 'paused') return;
|
||||
mr.resume();
|
||||
segStartRef.current = Date.now();
|
||||
setStatus('recording');
|
||||
}, [status]);
|
||||
|
||||
const stop = useCallback(async (): Promise<RecordingResult | null> => {
|
||||
const mr = recorderRef.current;
|
||||
if (!mr) return null;
|
||||
const mime = mr.mimeType || 'audio/webm';
|
||||
const blob: Blob = await new Promise((resolve) => {
|
||||
mr.onstop = () => resolve(new Blob(chunksRef.current, { type: mime }));
|
||||
mr.stop();
|
||||
});
|
||||
recorderRef.current = null;
|
||||
cleanup();
|
||||
setStatus('stopped');
|
||||
if (blob.size === 0) return null;
|
||||
const data = await blob.arrayBuffer();
|
||||
const ext = mime.includes('ogg') ? 'ogg' : mime.includes('mp4') ? 'mp4' : 'webm';
|
||||
return { data, mime, name: `Recording.${ext}` };
|
||||
}, [cleanup]);
|
||||
|
||||
return { supported, status, elapsedMs, level, error, start, pause, resume, stop };
|
||||
}
|
||||
@@ -1,27 +1,30 @@
|
||||
// Native (iOS/Android) audio decoder.
|
||||
//
|
||||
// Unlike the web, React Native has no built-in media decoder we can rely on.
|
||||
// `ffmpeg-kit-react-native` — the package everyone used to reach for — was
|
||||
// RETIRED by its maintainer in early 2025, so we deliberately do NOT depend on
|
||||
// it. For now we support only WAV, which we can decode in pure JS via our own
|
||||
// `decodeWav`. Everything else throws a clear, actionable error.
|
||||
// WAV is decoded in pure JS (`decodeWav`) — fast and dependency-free. Every
|
||||
// other container/codec (m4a/aac, mp3, ogg/opus, flac, …) is decoded by
|
||||
// `react-native-audio-api`, which uses the platform codecs (Android MediaCodec /
|
||||
// iOS AVFoundation, with a bundled FFmpeg fallback). This mirrors the web
|
||||
// decoder's `AudioContext.decodeAudioData` path. Decoding is fully on-device —
|
||||
// the audio bytes never leave the phone.
|
||||
//
|
||||
// TODO(audio/native): add a maintained native decoder for compressed formats
|
||||
// (mp3/m4a/aac/ogg/flac). The current front-runner is
|
||||
// `react-native-audio-api` (a Web-Audio-style API for RN). When added, route
|
||||
// non-WAV URIs through it and downmix/resample with `toMono16k`, mirroring the
|
||||
// web path.
|
||||
// Why this matters on native: in-app recording is web-only, so importing a file
|
||||
// from the phone's own recorder is the ONLY way to get audio in — and those are
|
||||
// almost always m4a/aac. (Previously native accepted WAV only.)
|
||||
//
|
||||
// We hand react-native-audio-api the raw file BYTES (ArrayBuffer), not a URI, so
|
||||
// we don't depend on how the OS file picker spells the path (file:// vs
|
||||
// content://) — `readFileBytes` already reads either via Expo's File API.
|
||||
//
|
||||
// File reading uses Expo SDK 56's object-oriented `File` API
|
||||
// (`new File(uri).bytes()`), which returns the raw bytes as a `Uint8Array`
|
||||
// directly — no base64 round-trip needed. We still keep a base64 decode helper
|
||||
// below as a documented fallback for environments/URIs where `.bytes()` isn't
|
||||
// available (it reads via `.base64()` and decodes to a Uint8Array by hand).
|
||||
// directly — no base64 round-trip needed. We keep a base64 fallback for
|
||||
// environments/URIs where `.bytes()` isn't available.
|
||||
|
||||
import { File } from 'expo-file-system';
|
||||
import { decodeAudioData } from 'react-native-audio-api';
|
||||
|
||||
import type { AudioDecoder, AudioFileInput } from './decode';
|
||||
import type { PcmAudio } from '../types';
|
||||
import { WHISPER_SAMPLE_RATE, type PcmAudio } from '../types';
|
||||
import { decodeWav } from './wav';
|
||||
import { toMono16k } from './resample';
|
||||
|
||||
@@ -90,6 +93,25 @@ async function readFileBytes(uri: string): Promise<Uint8Array> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True if these bytes are a RIFF/WAVE container. Sniffing the content (rather
|
||||
* than trusting the file extension) lets us take the fast pure-JS WAV path even
|
||||
* for mislabeled files, and avoids handing WAV to the heavier native decoder.
|
||||
*/
|
||||
function isWavBytes(b: Uint8Array): boolean {
|
||||
return (
|
||||
b.length >= 12 &&
|
||||
b[0] === 0x52 && // 'R'
|
||||
b[1] === 0x49 && // 'I'
|
||||
b[2] === 0x46 && // 'F'
|
||||
b[3] === 0x46 && // 'F'
|
||||
b[8] === 0x57 && // 'W'
|
||||
b[9] === 0x41 && // 'A'
|
||||
b[10] === 0x56 && // 'V'
|
||||
b[11] === 0x45 // 'E'
|
||||
);
|
||||
}
|
||||
|
||||
export const decoder: AudioDecoder = {
|
||||
async decode(input: AudioFileInput): Promise<PcmAudio> {
|
||||
const { uri } = input;
|
||||
@@ -97,22 +119,42 @@ export const decoder: AudioDecoder = {
|
||||
throw new Error('Native decoder requires a file `uri`.');
|
||||
}
|
||||
|
||||
// Sniff the container from the URI extension (case-insensitive), ignoring
|
||||
// any query string / fragment that content URIs sometimes carry.
|
||||
const path = uri.split(/[?#]/, 1)[0] ?? uri;
|
||||
const isWav = path.toLowerCase().endsWith('.wav');
|
||||
const bytes = await readFileBytes(uri);
|
||||
|
||||
if (!isWav) {
|
||||
// Fast path: WAV is decoded in pure JS (no native call).
|
||||
if (isWavBytes(bytes)) {
|
||||
const { sampleRate, channelData } = decodeWav(bytes);
|
||||
return { sampleRate: 16000, samples: toMono16k(channelData, sampleRate) };
|
||||
}
|
||||
|
||||
// Everything else (m4a/aac, mp3, ogg/opus, flac, …) -> platform codecs via
|
||||
// react-native-audio-api. A fresh, contiguous ArrayBuffer of exactly the
|
||||
// file bytes is what `decodeAudioData` expects.
|
||||
const arrayBuffer = new Uint8Array(bytes).buffer as ArrayBuffer;
|
||||
let audioBuffer;
|
||||
try {
|
||||
// Ask the native engine to resample to 16kHz DURING decode (platform DSP).
|
||||
// This avoids materializing a full-rate (e.g. 44.1kHz stereo) Float32
|
||||
// buffer in JS and skips the pure-JS resample loop — a big memory/CPU win
|
||||
// and OOM guard on long lectures. toMono16k below still downmixes to mono
|
||||
// and short-circuits its resampler since the rate already matches.
|
||||
audioBuffer = await decodeAudioData(arrayBuffer, WHISPER_SAMPLE_RATE);
|
||||
} catch (e) {
|
||||
const detail = e instanceof Error ? e.message : String(e);
|
||||
throw new Error(
|
||||
'Only WAV is supported on native for now — ffmpeg-kit-react-native ' +
|
||||
'was retired in 2025; a maintained native decoder (e.g. ' +
|
||||
'react-native-audio-api) is a follow-up.',
|
||||
`Couldn't decode this audio on device (${detail}). Supported formats: ` +
|
||||
'WAV, m4a/aac, mp3, ogg, flac. If it keeps failing, convert it to ' +
|
||||
'WAV and import again.',
|
||||
);
|
||||
}
|
||||
|
||||
const bytes = await readFileBytes(uri);
|
||||
const { sampleRate, channelData } = decodeWav(bytes);
|
||||
const samples = toMono16k(channelData, sampleRate);
|
||||
return { sampleRate: 16000, samples };
|
||||
const channelData: Float32Array[] = Array.from(
|
||||
{ length: audioBuffer.numberOfChannels },
|
||||
(_unused, c) => audioBuffer.getChannelData(c),
|
||||
);
|
||||
return {
|
||||
sampleRate: 16000,
|
||||
samples: toMono16k(channelData, audioBuffer.sampleRate),
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
@@ -197,6 +197,62 @@ describe('decodeWav stereo de-interleaving', () => {
|
||||
expect(decoded.channelData[0]![0]!).toBeCloseTo(12345 / 0x7fff, 4);
|
||||
});
|
||||
|
||||
it('decodes 24-bit PCM (format 1) data', () => {
|
||||
// mono, 2 samples: +0x400000 (~+0.5) and -0x400000 (-0.5), 3 bytes LE each.
|
||||
const data = new Uint8Array([0x00, 0x00, 0x40, 0x00, 0x00, 0xc0]);
|
||||
const wav = buildWav({
|
||||
audioFormat: 1,
|
||||
numChannels: 1,
|
||||
sampleRate: 16000,
|
||||
bitsPerSample: 24,
|
||||
data,
|
||||
});
|
||||
const decoded = decodeWav(wav);
|
||||
expect(decoded.channelData.length).toBe(1);
|
||||
expect(decoded.channelData[0]![0]!).toBeCloseTo(0x400000 / 0x7fffff, 3);
|
||||
expect(decoded.channelData[0]![1]!).toBeCloseTo(-0.5, 3);
|
||||
});
|
||||
|
||||
it('decodes WAVE_FORMAT_EXTENSIBLE (0xFFFE) wrapping 16-bit PCM', () => {
|
||||
// 40-byte fmt: 16 standard + 24-byte extension whose SubFormat GUID begins
|
||||
// with the real format tag (1 = PCM).
|
||||
const fmt = new Uint8Array(40);
|
||||
const fv = new DataView(fmt.buffer);
|
||||
fv.setUint16(0, 0xfffe, true); // WAVE_FORMAT_EXTENSIBLE
|
||||
fv.setUint16(2, 1, true); // channels
|
||||
fv.setUint32(4, 16000, true); // sampleRate
|
||||
fv.setUint32(8, 32000, true); // byteRate
|
||||
fv.setUint16(12, 2, true); // blockAlign
|
||||
fv.setUint16(14, 16, true); // bitsPerSample
|
||||
fv.setUint16(16, 22, true); // cbSize
|
||||
fv.setUint16(18, 16, true); // validBitsPerSample
|
||||
fv.setUint32(20, 0, true); // channelMask
|
||||
fv.setUint16(24, 1, true); // SubFormat GUID first 2 bytes -> PCM
|
||||
const data = new Uint8Array(2);
|
||||
new DataView(data.buffer).setInt16(0, 16383, true); // ~0.5
|
||||
|
||||
const total = 12 + (8 + 40) + (8 + 2);
|
||||
const out = new Uint8Array(total);
|
||||
const view = new DataView(out.buffer);
|
||||
const tag = (off: number, s: string) => {
|
||||
for (let i = 0; i < 4; i++) view.setUint8(off + i, s.charCodeAt(i));
|
||||
};
|
||||
tag(0, 'RIFF');
|
||||
view.setUint32(4, total - 8, true);
|
||||
tag(8, 'WAVE');
|
||||
tag(12, 'fmt ');
|
||||
view.setUint32(16, 40, true);
|
||||
out.set(fmt, 20);
|
||||
tag(60, 'data');
|
||||
view.setUint32(64, 2, true);
|
||||
out.set(data, 68);
|
||||
|
||||
const decoded = decodeWav(out);
|
||||
expect(decoded.sampleRate).toBe(16000);
|
||||
expect(decoded.channelData.length).toBe(1);
|
||||
expect(decoded.channelData[0]![0]!).toBeCloseTo(16383 / 0x7fff, 4);
|
||||
});
|
||||
|
||||
it('decodes 32-bit IEEE float (format 3) data', () => {
|
||||
// mono, 2 samples: 0.25 and -0.75
|
||||
const data = new Uint8Array(8);
|
||||
|
||||
+32
-1
@@ -20,6 +20,10 @@ export interface DecodedWav {
|
||||
|
||||
const WAVE_FORMAT_PCM = 1;
|
||||
const WAVE_FORMAT_IEEE_FLOAT = 3;
|
||||
// Many encoders (ffmpeg, Audacity, Windows recorders) wrap PCM/float in
|
||||
// WAVE_FORMAT_EXTENSIBLE; the real format tag is the first 2 bytes of the
|
||||
// extension's SubFormat GUID.
|
||||
const WAVE_FORMAT_EXTENSIBLE = 0xfffe;
|
||||
|
||||
/** Read a 4-byte ASCII tag (e.g. "RIFF") at the given byte offset. */
|
||||
function readTag(view: DataView, offset: number): string {
|
||||
@@ -81,6 +85,12 @@ export function decodeWav(bytes: Uint8Array): DecodedWav {
|
||||
sampleRate = view.getUint32(chunkBody + 4, true);
|
||||
// bytes 8..12: byteRate, bytes 12..14: blockAlign (derived; ignored)
|
||||
bitsPerSample = view.getUint16(chunkBody + 14, true);
|
||||
// WAVE_FORMAT_EXTENSIBLE: the actual PCM/float tag lives in the extension's
|
||||
// SubFormat GUID (first 2 bytes), at fmt-body + 24. Resolve it so common
|
||||
// ffmpeg/Windows EXTENSIBLE WAVs decode instead of throwing.
|
||||
if (audioFormat === WAVE_FORMAT_EXTENSIBLE && chunkBody + 26 <= bytes.byteLength) {
|
||||
audioFormat = view.getUint16(chunkBody + 24, true);
|
||||
}
|
||||
haveFmt = true;
|
||||
} else if (chunkId === 'data') {
|
||||
dataOffset = chunkBody;
|
||||
@@ -124,6 +134,26 @@ export function decodeWav(bytes: Uint8Array): DecodedWav {
|
||||
channelData[c]![frame] = f;
|
||||
}
|
||||
}
|
||||
} else if (audioFormat === WAVE_FORMAT_PCM && bitsPerSample === 24) {
|
||||
// 24-bit signed little-endian PCM (common in EXTENSIBLE WAVs).
|
||||
const bytesPerSample = 3;
|
||||
const frameSize = bytesPerSample * numChannels;
|
||||
const numFrames = Math.floor(dataLength / frameSize);
|
||||
for (let c = 0; c < numChannels; c++) {
|
||||
channelData.push(new Float32Array(numFrames));
|
||||
}
|
||||
for (let frame = 0; frame < numFrames; frame++) {
|
||||
const base = dataOffset + frame * frameSize;
|
||||
for (let c = 0; c < numChannels; c++) {
|
||||
const o = base + c * bytesPerSample;
|
||||
let v =
|
||||
view.getUint8(o) |
|
||||
(view.getUint8(o + 1) << 8) |
|
||||
(view.getUint8(o + 2) << 16);
|
||||
if (v & 0x800000) v -= 0x1000000; // sign-extend the 24th bit
|
||||
channelData[c]![frame] = v < 0 ? v / 0x800000 : v / 0x7fffff;
|
||||
}
|
||||
}
|
||||
} else if (
|
||||
audioFormat === WAVE_FORMAT_IEEE_FLOAT &&
|
||||
bitsPerSample === 32
|
||||
@@ -146,7 +176,8 @@ export function decodeWav(bytes: Uint8Array): DecodedWav {
|
||||
} else {
|
||||
throw new Error(
|
||||
`Unsupported WAV format: audioFormat=${audioFormat}, ` +
|
||||
`bitsPerSample=${bitsPerSample} (supported: 16-bit PCM, 32-bit float)`,
|
||||
`bitsPerSample=${bitsPerSample} ` +
|
||||
`(supported: 16/24-bit PCM, 32-bit float, incl. WAVE_FORMAT_EXTENSIBLE)`,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
+454
-10
@@ -22,6 +22,7 @@ import { newId } from '../ids';
|
||||
import {
|
||||
parseDraft,
|
||||
parseCourseDraft,
|
||||
parseFlashcardDraft,
|
||||
zSegment,
|
||||
type TranscriptDraft,
|
||||
type TranscriptMeta,
|
||||
@@ -29,8 +30,16 @@ import {
|
||||
type TranscriptPatch,
|
||||
type Course,
|
||||
type CourseDraft,
|
||||
type Flashcard,
|
||||
type FlashcardDraft,
|
||||
type SrsState,
|
||||
} from './schema';
|
||||
import type { MediaInput, StorageRepo } from './repo';
|
||||
import type {
|
||||
MediaInput,
|
||||
StorageRepo,
|
||||
SegmentVector,
|
||||
VectorHit,
|
||||
} from './repo';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Row shapes (as returned by getAllAsync). SQLite has no boolean/undefined:
|
||||
@@ -80,6 +89,38 @@ interface MediaRow {
|
||||
mime: string;
|
||||
}
|
||||
|
||||
// One stored segment embedding. `vector` is a SQLite BLOB; on read expo-sqlite
|
||||
// hands it back as a Uint8Array whose underlying bytes we reinterpret as a
|
||||
// Float32Array. start/end are REAL, courseId is nullable text.
|
||||
interface SegVecRow {
|
||||
transcriptId: string;
|
||||
segmentId: string;
|
||||
start: number;
|
||||
end: number;
|
||||
courseId: string | null;
|
||||
text: string;
|
||||
vector: Uint8Array;
|
||||
model: string;
|
||||
}
|
||||
|
||||
// One stored flashcard. srs is JSON-encoded SrsState; dueAt is a denormalized
|
||||
// copy of srs.due kept as an INTEGER column so due-card queries can be indexed.
|
||||
// segmentId/start are nullable (cards may not anchor to a segment).
|
||||
interface FlashcardRow {
|
||||
id: string;
|
||||
transcriptId: string;
|
||||
courseId: string | null;
|
||||
segmentId: string | null;
|
||||
start: number | null;
|
||||
front: string;
|
||||
back: string;
|
||||
createdAt: number;
|
||||
/** JSON-encoded SrsState. */
|
||||
srs: string;
|
||||
/** Denormalized srs.due (ms since epoch) for indexed due queries. */
|
||||
dueAt: number;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pure derivation helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -141,6 +182,50 @@ function rowToCourse(r: CourseRow): Course {
|
||||
return course;
|
||||
}
|
||||
|
||||
/** Map a flashcard row to the public Flashcard (parsing the JSON srs). */
|
||||
function rowToFlashcard(r: FlashcardRow): Flashcard {
|
||||
const card: Flashcard = {
|
||||
id: r.id,
|
||||
transcriptId: r.transcriptId,
|
||||
// Keep courseId as null (the "Unsorted" sentinel) rather than dropping it.
|
||||
courseId: r.courseId,
|
||||
front: r.front,
|
||||
back: r.back,
|
||||
createdAt: r.createdAt,
|
||||
srs: JSON.parse(r.srs) as SrsState,
|
||||
};
|
||||
if (r.segmentId !== null) card.segmentId = r.segmentId;
|
||||
if (r.start !== null) card.start = r.start;
|
||||
return card;
|
||||
}
|
||||
|
||||
// ---- vector <-> BLOB codecs --------------------------------------------------
|
||||
// SQLite stores embeddings as raw little-endian float32 bytes. We bind a
|
||||
// Uint8Array view of the Float32Array's buffer on write and reconstruct a
|
||||
// Float32Array from the returned bytes on read. We copy on read so the result
|
||||
// is 4-byte aligned (a Uint8Array sub-view may start at an unaligned offset,
|
||||
// which `new Float32Array(buffer)` requires to be a multiple of 4).
|
||||
function vectorToBlob(vector: Float32Array): Uint8Array {
|
||||
return new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength);
|
||||
}
|
||||
|
||||
function blobToVector(blob: Uint8Array): Float32Array {
|
||||
// Copy the bytes into a fresh, aligned ArrayBuffer, then view as float32.
|
||||
const copy = new Uint8Array(blob.length);
|
||||
copy.set(blob);
|
||||
return new Float32Array(copy.buffer);
|
||||
}
|
||||
|
||||
/** Cosine similarity = dot product of two unit-normalized vectors. */
|
||||
function dot(a: Float32Array, b: Float32Array): number {
|
||||
const n = Math.min(a.length, b.length);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
sum += a[i]! * b[i]!;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Migration runner
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -151,7 +236,7 @@ function rowToCourse(r: CourseRow): Course {
|
||||
// idempotent (IF NOT EXISTS / pragma_table_info guards) so they also no-op on
|
||||
// DBs the original single-table code already created at version 0.
|
||||
|
||||
const TARGET_VERSION = 2;
|
||||
const TARGET_VERSION = 4;
|
||||
|
||||
type Migration = (db: SQLite.SQLiteDatabase) => Promise<void>;
|
||||
|
||||
@@ -246,6 +331,52 @@ const MIGRATIONS: Migration[] = [
|
||||
]);
|
||||
}
|
||||
},
|
||||
|
||||
// ---- v2 -> v3: segment-vector store for Phase 1 semantic search -------
|
||||
async (db) => {
|
||||
// One row per (transcript, segment) embedding. courseId is denormalized
|
||||
// from the owning transcript so course-scoped search needs no join; model
|
||||
// tags the embedding model so a model change can be detected/backfilled.
|
||||
await db.execAsync(`
|
||||
CREATE TABLE IF NOT EXISTS segvecs (
|
||||
transcriptId TEXT NOT NULL,
|
||||
segmentId TEXT NOT NULL,
|
||||
start REAL NOT NULL,
|
||||
end REAL NOT NULL,
|
||||
courseId TEXT,
|
||||
text TEXT NOT NULL,
|
||||
vector BLOB NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
PRIMARY KEY (transcriptId, segmentId)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_segvecs_model ON segvecs (model);
|
||||
CREATE INDEX IF NOT EXISTS idx_segvecs_courseId ON segvecs (courseId);
|
||||
`);
|
||||
},
|
||||
|
||||
// ---- v3 -> v4: flashcards store for Phase 3 spaced repetition ----------
|
||||
async (db) => {
|
||||
// One row per flashcard. srs is JSON; dueAt is a denormalized copy of
|
||||
// srs.due as an INTEGER so due-card queries can use an index. The
|
||||
// transcriptId/courseId indexes drive the cascade + scoping paths.
|
||||
await db.execAsync(`
|
||||
CREATE TABLE IF NOT EXISTS flashcards (
|
||||
id TEXT PRIMARY KEY NOT NULL,
|
||||
transcriptId TEXT,
|
||||
courseId TEXT,
|
||||
segmentId TEXT,
|
||||
start REAL,
|
||||
front TEXT,
|
||||
back TEXT,
|
||||
createdAt INTEGER,
|
||||
srs TEXT,
|
||||
dueAt INTEGER
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_flashcards_transcriptId ON flashcards (transcriptId);
|
||||
CREATE INDEX IF NOT EXISTS idx_flashcards_courseId ON flashcards (courseId);
|
||||
CREATE INDEX IF NOT EXISTS idx_flashcards_dueAt ON flashcards (dueAt);
|
||||
`);
|
||||
},
|
||||
];
|
||||
|
||||
async function runMigrations(db: SQLite.SQLiteDatabase): Promise<void> {
|
||||
@@ -280,7 +411,14 @@ function getDb(): Promise<SQLite.SQLiteDatabase> {
|
||||
const db = await SQLite.openDatabaseAsync('wisp.db');
|
||||
await runMigrations(db);
|
||||
return db;
|
||||
})();
|
||||
})().catch((e) => {
|
||||
// Don't cache a REJECTED promise — otherwise a transient open/migration
|
||||
// failure (locked db, backgrounded mid-migration) would brick storage for
|
||||
// the whole session, with every later call returning the same rejection.
|
||||
// Clearing it lets the next getDb() retry from scratch.
|
||||
dbPromise = undefined;
|
||||
throw e;
|
||||
});
|
||||
}
|
||||
return dbPromise;
|
||||
}
|
||||
@@ -449,6 +587,11 @@ export const repo: StorageRepo = {
|
||||
id,
|
||||
],
|
||||
);
|
||||
// Segments were patched => stored vectors are stale; drop them so the
|
||||
// backfill re-embeds from the new text/timing.
|
||||
if (patch.segments !== undefined) {
|
||||
await db.runAsync(`DELETE FROM segvecs WHERE transcriptId = ?`, [id]);
|
||||
}
|
||||
return updated;
|
||||
},
|
||||
|
||||
@@ -457,6 +600,9 @@ export const repo: StorageRepo = {
|
||||
// Delete persisted media first so we never orphan a file.
|
||||
await this.removeMedia(id);
|
||||
await db.runAsync(`DELETE FROM transcripts WHERE id = ?`, [id]);
|
||||
// Cascade: drop this transcript's vectors + flashcards too.
|
||||
await db.runAsync(`DELETE FROM segvecs WHERE transcriptId = ?`, [id]);
|
||||
await db.runAsync(`DELETE FROM flashcards WHERE transcriptId = ?`, [id]);
|
||||
},
|
||||
|
||||
async search(query: string): Promise<TranscriptMeta[]> {
|
||||
@@ -514,10 +660,21 @@ export const repo: StorageRepo = {
|
||||
const transcript = JSON.parse(row.json) as Transcript;
|
||||
const updatedAt = Date.now();
|
||||
const next: Transcript = { ...transcript, courseId, updatedAt };
|
||||
await db.runAsync(
|
||||
`UPDATE transcripts SET courseId = ?, updatedAt = ?, json = ? WHERE id = ?`,
|
||||
[courseId, updatedAt, JSON.stringify(next), transcriptId],
|
||||
);
|
||||
// Exclusive so the two writes commit atomically and can't interleave with
|
||||
// concurrent repo queries (withTransactionAsync is NOT isolated on a shared
|
||||
// connection — see expo-sqlite docs).
|
||||
await db.withExclusiveTransactionAsync(async (txn) => {
|
||||
await txn.runAsync(
|
||||
`UPDATE transcripts SET courseId = ?, updatedAt = ?, json = ? WHERE id = ?`,
|
||||
[courseId, updatedAt, JSON.stringify(next), transcriptId],
|
||||
);
|
||||
// Keep the denormalized courseId on every vector row in sync so
|
||||
// course-scoped search keeps matching after a move.
|
||||
await txn.runAsync(`UPDATE segvecs SET courseId = ? WHERE transcriptId = ?`, [
|
||||
courseId,
|
||||
transcriptId,
|
||||
]);
|
||||
});
|
||||
},
|
||||
|
||||
// --- courses -------------------------------------------------------------
|
||||
@@ -616,12 +773,22 @@ export const repo: StorageRepo = {
|
||||
const db = await getDb();
|
||||
// First reassign this course's transcripts to Unsorted, THEN delete the
|
||||
// course row, so we never leave dangling courseId references.
|
||||
await db.withTransactionAsync(async () => {
|
||||
await db.runAsync(
|
||||
await db.withExclusiveTransactionAsync(async (txn) => {
|
||||
await txn.runAsync(
|
||||
`UPDATE transcripts SET courseId = NULL WHERE courseId = ?`,
|
||||
[id],
|
||||
);
|
||||
await db.runAsync(`DELETE FROM courses WHERE id = ?`, [id]);
|
||||
// Keep denormalized vector rows consistent: their transcripts are now
|
||||
// Unsorted, so their courseId must follow.
|
||||
await txn.runAsync(`UPDATE segvecs SET courseId = NULL WHERE courseId = ?`, [
|
||||
id,
|
||||
]);
|
||||
// Flashcards in this course follow their transcripts to Unsorted.
|
||||
await txn.runAsync(
|
||||
`UPDATE flashcards SET courseId = NULL WHERE courseId = ?`,
|
||||
[id],
|
||||
);
|
||||
await txn.runAsync(`DELETE FROM courses WHERE id = ?`, [id]);
|
||||
});
|
||||
},
|
||||
|
||||
@@ -695,4 +862,281 @@ export const repo: StorageRepo = {
|
||||
transcriptId,
|
||||
]);
|
||||
},
|
||||
|
||||
// --- semantic search vectors (Phase 1) ----------------------------------
|
||||
async upsertVectors(
|
||||
transcriptId: string,
|
||||
model: string,
|
||||
vectors: SegmentVector[],
|
||||
): Promise<void> {
|
||||
const db = await getDb();
|
||||
// Denormalize the transcript's current courseId onto each row so
|
||||
// course-scoped search never has to join back to transcripts.
|
||||
const owner = await db.getFirstAsync<{ courseId: string | null }>(
|
||||
`SELECT courseId FROM transcripts WHERE id = ?`,
|
||||
[transcriptId],
|
||||
);
|
||||
const courseId = owner ? owner.courseId : null;
|
||||
// Exclusive: the DELETE-then-INSERT must be atomic + isolated, else a
|
||||
// concurrent searchVectors during an index rebuild can read zero vectors,
|
||||
// and a crash between the DELETE and the INSERTs would lose this
|
||||
// transcript's embeddings entirely.
|
||||
await db.withExclusiveTransactionAsync(async (txn) => {
|
||||
// Replace, not merge: clear existing rows so a re-embed with fewer
|
||||
// segments can't leave stale rows behind.
|
||||
await txn.runAsync(`DELETE FROM segvecs WHERE transcriptId = ?`, [
|
||||
transcriptId,
|
||||
]);
|
||||
for (const v of vectors) {
|
||||
await txn.runAsync(
|
||||
`INSERT INTO segvecs
|
||||
(transcriptId, segmentId, start, end, courseId, text, vector, model)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
transcriptId,
|
||||
v.segmentId,
|
||||
v.start,
|
||||
v.end,
|
||||
courseId,
|
||||
v.text,
|
||||
vectorToBlob(v.vector),
|
||||
model,
|
||||
],
|
||||
);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
async searchVectors(
|
||||
query: Float32Array,
|
||||
opts?: { courseId?: string | null; limit?: number },
|
||||
): Promise<VectorHit[]> {
|
||||
const db = await getDb();
|
||||
// Scope: opts.courseId provided => filter (null => Unsorted via IS NULL);
|
||||
// omitted => search every course.
|
||||
let rows: SegVecRow[];
|
||||
if (opts && opts.courseId !== undefined) {
|
||||
if (opts.courseId === null) {
|
||||
rows = await db.getAllAsync<SegVecRow>(
|
||||
`SELECT transcriptId, segmentId, start, end, courseId, text, vector, model
|
||||
FROM segvecs WHERE courseId IS NULL`,
|
||||
);
|
||||
} else {
|
||||
rows = await db.getAllAsync<SegVecRow>(
|
||||
`SELECT transcriptId, segmentId, start, end, courseId, text, vector, model
|
||||
FROM segvecs WHERE courseId = ?`,
|
||||
[opts.courseId],
|
||||
);
|
||||
}
|
||||
} else {
|
||||
rows = await db.getAllAsync<SegVecRow>(
|
||||
`SELECT transcriptId, segmentId, start, end, courseId, text, vector, model
|
||||
FROM segvecs`,
|
||||
);
|
||||
}
|
||||
|
||||
const limit = opts?.limit ?? 30;
|
||||
const hits: VectorHit[] = rows.map((r) => ({
|
||||
transcriptId: r.transcriptId,
|
||||
segmentId: r.segmentId,
|
||||
start: r.start,
|
||||
end: r.end,
|
||||
courseId: r.courseId,
|
||||
text: r.text,
|
||||
score: dot(query, blobToVector(r.vector)),
|
||||
}));
|
||||
hits.sort((a, b) => b.score - a.score);
|
||||
return hits.slice(0, limit);
|
||||
},
|
||||
|
||||
async clearVectors(transcriptId: string): Promise<void> {
|
||||
const db = await getDb();
|
||||
await db.runAsync(`DELETE FROM segvecs WHERE transcriptId = ?`, [
|
||||
transcriptId,
|
||||
]);
|
||||
},
|
||||
|
||||
async unembeddedIds(model: string): Promise<string[]> {
|
||||
const db = await getDb();
|
||||
// Transcript ids with ZERO segvecs rows for this model: a LEFT-anti-join
|
||||
// against the subset of segvecs tagged with the given model.
|
||||
const rows = await db.getAllAsync<{ id: string }>(
|
||||
`SELECT t.id AS id FROM transcripts t
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM segvecs s
|
||||
WHERE s.transcriptId = t.id AND s.model = ?
|
||||
)`,
|
||||
[model],
|
||||
);
|
||||
return rows.map((r) => r.id);
|
||||
},
|
||||
|
||||
// --- flashcards + spaced repetition (Phase 3) ---------------------------
|
||||
async createFlashcards(drafts: FlashcardDraft[]): Promise<Flashcard[]> {
|
||||
const db = await getDb();
|
||||
const now = Date.now();
|
||||
const cards: Flashcard[] = drafts.map((draft) => {
|
||||
const valid = parseFlashcardDraft(draft);
|
||||
// Initial SM-2 state for a brand-new card: due immediately (now). We
|
||||
// compute the SRS inline here rather than importing learn/srs to keep the
|
||||
// repo storage-only (no dependency on the study logic).
|
||||
const srs: SrsState = {
|
||||
ease: 2.5,
|
||||
intervalDays: 0,
|
||||
reps: 0,
|
||||
lapses: 0,
|
||||
due: now,
|
||||
};
|
||||
return {
|
||||
id: newId('f_'),
|
||||
transcriptId: valid.transcriptId,
|
||||
// courseId defaults to null ("Unsorted") when absent.
|
||||
courseId: valid.courseId ?? null,
|
||||
...(valid.segmentId !== undefined ? { segmentId: valid.segmentId } : {}),
|
||||
...(valid.start !== undefined ? { start: valid.start } : {}),
|
||||
front: valid.front,
|
||||
back: valid.back,
|
||||
createdAt: now,
|
||||
srs,
|
||||
};
|
||||
});
|
||||
await db.withExclusiveTransactionAsync(async (txn) => {
|
||||
for (const card of cards) {
|
||||
await txn.runAsync(
|
||||
`INSERT INTO flashcards
|
||||
(id, transcriptId, courseId, segmentId, start, front, back,
|
||||
createdAt, srs, dueAt)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
card.id,
|
||||
card.transcriptId,
|
||||
card.courseId ?? null,
|
||||
card.segmentId ?? null,
|
||||
card.start ?? null,
|
||||
card.front,
|
||||
card.back,
|
||||
card.createdAt,
|
||||
JSON.stringify(card.srs),
|
||||
card.srs.due,
|
||||
],
|
||||
);
|
||||
}
|
||||
});
|
||||
return cards;
|
||||
},
|
||||
|
||||
async listFlashcards(opts?: {
|
||||
transcriptId?: string;
|
||||
courseId?: string | null;
|
||||
}): Promise<Flashcard[]> {
|
||||
const db = await getDb();
|
||||
const where: string[] = [];
|
||||
const params: (string | null)[] = [];
|
||||
if (opts?.transcriptId !== undefined) {
|
||||
where.push('transcriptId = ?');
|
||||
params.push(opts.transcriptId);
|
||||
}
|
||||
if (opts && opts.courseId !== undefined) {
|
||||
if (opts.courseId === null) {
|
||||
where.push('courseId IS NULL');
|
||||
} else {
|
||||
where.push('courseId = ?');
|
||||
params.push(opts.courseId);
|
||||
}
|
||||
}
|
||||
const clause = where.length ? ` WHERE ${where.join(' AND ')}` : '';
|
||||
const rows = await db.getAllAsync<FlashcardRow>(
|
||||
`SELECT id, transcriptId, courseId, segmentId, start, front, back,
|
||||
createdAt, srs, dueAt
|
||||
FROM flashcards${clause}`,
|
||||
params,
|
||||
);
|
||||
return rows.map(rowToFlashcard);
|
||||
},
|
||||
|
||||
async listDueFlashcards(opts?: {
|
||||
courseId?: string | null;
|
||||
now?: number;
|
||||
limit?: number;
|
||||
}): Promise<Flashcard[]> {
|
||||
const db = await getDb();
|
||||
const now = opts?.now ?? Date.now();
|
||||
// Due means dueAt <= now (the denormalized srs.due column). Soonest first.
|
||||
const where: string[] = ['dueAt <= ?'];
|
||||
const params: (string | number)[] = [now];
|
||||
if (opts && opts.courseId !== undefined) {
|
||||
if (opts.courseId === null) {
|
||||
where.push('courseId IS NULL');
|
||||
} else {
|
||||
where.push('courseId = ?');
|
||||
params.push(opts.courseId);
|
||||
}
|
||||
}
|
||||
let sql = `SELECT id, transcriptId, courseId, segmentId, start, front, back,
|
||||
createdAt, srs, dueAt
|
||||
FROM flashcards
|
||||
WHERE ${where.join(' AND ')}
|
||||
ORDER BY dueAt ASC`;
|
||||
if (opts?.limit !== undefined) {
|
||||
sql += ` LIMIT ?`;
|
||||
params.push(opts.limit);
|
||||
}
|
||||
const rows = await db.getAllAsync<FlashcardRow>(sql, params);
|
||||
return rows.map(rowToFlashcard);
|
||||
},
|
||||
|
||||
async updateFlashcardSrs(id: string, srs: SrsState): Promise<Flashcard> {
|
||||
const db = await getDb();
|
||||
const row = await db.getFirstAsync<FlashcardRow>(
|
||||
`SELECT id, transcriptId, courseId, segmentId, start, front, back,
|
||||
createdAt, srs, dueAt
|
||||
FROM flashcards WHERE id = ?`,
|
||||
[id],
|
||||
);
|
||||
if (!row) {
|
||||
throw new Error(`flashcard not found: ${id}`);
|
||||
}
|
||||
// Persist the new schedule, keeping the denormalized dueAt column in sync.
|
||||
await db.runAsync(`UPDATE flashcards SET srs = ?, dueAt = ? WHERE id = ?`, [
|
||||
JSON.stringify(srs),
|
||||
srs.due,
|
||||
id,
|
||||
]);
|
||||
return rowToFlashcard({ ...row, srs: JSON.stringify(srs), dueAt: srs.due });
|
||||
},
|
||||
|
||||
async deleteFlashcard(id: string): Promise<void> {
|
||||
const db = await getDb();
|
||||
await db.runAsync(`DELETE FROM flashcards WHERE id = ?`, [id]);
|
||||
},
|
||||
|
||||
async flashcardCounts(
|
||||
courseId?: string | null,
|
||||
): Promise<{ total: number; due: number }> {
|
||||
const db = await getDb();
|
||||
const now = Date.now();
|
||||
const scopeWhere: string[] = [];
|
||||
const scopeParams: (string | null)[] = [];
|
||||
if (courseId !== undefined) {
|
||||
if (courseId === null) {
|
||||
scopeWhere.push('courseId IS NULL');
|
||||
} else {
|
||||
scopeWhere.push('courseId = ?');
|
||||
scopeParams.push(courseId);
|
||||
}
|
||||
}
|
||||
const scopeClause = scopeWhere.length ? ` WHERE ${scopeWhere.join(' AND ')}` : '';
|
||||
const totalRow = await db.getFirstAsync<{ n: number }>(
|
||||
`SELECT COUNT(*) AS n FROM flashcards${scopeClause}`,
|
||||
scopeParams,
|
||||
);
|
||||
const dueClause = scopeWhere.length
|
||||
? ` WHERE ${scopeWhere.join(' AND ')} AND dueAt <= ?`
|
||||
: ` WHERE dueAt <= ?`;
|
||||
const dueRow = await db.getFirstAsync<{ n: number }>(
|
||||
`SELECT COUNT(*) AS n FROM flashcards${dueClause}`,
|
||||
[...scopeParams, now],
|
||||
);
|
||||
return { total: totalRow?.n ?? 0, due: dueRow?.n ?? 0 };
|
||||
},
|
||||
};
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
parseDraft,
|
||||
type TranscriptDraft,
|
||||
type StorageRepo,
|
||||
type SegmentVector,
|
||||
} from './index';
|
||||
|
||||
// Populated by the migration beforeAll (which seeds v1 then imports repo.web).
|
||||
@@ -322,3 +323,391 @@ describe('StorageRepo (Dexie web impl)', () => {
|
||||
await expect(repo.create(makeDraft({ durationSec: -1 }))).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Phase 1: semantic-search vector store
|
||||
// ---------------------------------------------------------------------------
|
||||
//
|
||||
// We exercise the brute-force cosine store with tiny hand-made UNIT vectors so
|
||||
// the math is checkable by eye. The repo must not hard-code 384 dims — these
|
||||
// tests use 3-d vectors throughout. Cosine of two unit vectors is their dot
|
||||
// product, so [1,0,0]·[1,0,0] = 1 and [1,0,0]·[0,1,0] = 0.
|
||||
|
||||
const MODEL = 'Xenova/multilingual-e5-small';
|
||||
|
||||
// Three orthonormal basis vectors, used as stand-in segment embeddings.
|
||||
const E0 = () => new Float32Array([1, 0, 0]);
|
||||
const E1 = () => new Float32Array([0, 1, 0]);
|
||||
const E2 = () => new Float32Array([0, 0, 1]);
|
||||
|
||||
describe('vectors (semantic search store)', () => {
|
||||
beforeEach(async () => {
|
||||
// Wipe transcripts (cascades drop their vectors) + courses between tests.
|
||||
const all = await repo.list();
|
||||
await Promise.all(all.map((m) => repo.remove(m.id)));
|
||||
const courses = await repo.listCourses();
|
||||
await Promise.all(courses.map((c) => repo.deleteCourse(c.id)));
|
||||
});
|
||||
|
||||
// Create a transcript whose segments carry stable ids, then return it.
|
||||
async function makeTranscript(over: Partial<TranscriptDraft> = {}) {
|
||||
return repo.create(
|
||||
makeDraft({
|
||||
segments: [
|
||||
{ start: 0, end: 1, text: 'alpha' },
|
||||
{ start: 1, end: 2, text: 'beta' },
|
||||
{ start: 2, end: 3, text: 'gamma' },
|
||||
],
|
||||
...over,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// Build SegmentVector rows pairing each segment id with one of the vectors.
|
||||
function vecsFor(
|
||||
t: { segments: { id?: string; start: number; end: number; text: string }[] },
|
||||
vectors: Float32Array[],
|
||||
): SegmentVector[] {
|
||||
return t.segments.map((s, i) => ({
|
||||
segmentId: s.id!,
|
||||
start: s.start,
|
||||
end: s.end,
|
||||
text: s.text,
|
||||
vector: vectors[i]!,
|
||||
}));
|
||||
}
|
||||
|
||||
it('upsertVectors + searchVectors ranks the matching segment first with score ~1', async () => {
|
||||
const t = await makeTranscript();
|
||||
await repo.upsertVectors(t.id, MODEL, vecsFor(t, [E0(), E1(), E2()]));
|
||||
|
||||
const hits = await repo.searchVectors(E0());
|
||||
expect(hits).toHaveLength(3);
|
||||
// The [1,0,0] segment ("alpha") ranks first with cosine ~1.
|
||||
expect(hits[0]!.segmentId).toBe(t.segments[0]!.id);
|
||||
expect(hits[0]!.text).toBe('alpha');
|
||||
expect(hits[0]!.score).toBeCloseTo(1, 6);
|
||||
// The orthogonal segments score ~0.
|
||||
expect(hits[1]!.score).toBeCloseTo(0, 6);
|
||||
expect(hits[2]!.score).toBeCloseTo(0, 6);
|
||||
// Hits carry the segment-jump anchors + denormalized courseId.
|
||||
expect(hits[0]!.transcriptId).toBe(t.id);
|
||||
expect(hits[0]!.start).toBe(0);
|
||||
expect(hits[0]!.end).toBe(1);
|
||||
expect(hits[0]!.courseId).toBeNull();
|
||||
});
|
||||
|
||||
it('searchVectors honours the limit option', async () => {
|
||||
const t = await makeTranscript();
|
||||
await repo.upsertVectors(t.id, MODEL, vecsFor(t, [E0(), E1(), E2()]));
|
||||
|
||||
const hits = await repo.searchVectors(E0(), { limit: 1 });
|
||||
expect(hits).toHaveLength(1);
|
||||
expect(hits[0]!.segmentId).toBe(t.segments[0]!.id);
|
||||
});
|
||||
|
||||
it('upsertVectors replaces prior vectors for a transcript (no stale rows)', async () => {
|
||||
const t = await makeTranscript();
|
||||
await repo.upsertVectors(t.id, MODEL, vecsFor(t, [E0(), E1(), E2()]));
|
||||
// Re-embed with only the first two segments.
|
||||
await repo.upsertVectors(t.id, MODEL, [
|
||||
{
|
||||
segmentId: t.segments[0]!.id!,
|
||||
start: 0,
|
||||
end: 1,
|
||||
text: 'alpha',
|
||||
vector: E0(),
|
||||
},
|
||||
{
|
||||
segmentId: t.segments[1]!.id!,
|
||||
start: 1,
|
||||
end: 2,
|
||||
text: 'beta',
|
||||
vector: E1(),
|
||||
},
|
||||
]);
|
||||
const hits = await repo.searchVectors(E0());
|
||||
expect(hits).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('courseId filter scopes search (course vs Unsorted vs all)', async () => {
|
||||
const course = await repo.createCourse({ name: 'Physics' });
|
||||
const inCourse = await makeTranscript({ courseId: course.id });
|
||||
const unsorted = await makeTranscript();
|
||||
|
||||
await repo.upsertVectors(inCourse.id, MODEL, vecsFor(inCourse, [E0(), E1(), E2()]));
|
||||
await repo.upsertVectors(unsorted.id, MODEL, vecsFor(unsorted, [E0(), E1(), E2()]));
|
||||
|
||||
// Scoped to the course: only that transcript's rows are searched.
|
||||
const courseHits = await repo.searchVectors(E0(), { courseId: course.id });
|
||||
expect(courseHits.every((h) => h.transcriptId === inCourse.id)).toBe(true);
|
||||
expect(courseHits.every((h) => h.courseId === course.id)).toBe(true);
|
||||
expect(courseHits).toHaveLength(3);
|
||||
|
||||
// Scoped to Unsorted (null): only the unsorted transcript's rows.
|
||||
const unsortedHits = await repo.searchVectors(E0(), { courseId: null });
|
||||
expect(unsortedHits.every((h) => h.transcriptId === unsorted.id)).toBe(true);
|
||||
expect(unsortedHits.every((h) => h.courseId === null)).toBe(true);
|
||||
expect(unsortedHits).toHaveLength(3);
|
||||
|
||||
// No scope => everything (both transcripts).
|
||||
const allHits = await repo.searchVectors(E0());
|
||||
expect(allHits).toHaveLength(6);
|
||||
});
|
||||
|
||||
it('unembeddedIds excludes embedded transcripts and includes fresh ones', async () => {
|
||||
const embedded = await makeTranscript();
|
||||
const fresh = await makeTranscript();
|
||||
await repo.upsertVectors(embedded.id, MODEL, vecsFor(embedded, [E0(), E1(), E2()]));
|
||||
|
||||
const pending = await repo.unembeddedIds(MODEL);
|
||||
expect(pending).not.toContain(embedded.id);
|
||||
expect(pending).toContain(fresh.id);
|
||||
|
||||
// A different model id has no vectors at all => both are unembedded.
|
||||
const otherModel = await repo.unembeddedIds('some/other-model');
|
||||
expect(otherModel).toContain(embedded.id);
|
||||
expect(otherModel).toContain(fresh.id);
|
||||
});
|
||||
|
||||
it('clearVectors empties a transcript and re-adds it to unembeddedIds', async () => {
|
||||
const t = await makeTranscript();
|
||||
await repo.upsertVectors(t.id, MODEL, vecsFor(t, [E0(), E1(), E2()]));
|
||||
expect(await repo.searchVectors(E0())).toHaveLength(3);
|
||||
|
||||
await repo.clearVectors(t.id);
|
||||
expect(await repo.searchVectors(E0())).toHaveLength(0);
|
||||
expect(await repo.unembeddedIds(MODEL)).toContain(t.id);
|
||||
});
|
||||
|
||||
it('reassign updates the denormalized courseId on existing hits', async () => {
|
||||
const course = await repo.createCourse({ name: 'Chemistry' });
|
||||
const t = await makeTranscript();
|
||||
await repo.upsertVectors(t.id, MODEL, vecsFor(t, [E0(), E1(), E2()]));
|
||||
|
||||
// Initially Unsorted: hits carry courseId null and match the null scope.
|
||||
expect((await repo.searchVectors(E0(), { courseId: null }))).toHaveLength(3);
|
||||
|
||||
await repo.reassign(t.id, course.id);
|
||||
|
||||
// Now the hits carry the new courseId and match the course scope...
|
||||
const courseHits = await repo.searchVectors(E0(), { courseId: course.id });
|
||||
expect(courseHits).toHaveLength(3);
|
||||
expect(courseHits.every((h) => h.courseId === course.id)).toBe(true);
|
||||
// ...and no longer appear under Unsorted.
|
||||
expect(await repo.searchVectors(E0(), { courseId: null })).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('updating segments clears stale vectors (transcript becomes unembedded again)', async () => {
|
||||
const t = await makeTranscript();
|
||||
await repo.upsertVectors(t.id, MODEL, vecsFor(t, [E0(), E1(), E2()]));
|
||||
expect(await repo.unembeddedIds(MODEL)).not.toContain(t.id);
|
||||
|
||||
// Patching segments invalidates the embeddings => repo drops them.
|
||||
await repo.update(t.id, {
|
||||
segments: [{ start: 0, end: 1, text: 'rewritten' }],
|
||||
});
|
||||
|
||||
expect(await repo.searchVectors(E0())).toHaveLength(0);
|
||||
expect(await repo.unembeddedIds(MODEL)).toContain(t.id);
|
||||
});
|
||||
|
||||
it('remove cascades to a transcript vectors', async () => {
|
||||
const t = await makeTranscript();
|
||||
await repo.upsertVectors(t.id, MODEL, vecsFor(t, [E0(), E1(), E2()]));
|
||||
expect(await repo.searchVectors(E0())).toHaveLength(3);
|
||||
|
||||
await repo.remove(t.id);
|
||||
expect(await repo.searchVectors(E0())).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Phase 3: flashcards + SM-2 spaced repetition store
|
||||
// ---------------------------------------------------------------------------
|
||||
//
|
||||
// These exercise the storage-only flashcard methods: a brand-new card is due
|
||||
// immediately (initial srs.due ~ now); listDueFlashcards honours the due cutoff
|
||||
// and course scope; updateFlashcardSrs persists a recomputed schedule; counts,
|
||||
// deletes, and the remove(transcript) cascade all behave.
|
||||
|
||||
describe('flashcards', () => {
|
||||
beforeEach(async () => {
|
||||
// Wipe transcripts (cascades drop their flashcards) + courses between tests.
|
||||
const all = await repo.list();
|
||||
await Promise.all(all.map((m) => repo.remove(m.id)));
|
||||
const courses = await repo.listCourses();
|
||||
await Promise.all(courses.map((c) => repo.deleteCourse(c.id)));
|
||||
});
|
||||
|
||||
it('createFlashcards returns ids + an initial srs due ~ now', async () => {
|
||||
const t = await repo.create(makeDraft());
|
||||
const before = Date.now();
|
||||
const [card] = await repo.createFlashcards([
|
||||
{
|
||||
transcriptId: t.id,
|
||||
front: 'What is Q?',
|
||||
back: 'The answer.',
|
||||
segmentId: t.segments[0]!.id,
|
||||
start: 0,
|
||||
},
|
||||
]);
|
||||
const after = Date.now();
|
||||
|
||||
expect(card!.id).toMatch(/^f_/);
|
||||
expect(card!.transcriptId).toBe(t.id);
|
||||
expect(card!.front).toBe('What is Q?');
|
||||
expect(card!.back).toBe('The answer.');
|
||||
expect(card!.segmentId).toBe(t.segments[0]!.id);
|
||||
expect(card!.start).toBe(0);
|
||||
expect(card!.createdAt).toBeTypeOf('number');
|
||||
// Initial SM-2 state.
|
||||
expect(card!.srs.ease).toBe(2.5);
|
||||
expect(card!.srs.intervalDays).toBe(0);
|
||||
expect(card!.srs.reps).toBe(0);
|
||||
expect(card!.srs.lapses).toBe(0);
|
||||
// Due immediately: srs.due falls within the create() window.
|
||||
expect(card!.srs.due).toBeGreaterThanOrEqual(before);
|
||||
expect(card!.srs.due).toBeLessThanOrEqual(after);
|
||||
|
||||
// It round-trips through listFlashcards.
|
||||
const listed = await repo.listFlashcards({ transcriptId: t.id });
|
||||
expect(listed.map((c) => c.id)).toEqual([card!.id]);
|
||||
});
|
||||
|
||||
it('listDueFlashcards includes a due card and excludes a future-due one', async () => {
|
||||
const t = await repo.create(makeDraft());
|
||||
const [dueCard, futureCard] = await repo.createFlashcards([
|
||||
{ transcriptId: t.id, front: 'due', back: 'now' },
|
||||
{ transcriptId: t.id, front: 'later', back: 'future' },
|
||||
]);
|
||||
|
||||
// Push the second card's due far into the future via a recomputed schedule.
|
||||
const future = Date.now() + 7 * 24 * 60 * 60 * 1000;
|
||||
await repo.updateFlashcardSrs(futureCard!.id, {
|
||||
ease: 2.5,
|
||||
intervalDays: 7,
|
||||
reps: 1,
|
||||
lapses: 0,
|
||||
due: future,
|
||||
});
|
||||
|
||||
const due = await repo.listDueFlashcards();
|
||||
expect(due.map((c) => c.id)).toContain(dueCard!.id);
|
||||
expect(due.map((c) => c.id)).not.toContain(futureCard!.id);
|
||||
});
|
||||
|
||||
it('updateFlashcardSrs persists the recomputed schedule', async () => {
|
||||
const t = await repo.create(makeDraft());
|
||||
const [card] = await repo.createFlashcards([
|
||||
{ transcriptId: t.id, front: 'q', back: 'a' },
|
||||
]);
|
||||
|
||||
const reviewed = Date.now();
|
||||
const next = {
|
||||
ease: 2.6,
|
||||
intervalDays: 1,
|
||||
reps: 1,
|
||||
lapses: 0,
|
||||
due: reviewed + 24 * 60 * 60 * 1000,
|
||||
lastReviewed: reviewed,
|
||||
};
|
||||
const updated = await repo.updateFlashcardSrs(card!.id, next);
|
||||
expect(updated.srs).toEqual(next);
|
||||
|
||||
// The change is durable.
|
||||
const [reloaded] = await repo.listFlashcards({ transcriptId: t.id });
|
||||
expect(reloaded!.srs).toEqual(next);
|
||||
});
|
||||
|
||||
it('updateFlashcardSrs throws for an unknown id', async () => {
|
||||
await expect(
|
||||
repo.updateFlashcardSrs('f_nope', {
|
||||
ease: 2.5,
|
||||
intervalDays: 0,
|
||||
reps: 0,
|
||||
lapses: 0,
|
||||
due: Date.now(),
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('flashcardCounts reports total and due', async () => {
|
||||
const t = await repo.create(makeDraft());
|
||||
const [, b, c] = await repo.createFlashcards([
|
||||
{ transcriptId: t.id, front: '1', back: 'a' },
|
||||
{ transcriptId: t.id, front: '2', back: 'b' },
|
||||
{ transcriptId: t.id, front: '3', back: 'c' },
|
||||
]);
|
||||
|
||||
// All three start due.
|
||||
expect(await repo.flashcardCounts()).toEqual({ total: 3, due: 3 });
|
||||
|
||||
// Push two out into the future => only one remains due.
|
||||
const future = Date.now() + 60 * 60 * 1000;
|
||||
const futureSrs = {
|
||||
ease: 2.5,
|
||||
intervalDays: 1,
|
||||
reps: 1,
|
||||
lapses: 0,
|
||||
due: future,
|
||||
};
|
||||
await repo.updateFlashcardSrs(b!.id, futureSrs);
|
||||
await repo.updateFlashcardSrs(c!.id, futureSrs);
|
||||
|
||||
expect(await repo.flashcardCounts()).toEqual({ total: 3, due: 1 });
|
||||
});
|
||||
|
||||
it('deleteFlashcard removes a single card', async () => {
|
||||
const t = await repo.create(makeDraft());
|
||||
const [a, b] = await repo.createFlashcards([
|
||||
{ transcriptId: t.id, front: '1', back: 'a' },
|
||||
{ transcriptId: t.id, front: '2', back: 'b' },
|
||||
]);
|
||||
|
||||
await repo.deleteFlashcard(a!.id);
|
||||
const remaining = await repo.listFlashcards({ transcriptId: t.id });
|
||||
expect(remaining.map((c) => c.id)).toEqual([b!.id]);
|
||||
});
|
||||
|
||||
it('remove(transcript) cascades to its flashcards', async () => {
|
||||
const t = await repo.create(makeDraft());
|
||||
await repo.createFlashcards([
|
||||
{ transcriptId: t.id, front: '1', back: 'a' },
|
||||
{ transcriptId: t.id, front: '2', back: 'b' },
|
||||
]);
|
||||
expect(await repo.listFlashcards({ transcriptId: t.id })).toHaveLength(2);
|
||||
|
||||
await repo.remove(t.id);
|
||||
expect(await repo.listFlashcards({ transcriptId: t.id })).toHaveLength(0);
|
||||
expect(await repo.flashcardCounts()).toEqual({ total: 0, due: 0 });
|
||||
});
|
||||
|
||||
it('courseId scopes listing, due, and counts; deleteCourse moves cards to Unsorted', async () => {
|
||||
const course = await repo.createCourse({ name: 'Physics' });
|
||||
const inCourse = await repo.create(makeDraft({ courseId: course.id }));
|
||||
const unsorted = await repo.create(makeDraft());
|
||||
|
||||
await repo.createFlashcards([
|
||||
{ transcriptId: inCourse.id, courseId: course.id, front: 'c1', back: 'a' },
|
||||
]);
|
||||
await repo.createFlashcards([
|
||||
{ transcriptId: unsorted.id, front: 'u1', back: 'b' },
|
||||
]);
|
||||
|
||||
// Listing is scoped.
|
||||
expect((await repo.listFlashcards({ courseId: course.id })).map((c) => c.front)).toEqual(['c1']);
|
||||
expect((await repo.listFlashcards({ courseId: null })).map((c) => c.front)).toEqual(['u1']);
|
||||
|
||||
// Counts + due are scoped.
|
||||
expect(await repo.flashcardCounts(course.id)).toEqual({ total: 1, due: 1 });
|
||||
expect(await repo.flashcardCounts(null)).toEqual({ total: 1, due: 1 });
|
||||
expect((await repo.listDueFlashcards({ courseId: course.id })).map((c) => c.front)).toEqual(['c1']);
|
||||
|
||||
// Deleting the course moves its cards to Unsorted (courseId=null).
|
||||
await repo.deleteCourse(course.id);
|
||||
const movedToUnsorted = await repo.listFlashcards({ courseId: null });
|
||||
expect(movedToUnsorted.map((c) => c.front).sort()).toEqual(['c1', 'u1']);
|
||||
expect(await repo.listFlashcards({ courseId: course.id })).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,6 +12,9 @@ import type {
|
||||
TranscriptPatch,
|
||||
Course,
|
||||
CourseDraft,
|
||||
Flashcard,
|
||||
FlashcardDraft,
|
||||
SrsState,
|
||||
} from './schema';
|
||||
|
||||
/** Source of audio to persist for a transcript (web passes data, native a uri). */
|
||||
@@ -24,6 +27,27 @@ export interface MediaInput {
|
||||
mime: string;
|
||||
}
|
||||
|
||||
/** One stored embedding for a segment (vector is unit-normalized). */
|
||||
export interface SegmentVector {
|
||||
segmentId: string;
|
||||
start: number;
|
||||
end: number;
|
||||
text: string;
|
||||
vector: Float32Array;
|
||||
}
|
||||
|
||||
/** A semantic-search hit at segment granularity. */
|
||||
export interface VectorHit {
|
||||
transcriptId: string;
|
||||
segmentId: string;
|
||||
start: number;
|
||||
end: number;
|
||||
courseId: string | null;
|
||||
text: string;
|
||||
/** Cosine similarity (dot product of unit vectors). */
|
||||
score: number;
|
||||
}
|
||||
|
||||
export interface StorageRepo {
|
||||
// --- transcripts ---------------------------------------------------------
|
||||
/** All transcript metadatas, newest first (by createdAt desc). */
|
||||
@@ -84,4 +108,45 @@ export interface StorageRepo {
|
||||
|
||||
/** Delete the persisted audio for a transcript. No-op if absent. */
|
||||
removeMedia(transcriptId: string): Promise<void>;
|
||||
|
||||
// --- semantic search vectors (Phase 1) ----------------------------------
|
||||
/**
|
||||
* Replace all stored vectors for a transcript with `vectors`, tagged with the
|
||||
* embedding `model` id (so a model change can be detected and re-embedded).
|
||||
*/
|
||||
upsertVectors(transcriptId: string, model: string, vectors: SegmentVector[]): Promise<void>;
|
||||
|
||||
/**
|
||||
* Brute-force cosine search over stored vectors (optionally scoped to a
|
||||
* course; null = Unsorted), returning the top `limit` segment hits.
|
||||
*/
|
||||
searchVectors(
|
||||
query: Float32Array,
|
||||
opts?: { courseId?: string | null; limit?: number },
|
||||
): Promise<VectorHit[]>;
|
||||
|
||||
/** Drop all vectors for a transcript. */
|
||||
clearVectors(transcriptId: string): Promise<void>;
|
||||
|
||||
/** Transcript ids that have NO vectors for `model` (need embedding/backfill). */
|
||||
unembeddedIds(model: string): Promise<string[]>;
|
||||
|
||||
// --- flashcards + spaced repetition (Phase 3) ---------------------------
|
||||
/** Validate + persist new flashcards (each gets an id, createdAt, initial SRS). */
|
||||
createFlashcards(drafts: FlashcardDraft[]): Promise<Flashcard[]>;
|
||||
|
||||
/** All flashcards, optionally scoped to a transcript or course. */
|
||||
listFlashcards(opts?: { transcriptId?: string; courseId?: string | null }): Promise<Flashcard[]>;
|
||||
|
||||
/** Cards due for review (srs.due <= now), optionally course-scoped. */
|
||||
listDueFlashcards(opts?: { courseId?: string | null; now?: number; limit?: number }): Promise<Flashcard[]>;
|
||||
|
||||
/** Persist a recomputed SRS schedule after a review. */
|
||||
updateFlashcardSrs(id: string, srs: SrsState): Promise<Flashcard>;
|
||||
|
||||
/** Delete a flashcard. */
|
||||
deleteFlashcard(id: string): Promise<void>;
|
||||
|
||||
/** Counts for badges (optionally course-scoped). */
|
||||
flashcardCounts(courseId?: string | null): Promise<{ total: number; due: number }>;
|
||||
}
|
||||
|
||||
+302
-22
@@ -16,6 +16,7 @@ import { newId } from '../ids';
|
||||
import {
|
||||
parseDraft,
|
||||
parseCourseDraft,
|
||||
parseFlashcardDraft,
|
||||
zSegment,
|
||||
type TranscriptDraft,
|
||||
type TranscriptMeta,
|
||||
@@ -23,8 +24,16 @@ import {
|
||||
type TranscriptPatch,
|
||||
type Course,
|
||||
type CourseDraft,
|
||||
type Flashcard,
|
||||
type FlashcardDraft,
|
||||
type SrsState,
|
||||
} from './schema';
|
||||
import type { StorageRepo, MediaInput } from './repo';
|
||||
import type {
|
||||
StorageRepo,
|
||||
MediaInput,
|
||||
SegmentVector,
|
||||
VectorHit,
|
||||
} from './repo';
|
||||
|
||||
// The shape persisted in the 'transcripts' store: a full Transcript plus the
|
||||
// derived searchText column. searchText is never returned to callers.
|
||||
@@ -40,6 +49,22 @@ interface StoredMedia {
|
||||
mime: string;
|
||||
}
|
||||
|
||||
// One row in the 'segvecs' store: a single segment's embedding plus the
|
||||
// denormalized courseId of its owning transcript (so course-scoped search can
|
||||
// filter without joining back to transcripts) and the embedding model tag.
|
||||
// The compound key [transcriptId+segmentId] keeps one row per segment; the
|
||||
// transcriptId / courseId / model indexes drive the cascade + filter paths.
|
||||
interface StoredSegVec {
|
||||
transcriptId: string;
|
||||
segmentId: string;
|
||||
start: number;
|
||||
end: number;
|
||||
courseId: string | null;
|
||||
text: string;
|
||||
vector: Float32Array;
|
||||
model: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pure derivation helpers (shared by create/update within this file)
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -68,6 +93,21 @@ function toMeta(row: StoredTranscript): TranscriptMeta {
|
||||
return meta;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cosine similarity between two vectors. Both the query and stored vectors are
|
||||
* unit-normalized by contract, so this is just their dot product; we do not
|
||||
* re-normalize here. Mismatched lengths fall back to the common prefix so a
|
||||
* stale-dim row can never throw (it will simply score poorly).
|
||||
*/
|
||||
function dot(a: Float32Array, b: Float32Array): number {
|
||||
const n = Math.min(a.length, b.length);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
sum += a[i]! * b[i]!;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Dexie database
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -76,6 +116,12 @@ class WispDexie extends Dexie {
|
||||
transcripts!: Dexie.Table<StoredTranscript, string>;
|
||||
courses!: Dexie.Table<Course, string>;
|
||||
media!: Dexie.Table<StoredMedia, string>;
|
||||
// Keyed by the compound [transcriptId+segmentId]; secondary indexes on
|
||||
// transcriptId (cascade), courseId (scoped search) and model (backfill).
|
||||
segvecs!: Dexie.Table<StoredSegVec, [string, string]>;
|
||||
// Phase 3 flashcards, keyed by id; secondary indexes on transcriptId
|
||||
// (cascade on remove) and courseId (scoping + cascade on deleteCourse).
|
||||
flashcards!: Dexie.Table<Flashcard, string>;
|
||||
|
||||
constructor() {
|
||||
super('wisp');
|
||||
@@ -102,6 +148,26 @@ class WispDexie extends Dexie {
|
||||
row.segments = withSegmentIds(row.segments);
|
||||
});
|
||||
});
|
||||
// v3: add the segvecs store for Phase 1 on-device semantic search. The
|
||||
// existing transcripts/courses/media stores are re-declared unchanged so
|
||||
// Dexie keeps them; only the new store is added. No data backfill is needed
|
||||
// — vectors are produced lazily by the embedding pipeline.
|
||||
this.version(3).stores({
|
||||
transcripts: 'id, createdAt, courseId',
|
||||
courses: 'id, name',
|
||||
media: 'transcriptId',
|
||||
segvecs: '[transcriptId+segmentId], transcriptId, courseId, model',
|
||||
});
|
||||
// v4: add the flashcards store for Phase 3 spaced repetition. Existing
|
||||
// stores are re-declared unchanged so Dexie keeps them; only the new store
|
||||
// is added. No backfill — flashcards are produced lazily by the study UI.
|
||||
this.version(4).stores({
|
||||
transcripts: 'id, createdAt, courseId',
|
||||
courses: 'id, name',
|
||||
media: 'transcriptId',
|
||||
segvecs: '[transcriptId+segmentId], transcriptId, courseId, model',
|
||||
flashcards: 'id, transcriptId, courseId',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,15 +257,29 @@ export const repo: StorageRepo = {
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
await db.transcripts.put(updated);
|
||||
// Segments were patched => any stored vectors are now stale (they were
|
||||
// embedded from the old text/timing). Drop them so the backfill re-embeds.
|
||||
if (patch.segments !== undefined) {
|
||||
await db.segvecs.where('transcriptId').equals(id).delete();
|
||||
}
|
||||
return toTranscript(updated);
|
||||
},
|
||||
|
||||
async remove(id: string): Promise<void> {
|
||||
// Also delete any persisted media for this transcript.
|
||||
await db.transaction('rw', db.transcripts, db.media, async () => {
|
||||
await db.transcripts.delete(id);
|
||||
await db.media.delete(id);
|
||||
});
|
||||
// Also delete any persisted media + vectors + flashcards for this transcript.
|
||||
await db.transaction(
|
||||
'rw',
|
||||
db.transcripts,
|
||||
db.media,
|
||||
db.segvecs,
|
||||
db.flashcards,
|
||||
async () => {
|
||||
await db.transcripts.delete(id);
|
||||
await db.media.delete(id);
|
||||
await db.segvecs.where('transcriptId').equals(id).delete();
|
||||
await db.flashcards.where('transcriptId').equals(id).delete();
|
||||
},
|
||||
);
|
||||
},
|
||||
|
||||
async search(query: string): Promise<TranscriptMeta[]> {
|
||||
@@ -226,10 +306,18 @@ export const repo: StorageRepo = {
|
||||
async reassign(transcriptId: string, courseId: string | null): Promise<void> {
|
||||
const existing = await db.transcripts.get(transcriptId);
|
||||
if (!existing) return;
|
||||
await db.transcripts.put({
|
||||
...existing,
|
||||
courseId,
|
||||
updatedAt: Date.now(),
|
||||
await db.transaction('rw', db.transcripts, db.segvecs, async () => {
|
||||
await db.transcripts.put({
|
||||
...existing,
|
||||
courseId,
|
||||
updatedAt: Date.now(),
|
||||
});
|
||||
// Keep the denormalized courseId on every vector row in sync so
|
||||
// course-scoped search keeps matching after a move.
|
||||
await db.segvecs
|
||||
.where('transcriptId')
|
||||
.equals(transcriptId)
|
||||
.modify({ courseId });
|
||||
});
|
||||
},
|
||||
|
||||
@@ -281,18 +369,29 @@ export const repo: StorageRepo = {
|
||||
async deleteCourse(id: string): Promise<void> {
|
||||
// First reassign this course's transcripts to "Unsorted" (courseId=null),
|
||||
// THEN delete the course row, so no transcript is ever orphaned.
|
||||
await db.transaction('rw', db.transcripts, db.courses, async () => {
|
||||
const now = Date.now();
|
||||
const owned = await db.transcripts
|
||||
.filter((r) => r.courseId === id)
|
||||
.toArray();
|
||||
await Promise.all(
|
||||
owned.map((r) =>
|
||||
db.transcripts.put({ ...r, courseId: null, updatedAt: now }),
|
||||
),
|
||||
);
|
||||
await db.courses.delete(id);
|
||||
});
|
||||
await db.transaction(
|
||||
'rw',
|
||||
db.transcripts,
|
||||
db.courses,
|
||||
db.segvecs,
|
||||
db.flashcards,
|
||||
async () => {
|
||||
const now = Date.now();
|
||||
const owned = await db.transcripts
|
||||
.filter((r) => r.courseId === id)
|
||||
.toArray();
|
||||
await Promise.all(
|
||||
owned.map((r) =>
|
||||
db.transcripts.put({ ...r, courseId: null, updatedAt: now }),
|
||||
),
|
||||
);
|
||||
// Keep denormalized vector rows consistent with their now-Unsorted owners.
|
||||
await db.segvecs.where('courseId').equals(id).modify({ courseId: null });
|
||||
// Flashcards in this course follow their transcripts to Unsorted.
|
||||
await db.flashcards.where('courseId').equals(id).modify({ courseId: null });
|
||||
await db.courses.delete(id);
|
||||
},
|
||||
);
|
||||
},
|
||||
|
||||
// --- media (persisted source audio) -------------------------------------
|
||||
@@ -326,4 +425,185 @@ export const repo: StorageRepo = {
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
// --- semantic search vectors (Phase 1) ----------------------------------
|
||||
async upsertVectors(
|
||||
transcriptId: string,
|
||||
model: string,
|
||||
vectors: SegmentVector[],
|
||||
): Promise<void> {
|
||||
await db.transaction('rw', db.transcripts, db.segvecs, async () => {
|
||||
// Denormalize the transcript's current courseId onto each row so
|
||||
// course-scoped search never has to join back to transcripts.
|
||||
const owner = await db.transcripts.get(transcriptId);
|
||||
const courseId = owner ? owner.courseId ?? null : null;
|
||||
// Replace, not merge: drop every existing row for this transcript first
|
||||
// so a re-embed with fewer segments can't leave stale rows behind.
|
||||
await db.segvecs.where('transcriptId').equals(transcriptId).delete();
|
||||
const rows: StoredSegVec[] = vectors.map((v) => ({
|
||||
transcriptId,
|
||||
segmentId: v.segmentId,
|
||||
start: v.start,
|
||||
end: v.end,
|
||||
courseId,
|
||||
text: v.text,
|
||||
vector: v.vector,
|
||||
model,
|
||||
}));
|
||||
await db.segvecs.bulkPut(rows);
|
||||
});
|
||||
},
|
||||
|
||||
async searchVectors(
|
||||
query: Float32Array,
|
||||
opts?: { courseId?: string | null; limit?: number },
|
||||
): Promise<VectorHit[]> {
|
||||
// Scope: when opts.courseId is provided we filter (null => Unsorted, i.e.
|
||||
// courseId == null); when it is omitted entirely we search every course.
|
||||
let rows: StoredSegVec[];
|
||||
if (opts && opts.courseId !== undefined) {
|
||||
const scope = opts.courseId;
|
||||
if (scope === null) {
|
||||
// IndexedDB cannot index null keys, so Unsorted is filtered in memory.
|
||||
const all = await db.segvecs.toArray();
|
||||
rows = all.filter((r) => r.courseId === null);
|
||||
} else {
|
||||
rows = await db.segvecs.where('courseId').equals(scope).toArray();
|
||||
}
|
||||
} else {
|
||||
rows = await db.segvecs.toArray();
|
||||
}
|
||||
|
||||
const limit = opts?.limit ?? 30;
|
||||
const hits: VectorHit[] = rows.map((r) => ({
|
||||
transcriptId: r.transcriptId,
|
||||
segmentId: r.segmentId,
|
||||
start: r.start,
|
||||
end: r.end,
|
||||
courseId: r.courseId,
|
||||
text: r.text,
|
||||
score: dot(query, r.vector),
|
||||
}));
|
||||
hits.sort((a, b) => b.score - a.score);
|
||||
return hits.slice(0, limit);
|
||||
},
|
||||
|
||||
async clearVectors(transcriptId: string): Promise<void> {
|
||||
await db.segvecs.where('transcriptId').equals(transcriptId).delete();
|
||||
},
|
||||
|
||||
async unembeddedIds(model: string): Promise<string[]> {
|
||||
// Transcript ids with ZERO segvecs rows for this model. We collect the set
|
||||
// of embedded ids for the model, then return every transcript id not in it.
|
||||
const embedded = new Set<string>();
|
||||
await db.segvecs
|
||||
.where('model')
|
||||
.equals(model)
|
||||
.each((row) => {
|
||||
embedded.add(row.transcriptId);
|
||||
});
|
||||
const ids = await db.transcripts.toCollection().primaryKeys();
|
||||
return ids.filter((id) => !embedded.has(id));
|
||||
},
|
||||
|
||||
// --- flashcards + spaced repetition (Phase 3) ---------------------------
|
||||
async createFlashcards(drafts: FlashcardDraft[]): Promise<Flashcard[]> {
|
||||
const now = Date.now();
|
||||
const cards: Flashcard[] = drafts.map((draft) => {
|
||||
const valid = parseFlashcardDraft(draft);
|
||||
// Initial SM-2 state for a brand-new card: due immediately (now). We
|
||||
// compute the SRS inline here rather than importing learn/srs to keep the
|
||||
// repo storage-only (no dependency on the study logic).
|
||||
const srs: SrsState = {
|
||||
ease: 2.5,
|
||||
intervalDays: 0,
|
||||
reps: 0,
|
||||
lapses: 0,
|
||||
due: now,
|
||||
};
|
||||
return {
|
||||
id: newId('f_'),
|
||||
transcriptId: valid.transcriptId,
|
||||
// courseId defaults to null ("Unsorted") when absent.
|
||||
courseId: valid.courseId ?? null,
|
||||
...(valid.segmentId !== undefined ? { segmentId: valid.segmentId } : {}),
|
||||
...(valid.start !== undefined ? { start: valid.start } : {}),
|
||||
front: valid.front,
|
||||
back: valid.back,
|
||||
createdAt: now,
|
||||
srs,
|
||||
};
|
||||
});
|
||||
await db.flashcards.bulkPut(cards);
|
||||
return cards;
|
||||
},
|
||||
|
||||
async listFlashcards(opts?: {
|
||||
transcriptId?: string;
|
||||
courseId?: string | null;
|
||||
}): Promise<Flashcard[]> {
|
||||
let rows = await db.flashcards.toArray();
|
||||
if (opts?.transcriptId !== undefined) {
|
||||
rows = rows.filter((c) => c.transcriptId === opts.transcriptId);
|
||||
}
|
||||
if (opts && opts.courseId !== undefined) {
|
||||
const scope = opts.courseId;
|
||||
rows = rows.filter((c) =>
|
||||
// null scope => Unsorted (courseId null or undefined).
|
||||
scope === null ? c.courseId == null : c.courseId === scope,
|
||||
);
|
||||
}
|
||||
return rows;
|
||||
},
|
||||
|
||||
async listDueFlashcards(opts?: {
|
||||
courseId?: string | null;
|
||||
now?: number;
|
||||
limit?: number;
|
||||
}): Promise<Flashcard[]> {
|
||||
const now = opts?.now ?? Date.now();
|
||||
let rows = await db.flashcards.toArray();
|
||||
if (opts && opts.courseId !== undefined) {
|
||||
const scope = opts.courseId;
|
||||
rows = rows.filter((c) =>
|
||||
scope === null ? c.courseId == null : c.courseId === scope,
|
||||
);
|
||||
}
|
||||
// Due means the next-review time is now or in the past.
|
||||
rows = rows.filter((c) => c.srs.due <= now);
|
||||
// Soonest-due first.
|
||||
rows.sort((a, b) => a.srs.due - b.srs.due);
|
||||
if (opts?.limit !== undefined) {
|
||||
rows = rows.slice(0, opts.limit);
|
||||
}
|
||||
return rows;
|
||||
},
|
||||
|
||||
async updateFlashcardSrs(id: string, srs: SrsState): Promise<Flashcard> {
|
||||
const card = await db.flashcards.get(id);
|
||||
if (!card) {
|
||||
throw new Error(`flashcard not found: ${id}`);
|
||||
}
|
||||
const updated: Flashcard = { ...card, srs };
|
||||
await db.flashcards.put(updated);
|
||||
return updated;
|
||||
},
|
||||
|
||||
async deleteFlashcard(id: string): Promise<void> {
|
||||
await db.flashcards.delete(id);
|
||||
},
|
||||
|
||||
async flashcardCounts(
|
||||
courseId?: string | null,
|
||||
): Promise<{ total: number; due: number }> {
|
||||
const now = Date.now();
|
||||
let rows = await db.flashcards.toArray();
|
||||
if (courseId !== undefined) {
|
||||
rows = rows.filter((c) =>
|
||||
courseId === null ? c.courseId == null : c.courseId === courseId,
|
||||
);
|
||||
}
|
||||
const due = rows.filter((c) => c.srs.due <= now).length;
|
||||
return { total: rows.length, due };
|
||||
},
|
||||
};
|
||||
|
||||
@@ -136,6 +136,48 @@ export function parseCourseDraft(input: unknown): CourseDraft {
|
||||
return zCourseDraft.parse(input);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Flashcards + spaced repetition (Phase 3)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** SM-2 spaced-repetition state for a card. */
|
||||
export interface SrsState {
|
||||
/** SM-2 ease factor (starts 2.5, floored at 1.3). */
|
||||
ease: number;
|
||||
/** Current interval in days. */
|
||||
intervalDays: number;
|
||||
/** Successful repetitions in a row. */
|
||||
reps: number;
|
||||
/** Times the card was forgotten. */
|
||||
lapses: number;
|
||||
/** Next-due time, ms since epoch. */
|
||||
due: number;
|
||||
/** Last review time, ms since epoch. */
|
||||
lastReviewed?: number;
|
||||
}
|
||||
|
||||
export const zFlashcardDraft = z.object({
|
||||
transcriptId: z.string(),
|
||||
courseId: z.string().nullable().optional(),
|
||||
/** Source segment for click-to-seek back to the lecture. */
|
||||
segmentId: z.string().optional(),
|
||||
start: zFiniteNonNeg.optional(),
|
||||
front: z.string().min(1),
|
||||
back: z.string().min(1),
|
||||
});
|
||||
export type FlashcardDraft = z.infer<typeof zFlashcardDraft>;
|
||||
|
||||
/** A stored flashcard with its SRS schedule. */
|
||||
export interface Flashcard extends FlashcardDraft {
|
||||
id: string;
|
||||
createdAt: number;
|
||||
srs: SrsState;
|
||||
}
|
||||
|
||||
export function parseFlashcardDraft(input: unknown): FlashcardDraft {
|
||||
return zFlashcardDraft.parse(input);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stored types (output of the repo)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
// Native (iOS/Android) implementation of downloadText: there's no browser
|
||||
// download, so we write the content to a cache file and hand it to the OS share
|
||||
// sheet (Save to Files, AirDrop, send to another app, …). Previously this path
|
||||
// returned false and the export buttons (transcript, ICS, Anki CSV, BibTeX/RIS)
|
||||
// silently did nothing on device.
|
||||
|
||||
import * as Sharing from 'expo-sharing';
|
||||
import { writeAsStringAsync, cacheDirectory } from 'expo-file-system/legacy';
|
||||
|
||||
/** Sanitize a filename for use as a path segment (keep word chars, dot, dash). */
|
||||
function safeName(name: string): string {
|
||||
const cleaned = name.replace(/[^\w.\-]+/g, '_').replace(/^_+|_+$/g, '');
|
||||
return cleaned.length > 0 ? cleaned : 'export';
|
||||
}
|
||||
|
||||
export async function downloadText(
|
||||
filename: string,
|
||||
mime: string,
|
||||
content: string,
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const dir = cacheDirectory;
|
||||
if (!dir || !(await Sharing.isAvailableAsync())) return false;
|
||||
const uri = `${dir}${safeName(filename)}`;
|
||||
await writeAsStringAsync(uri, content);
|
||||
await Sharing.shareAsync(uri, { mimeType: mime, dialogTitle: filename });
|
||||
return true;
|
||||
} catch {
|
||||
// User-cancelled share or any IO error: report failure so callers can decide
|
||||
// (today they ignore it, but the return keeps the contract honest).
|
||||
return false;
|
||||
}
|
||||
}
|
||||
+9
-8
@@ -1,11 +1,12 @@
|
||||
import { Platform } from 'react-native';
|
||||
|
||||
/**
|
||||
* Trigger a client-side file download on web (no server, nothing uploaded).
|
||||
* Returns false on native, where the caller should fall back to share/save.
|
||||
*/
|
||||
export function downloadText(filename: string, mime: string, content: string): boolean {
|
||||
if (Platform.OS !== 'web') return false;
|
||||
// Web implementation (also the tsc base; native uses download.native.ts).
|
||||
//
|
||||
// Triggers a client-side file download on web (no server, nothing uploaded).
|
||||
// Async so the signature matches the native share-sheet implementation.
|
||||
export async function downloadText(
|
||||
filename: string,
|
||||
mime: string,
|
||||
content: string,
|
||||
): Promise<boolean> {
|
||||
const blob = new Blob([content], { type: mime });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
// The embedding-engine interface: hides the platform-specific sentence-embedding
|
||||
// backend (transformers.js feature-extraction on web; a follow-up native impl)
|
||||
// behind one contract, mirroring TranscriptionEngine. Used for on-device
|
||||
// semantic search (ROADMAP Phase 1) — vectors never leave the device.
|
||||
|
||||
export interface EmbeddingEngine {
|
||||
readonly platform: 'web' | 'native';
|
||||
/** Vector dimensionality (must match EMBED_DIM). */
|
||||
readonly dim: number;
|
||||
|
||||
/** Synchronous check: is the model loaded in memory? */
|
||||
isLoaded(): boolean;
|
||||
|
||||
/** Ensure the model is loaded. `onProgress` (0..1) fires during download. */
|
||||
loadModel(onProgress?: (p: number) => void): Promise<void>;
|
||||
|
||||
/**
|
||||
* Embed texts into unit-normalized, mean-pooled vectors. `kind` adds the
|
||||
* asymmetric prefix the model expects (e5: "query:" for search text,
|
||||
* "passage:" for stored segments) so retrieval is well-calibrated.
|
||||
*/
|
||||
embed(texts: string[], kind: 'query' | 'passage'): Promise<Float32Array[]>;
|
||||
}
|
||||
|
||||
/** Embedding model id — also stored alongside vectors to detect model changes. */
|
||||
export const EMBED_MODEL = 'Xenova/multilingual-e5-small';
|
||||
/** Output dimensionality of EMBED_MODEL. */
|
||||
export const EMBED_DIM = 384;
|
||||
@@ -0,0 +1,28 @@
|
||||
// NATIVE-ONLY embedding engine stub. On-device sentence embeddings on native
|
||||
// (a transformers/onnxruntime-react-native or whisper.rn-style binding) are a
|
||||
// follow-up; until then this is a type-correct placeholder that throws clearly
|
||||
// so the rest of the app stays platform-agnostic and never silently mis-embeds.
|
||||
//
|
||||
// Selected by Metro at build time for native targets; never imported by tests.
|
||||
|
||||
import type { EmbeddingEngine } from './engine';
|
||||
import { EMBED_DIM } from './engine';
|
||||
|
||||
const NOT_AVAILABLE = 'On-device embeddings are not available on native yet';
|
||||
|
||||
export const engine: EmbeddingEngine = {
|
||||
platform: 'native',
|
||||
dim: EMBED_DIM,
|
||||
|
||||
isLoaded(): boolean {
|
||||
return false;
|
||||
},
|
||||
|
||||
async loadModel(): Promise<void> {
|
||||
throw new Error(NOT_AVAILABLE);
|
||||
},
|
||||
|
||||
async embed(): Promise<Float32Array[]> {
|
||||
throw new Error(NOT_AVAILABLE);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,3 @@
|
||||
// Base resolver for TypeScript. Metro picks engineImpl.web.ts / engineImpl.native.ts
|
||||
// by platform extension at build time; tsc resolves this file (defaults to web).
|
||||
export { engine } from './engineImpl.web';
|
||||
@@ -0,0 +1,114 @@
|
||||
// WEB-ONLY embedding engine, backed by transformers.js (@huggingface/
|
||||
// transformers) running the multilingual-e5-small feature-extraction model in
|
||||
// the browser (WebGPU when available, else WASM). Produces unit-normalized,
|
||||
// mean-pooled 384-d vectors for on-device semantic search (Phase 1).
|
||||
//
|
||||
// WHY WE LOAD IT FROM A CDN AT RUNTIME (not a static import):
|
||||
// transformers.js depends on onnxruntime-web, which uses a *computed* dynamic
|
||||
// import (`import(/*webpackIgnore*/ a)`) and ships WASM — Metro (Expo's web
|
||||
// bundler) cannot statically bundle either and fails the build. So we never let
|
||||
// Metro see the package: we load the ESM build from a CDN at runtime via a
|
||||
// dynamic import hidden behind `new Function` (so Metro's static analyzer can't
|
||||
// trip over it). This mirrors src/lib/transcription/engineImpl.web.ts exactly.
|
||||
//
|
||||
// NOTE: the page must be cross-origin isolated (COOP + COEP) for multi-threaded
|
||||
// WASM; see docker/nginx.conf. This module is web-only and is NEVER imported by
|
||||
// any vitest test.
|
||||
|
||||
import type { EmbeddingEngine } from './engine';
|
||||
import { EMBED_DIM, EMBED_MODEL } from './engine';
|
||||
|
||||
// Pin the transformers.js version we load at runtime (matches the ASR engine).
|
||||
const TRANSFORMERS_CDN = 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0';
|
||||
|
||||
// `new Function` hides the dynamic import() specifier from Metro's bundler so it
|
||||
// never tries to resolve/transform transformers.js or onnxruntime-web.
|
||||
const runtimeImport = new Function('u', 'return import(u)') as (u: string) => Promise<TransformersModule>;
|
||||
|
||||
// Minimal structural types for the bits of transformers.js we use.
|
||||
interface FeatureTensor {
|
||||
/** Nested arrays: one row of floats per input text. */
|
||||
tolist(): number[][];
|
||||
}
|
||||
type FeatureExtractor = (
|
||||
texts: string[],
|
||||
opts: { pooling: 'mean'; normalize: boolean },
|
||||
) => Promise<FeatureTensor>;
|
||||
interface PipelineOptions {
|
||||
device?: string;
|
||||
dtype?: string;
|
||||
progress_callback?: (e: { status?: string; progress?: number }) => void;
|
||||
}
|
||||
interface TransformersModule {
|
||||
pipeline: (task: string, model: string, opts?: PipelineOptions) => Promise<FeatureExtractor>;
|
||||
env: { allowLocalModels: boolean };
|
||||
}
|
||||
|
||||
let libPromise: Promise<TransformersModule> | null = null;
|
||||
async function lib(): Promise<TransformersModule> {
|
||||
if (!libPromise) {
|
||||
libPromise = runtimeImport(TRANSFORMERS_CDN).then((m) => {
|
||||
// Never read models off the local filesystem in the browser.
|
||||
m.env.allowLocalModels = false;
|
||||
return m;
|
||||
});
|
||||
}
|
||||
return libPromise;
|
||||
}
|
||||
|
||||
/** The single cached feature-extraction pipeline (one model). */
|
||||
let extractor: FeatureExtractor | null = null;
|
||||
let cachedWebGpu: boolean | undefined;
|
||||
|
||||
async function detectWebGpu(): Promise<boolean> {
|
||||
if (cachedWebGpu !== undefined) return cachedWebGpu;
|
||||
try {
|
||||
if (typeof navigator === 'undefined' || !('gpu' in navigator)) {
|
||||
cachedWebGpu = false;
|
||||
return false;
|
||||
}
|
||||
const gpu = (navigator as { gpu?: { requestAdapter(): Promise<unknown> } }).gpu;
|
||||
const adapter = await gpu?.requestAdapter();
|
||||
cachedWebGpu = adapter != null;
|
||||
} catch {
|
||||
cachedWebGpu = false;
|
||||
}
|
||||
return cachedWebGpu;
|
||||
}
|
||||
|
||||
export const engine: EmbeddingEngine = {
|
||||
platform: 'web',
|
||||
dim: EMBED_DIM,
|
||||
|
||||
isLoaded(): boolean {
|
||||
return extractor != null;
|
||||
},
|
||||
|
||||
async loadModel(onProgress?: (p: number) => void): Promise<void> {
|
||||
if (extractor != null) return; // idempotent
|
||||
const { pipeline } = await lib();
|
||||
const webgpu = await detectWebGpu();
|
||||
extractor = await pipeline('feature-extraction', EMBED_MODEL, {
|
||||
// WebGPU + fp16 when available; otherwise 8-bit weights on WASM, which
|
||||
// stays small to download and runs acceptably on a plain CPU.
|
||||
device: webgpu ? 'webgpu' : 'wasm',
|
||||
dtype: webgpu ? 'fp16' : 'q8',
|
||||
progress_callback: (e) => {
|
||||
if (e.status === 'progress' && e.progress != null) onProgress?.(e.progress / 100);
|
||||
},
|
||||
});
|
||||
},
|
||||
|
||||
async embed(texts: string[], kind: 'query' | 'passage'): Promise<Float32Array[]> {
|
||||
if (extractor == null) {
|
||||
throw new Error('Embedding model is not loaded; call loadModel() first.');
|
||||
}
|
||||
if (texts.length === 0) return [];
|
||||
// e5 asymmetric convention: prefix with "query: " or "passage: ".
|
||||
const prefix = `${kind}: `;
|
||||
const prefixed = texts.map((t) => prefix + t);
|
||||
const tensor = await extractor(prefixed, { pooling: 'mean', normalize: true });
|
||||
// tolist() yields one number[] row per input; convert each to Float32Array.
|
||||
return tensor.tolist().map((row) => Float32Array.from(row));
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,12 @@
|
||||
// Public entry point for the embedding engine.
|
||||
//
|
||||
// `./engineImpl` is resolved by Metro to engineImpl.web.ts or
|
||||
// engineImpl.native.ts by platform extension; the base engineImpl.ts re-export
|
||||
// (web) is what TypeScript resolves for typechecking. Consumers call
|
||||
// getEmbeddingEngine() and stay platform-agnostic.
|
||||
import { engine } from './engineImpl';
|
||||
|
||||
/** Return the platform-resolved embedding engine. */
|
||||
export const getEmbeddingEngine = () => engine;
|
||||
|
||||
export * from './engine';
|
||||
@@ -0,0 +1,78 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { toBibtex, toRis } from './bib';
|
||||
import type { RefMetadata } from './types';
|
||||
|
||||
describe('toBibtex', () => {
|
||||
it('returns "" for []', () => {
|
||||
expect(toBibtex([])).toBe('');
|
||||
});
|
||||
|
||||
it('emits an @article with a first-author+year key', () => {
|
||||
const refs: RefMetadata[] = [
|
||||
{
|
||||
title: 'Attention Is All You Need',
|
||||
authors: ['Ashish Vaswani', 'Noam Shazeer'],
|
||||
year: 2017,
|
||||
source: 'Crossref',
|
||||
url: 'https://doi.org/10.x',
|
||||
},
|
||||
];
|
||||
const out = toBibtex(refs);
|
||||
expect(out).toContain('@article{vaswani2017,');
|
||||
expect(out).toContain('title = {Attention Is All You Need}');
|
||||
expect(out).toContain('author = {Ashish Vaswani and Noam Shazeer}');
|
||||
expect(out).toContain('year = {2017}');
|
||||
expect(out).toContain('url = {https://doi.org/10.x}');
|
||||
});
|
||||
|
||||
it('uses @misc and ref<index> key when no authors', () => {
|
||||
const out = toBibtex([{ title: 'A book', source: 'Open Library' }]);
|
||||
expect(out).toContain('@misc{ref0,');
|
||||
expect(out).toContain('title = {A book}');
|
||||
});
|
||||
|
||||
it('omits missing fields', () => {
|
||||
const out = toBibtex([{ source: 'X' }]);
|
||||
expect(out).toContain('@misc{ref0,');
|
||||
expect(out).not.toContain('title =');
|
||||
expect(out).not.toContain('year =');
|
||||
});
|
||||
|
||||
it('brace-escapes special characters in values', () => {
|
||||
const out = toBibtex([{ title: 'a {b} c', source: 'X' }]);
|
||||
expect(out).toContain('title = {a \\{b\\} c}');
|
||||
});
|
||||
});
|
||||
|
||||
describe('toRis', () => {
|
||||
it('returns "" for []', () => {
|
||||
expect(toRis([])).toBe('');
|
||||
});
|
||||
|
||||
it('emits a JOUR record with one AU line per author', () => {
|
||||
const out = toRis([
|
||||
{
|
||||
title: 'T',
|
||||
authors: ['A One', 'B Two'],
|
||||
year: 2020,
|
||||
source: 'X',
|
||||
url: 'http://u',
|
||||
},
|
||||
]);
|
||||
expect(out).toContain('TY - JOUR');
|
||||
expect(out).toContain('TI - T');
|
||||
expect(out).toContain('AU - A One');
|
||||
expect(out).toContain('AU - B Two');
|
||||
expect(out).toContain('PY - 2020');
|
||||
expect(out).toContain('UR - http://u');
|
||||
expect(out.split('\n')).toContain('ER - ');
|
||||
});
|
||||
|
||||
it('uses GEN and omits missing fields', () => {
|
||||
const out = toRis([{ source: 'X' }]);
|
||||
expect(out).toContain('TY - GEN');
|
||||
expect(out).not.toContain('TI -');
|
||||
expect(out).not.toContain('AU -');
|
||||
expect(out).toContain('ER - ');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,80 @@
|
||||
// Pure bibliography serialization: BibTeX and RIS. No network, no I/O.
|
||||
|
||||
import type { RefMetadata } from './types';
|
||||
|
||||
/** Serialize references to BibTeX. Handles missing fields and []. */
|
||||
export function toBibtex(refs: RefMetadata[]): string {
|
||||
// BibTeX entry keys MUST be unique — same-surname/same-year refs would
|
||||
// otherwise collide and most processors drop the duplicate on import. Track
|
||||
// emitted keys and disambiguate collisions with an a/b/c… suffix.
|
||||
const used = new Map<string, number>();
|
||||
return refs
|
||||
.map((r, i) => {
|
||||
let key = bibKey(r, i);
|
||||
const seen = used.get(key);
|
||||
if (seen !== undefined) {
|
||||
const suffix = String.fromCharCode(97 + seen); // 0 -> 'a', 1 -> 'b', …
|
||||
used.set(key, seen + 1);
|
||||
key = `${key}${suffix}`;
|
||||
} else {
|
||||
used.set(key, 0);
|
||||
}
|
||||
return bibEntry(r, key);
|
||||
})
|
||||
.join('\n\n');
|
||||
}
|
||||
|
||||
function bibEntry(r: RefMetadata, key: string): string {
|
||||
const type = r.authors && r.authors.length ? 'article' : 'misc';
|
||||
|
||||
const fields: string[] = [];
|
||||
if (r.title) fields.push(` title = {${brace(r.title)}}`);
|
||||
if (r.authors && r.authors.length) {
|
||||
fields.push(` author = {${brace(r.authors.join(' and '))}}`);
|
||||
}
|
||||
if (r.year !== undefined) fields.push(` year = {${r.year}}`);
|
||||
if (r.url) fields.push(` url = {${brace(r.url)}}`);
|
||||
|
||||
return `@${type}{${key},\n${fields.join(',\n')}\n}`;
|
||||
}
|
||||
|
||||
function bibKey(r: RefMetadata, index: number): string {
|
||||
const first = r.authors && r.authors[0];
|
||||
if (first) {
|
||||
// Last token of the first author's name, alphanumerics only.
|
||||
const tokens = first.trim().split(/[\s,]+/);
|
||||
const surname = (tokens[tokens.length - 1] ?? first)
|
||||
.replace(/[^A-Za-z0-9]/g, '')
|
||||
.toLowerCase();
|
||||
if (surname) return `${surname}${r.year ?? ''}`;
|
||||
}
|
||||
return `ref${index}`;
|
||||
}
|
||||
|
||||
/** Escape characters special to BibTeX brace-delimited values. */
|
||||
function brace(s: string): string {
|
||||
// Inside `{...}` we still must escape a stray backslash and the braces.
|
||||
return s.replace(/\\/g, '\\\\').replace(/[{}]/g, (m) => `\\${m}`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// RIS
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Serialize references to RIS. Handles missing fields and []. */
|
||||
export function toRis(refs: RefMetadata[]): string {
|
||||
return refs.map(risEntry).join('\n\n');
|
||||
}
|
||||
|
||||
function risEntry(r: RefMetadata): string {
|
||||
const ty = r.authors && r.authors.length ? 'JOUR' : 'GEN';
|
||||
const lines: string[] = [`TY - ${ty}`];
|
||||
if (r.title) lines.push(`TI - ${r.title}`);
|
||||
if (r.authors) {
|
||||
for (const a of r.authors) lines.push(`AU - ${a}`);
|
||||
}
|
||||
if (r.year !== undefined) lines.push(`PY - ${r.year}`);
|
||||
if (r.url) lines.push(`UR - ${r.url}`);
|
||||
lines.push('ER - ');
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { detectCitations } from './citations';
|
||||
import type { Segment } from '../types';
|
||||
|
||||
const seg = (start: number, text: string, id?: string): Segment => ({
|
||||
id,
|
||||
start,
|
||||
end: start + 1,
|
||||
text,
|
||||
});
|
||||
|
||||
describe('detectCitations', () => {
|
||||
it('returns [] for empty input', () => {
|
||||
expect(detectCitations([])).toEqual([]);
|
||||
});
|
||||
|
||||
it('extracts a bare DOI', () => {
|
||||
const out = detectCitations([seg(0, 'See 10.1038/nphys1170 for details.')]);
|
||||
const doi = out.find((c) => c.kind === 'doi');
|
||||
expect(doi?.value).toBe('10.1038/nphys1170');
|
||||
});
|
||||
|
||||
it('extracts a DOI from a doi.org URL and a doi: prefix', () => {
|
||||
const a = detectCitations([seg(0, 'https://doi.org/10.1145/3292500.3330701')]);
|
||||
expect(a[0]!.value).toBe('10.1145/3292500.3330701');
|
||||
const b = detectCitations([seg(0, 'doi: 10.1000/xyz123')]);
|
||||
expect(b[0]!.value).toBe('10.1000/xyz123');
|
||||
});
|
||||
|
||||
it('trims trailing punctuation from a DOI', () => {
|
||||
const out = detectCitations([seg(0, '(see 10.1038/nphys1170).')]);
|
||||
expect(out[0]!.value).toBe('10.1038/nphys1170');
|
||||
});
|
||||
|
||||
it('extracts an explicit arXiv id', () => {
|
||||
const out = detectCitations([seg(0, 'Read arXiv:2103.00020 today.')]);
|
||||
const ax = out.find((c) => c.kind === 'arxiv');
|
||||
expect(ax?.value).toBe('2103.00020');
|
||||
});
|
||||
|
||||
it('extracts a bare arXiv id', () => {
|
||||
const out = detectCitations([seg(0, 'the paper 1706.03762 introduced it')]);
|
||||
const ax = out.find((c) => c.kind === 'arxiv');
|
||||
expect(ax?.value).toBe('1706.03762');
|
||||
});
|
||||
|
||||
it('extracts ISBN-13 and ISBN-10, normalized without hyphens', () => {
|
||||
const a = detectCitations([seg(0, 'ISBN 978-0-13-468599-1')]);
|
||||
expect(a.find((c) => c.kind === 'isbn')?.value).toBe('9780134685991');
|
||||
const b = detectCitations([seg(0, 'ISBN-10: 0-262-03384-4')]);
|
||||
expect(b.find((c) => c.kind === 'isbn')?.value).toBe('0262033844');
|
||||
});
|
||||
|
||||
it('extracts author-year (parenthesized)', () => {
|
||||
const out = detectCitations([seg(0, 'As Vaswani (2017) showed,')]);
|
||||
const ay = out.find((c) => c.kind === 'author-year');
|
||||
expect(ay?.value).toBe('Vaswani (2017)');
|
||||
});
|
||||
|
||||
it('extracts "Author et al., Year"', () => {
|
||||
const out = detectCitations([seg(0, 'per Smith et al., 2021 this holds')]);
|
||||
const ay = out.find((c) => c.kind === 'author-year');
|
||||
expect(ay?.value).toBe('Smith et al. (2021)');
|
||||
});
|
||||
|
||||
it('dedupes by value', () => {
|
||||
const out = detectCitations([
|
||||
seg(0, '10.1038/nphys1170'),
|
||||
seg(1, 'again 10.1038/nphys1170'),
|
||||
]);
|
||||
expect(out.filter((c) => c.kind === 'doi')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('tags sourceSegmentId and start', () => {
|
||||
const out = detectCitations([seg(9, 'arXiv:2103.00020', 'segA')]);
|
||||
expect(out[0]!.sourceSegmentId).toBe('segA');
|
||||
expect(out[0]!.start).toBe(9);
|
||||
});
|
||||
|
||||
it('does not treat plain prose years as citations', () => {
|
||||
const out = detectCitations([seg(0, 'In 2017 the weather was nice.')]);
|
||||
expect(out.filter((c) => c.kind === 'author-year')).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,165 @@
|
||||
// Pure citation extraction over lecture segments. Regex-only; no network.
|
||||
// Detects DOIs, arXiv ids, ISBN-10/13, and "Author (Year)" mentions.
|
||||
|
||||
import type { Segment } from '../types';
|
||||
import type { Citation } from './types';
|
||||
|
||||
/** Detect and normalize citations across all segments. Deduped by value. */
|
||||
export function detectCitations(segments: Segment[]): Citation[] {
|
||||
if (!segments.length) return [];
|
||||
|
||||
const out: Citation[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
const push = (c: Citation) => {
|
||||
const key = `${c.kind}:${c.value.toLowerCase()}`;
|
||||
if (seen.has(key)) return;
|
||||
seen.add(key);
|
||||
out.push(c);
|
||||
};
|
||||
|
||||
for (const seg of segments) {
|
||||
const text = seg.text ?? '';
|
||||
if (!text) continue;
|
||||
|
||||
for (const c of detectDois(text)) push(tag(c, seg));
|
||||
for (const c of detectArxiv(text)) push(tag(c, seg));
|
||||
for (const c of detectIsbn(text)) push(tag(c, seg));
|
||||
for (const c of detectAuthorYear(text)) push(tag(c, seg));
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
function tag(
|
||||
c: Omit<Citation, 'sourceSegmentId' | 'start'>,
|
||||
seg: Segment,
|
||||
): Citation {
|
||||
return { ...c, sourceSegmentId: seg.id, start: seg.start };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DOI
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function detectDois(text: string): Omit<Citation, 'sourceSegmentId' | 'start'>[] {
|
||||
const out: Omit<Citation, 'sourceSegmentId' | 'start'>[] = [];
|
||||
// DOIs start with 10.<registrant>/<suffix>. Allow an optional doi: / URL prefix.
|
||||
const re = /\b(?:doi:\s*|https?:\/\/(?:dx\.)?doi\.org\/)?(10\.\d{4,9}\/[^\s"<>]+)/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(text))) {
|
||||
// Trim common trailing punctuation that isn't part of the DOI.
|
||||
const value = m[1]!.replace(/[.,;)\]]+$/, '');
|
||||
out.push({ kind: 'doi', value, raw: m[0] });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// arXiv
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function detectArxiv(text: string): Omit<Citation, 'sourceSegmentId' | 'start'>[] {
|
||||
const out: Omit<Citation, 'sourceSegmentId' | 'start'>[] = [];
|
||||
|
||||
// Explicit "arXiv:NNNN.NNNNN" (with optional version) — most reliable.
|
||||
const reExplicit = /\barxiv:\s*(\d{4}\.\d{4,5}(?:v\d+)?)\b/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = reExplicit.exec(text))) {
|
||||
out.push({ kind: 'arxiv', value: m[1]!, raw: m[0] });
|
||||
}
|
||||
|
||||
// Bare "NNNN.NNNNN" — only when not already captured as part of an arXiv: hit.
|
||||
const reBare = /(?<![\w.])(\d{4}\.\d{4,5}(?:v\d+)?)(?![\w.])/g;
|
||||
while ((m = reBare.exec(text))) {
|
||||
const value = m[1]!;
|
||||
// Skip if this position is the tail of an explicit arXiv match.
|
||||
const preceding = text.slice(Math.max(0, m.index - 8), m.index).toLowerCase();
|
||||
if (preceding.includes('arxiv')) continue;
|
||||
out.push({ kind: 'arxiv', value, raw: m[0] });
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ISBN
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function detectIsbn(text: string): Omit<Citation, 'sourceSegmentId' | 'start'>[] {
|
||||
const out: Omit<Citation, 'sourceSegmentId' | 'start'>[] = [];
|
||||
// ISBN-13 (978/979 prefix) and ISBN-10, hyphens/spaces allowed internally.
|
||||
const re =
|
||||
/\bISBN(?:-1[03])?:?\s*((?:97[89][-\s]?)?(?:\d[-\s]?){9}[\dXx])\b/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(text))) {
|
||||
const value = m[1]!.replace(/[-\s]/g, '').toUpperCase();
|
||||
if (value.length === 10 || value.length === 13) {
|
||||
out.push({ kind: 'isbn', value, raw: m[0] });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Author-year
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function detectAuthorYear(
|
||||
text: string,
|
||||
): Omit<Citation, 'sourceSegmentId' | 'start'>[] {
|
||||
const out: Omit<Citation, 'sourceSegmentId' | 'start'>[] = [];
|
||||
|
||||
// "Smith (2020)", "Smith and Jones (2019)", "Smith et al., 2021",
|
||||
// "Smith et al. (2021)". Author = Capitalized word(s).
|
||||
const re =
|
||||
/\b([A-Z][a-z]+(?:[-\s][A-Z][a-z]+)*(?:\s+(?:and|&)\s+[A-Z][a-z]+)?(?:\s+et al\.?)?)\s*[,(]?\s*\(?(\b(?:18|19|20)\d{2})\)?/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(text))) {
|
||||
const author = stripLeadingStopword(m[1]!.trim().replace(/\s+/g, ' '));
|
||||
const year = m[2]!;
|
||||
// Require either a parenthesized year or an "et al"/"and" multi-author cue
|
||||
// to avoid matching plain prose like "World War 2 1945".
|
||||
const cue = m[0];
|
||||
const looksLikeCitation =
|
||||
/\(\s*(?:18|19|20)\d{2}\s*\)/.test(cue) ||
|
||||
/et al/i.test(cue) ||
|
||||
/\band\b|&/.test(cue);
|
||||
if (!looksLikeCitation) continue;
|
||||
const value = `${author} (${year})`;
|
||||
out.push({ kind: 'author-year', value, raw: m[0].trim() });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Common capitalized sentence-starters that can be mistaken for a surname when
|
||||
// they immediately precede the real author (e.g. "As Vaswani (2017)").
|
||||
const LEADING_STOPWORDS = new Set([
|
||||
'As',
|
||||
'In',
|
||||
'By',
|
||||
'See',
|
||||
'Per',
|
||||
'From',
|
||||
'After',
|
||||
'Before',
|
||||
'And',
|
||||
'The',
|
||||
'This',
|
||||
'That',
|
||||
'These',
|
||||
'Those',
|
||||
'Both',
|
||||
'When',
|
||||
'While',
|
||||
'Since',
|
||||
]);
|
||||
|
||||
function stripLeadingStopword(author: string): string {
|
||||
const tokens = author.split(' ');
|
||||
// Only strip when something real remains and the next token is itself a name.
|
||||
if (tokens.length > 1 && tokens[0] && LEADING_STOPWORDS.has(tokens[0])) {
|
||||
return tokens.slice(1).join(' ');
|
||||
}
|
||||
return author;
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { detectEvents } from './dates';
|
||||
import type { Segment } from '../types';
|
||||
|
||||
const seg = (start: number, text: string, id?: string): Segment => ({
|
||||
id,
|
||||
start,
|
||||
end: start + 1,
|
||||
text,
|
||||
});
|
||||
|
||||
// Anchor = 2026-06-13 (Saturday) 00:00 UTC. All assertions derive from this.
|
||||
const ANCHOR = Date.UTC(2026, 5, 13, 0, 0, 0, 0);
|
||||
|
||||
const utc = (y: number, mo: number, d: number, h = 0, mi = 0) =>
|
||||
Date.UTC(y, mo, d, h, mi, 0, 0);
|
||||
|
||||
describe('detectEvents', () => {
|
||||
it('returns [] for empty input', () => {
|
||||
expect(detectEvents([], ANCHOR)).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns [] for non-finite anchor', () => {
|
||||
expect(detectEvents([seg(0, 'exam tomorrow')], Number.NaN)).toEqual([]);
|
||||
});
|
||||
|
||||
it('resolves ISO YYYY-MM-DD to UTC midnight', () => {
|
||||
const out = detectEvents([seg(0, 'The exam is on 2026-06-20.')], ANCHOR);
|
||||
expect(out).toHaveLength(1);
|
||||
expect(out[0]!.dateMs).toBe(utc(2026, 5, 20));
|
||||
expect(out[0]!.allDay).toBe(true);
|
||||
expect(out[0]!.confidence).toBeGreaterThanOrEqual(0.9);
|
||||
});
|
||||
|
||||
it('resolves "March 3" using the anchor year (no roll within 6mo)', () => {
|
||||
const out = detectEvents([seg(0, 'deadline on March 3')], ANCHOR);
|
||||
expect(out).toHaveLength(1);
|
||||
// March 3 2026 is ~3mo before June 13 2026 -> stays in the anchor year.
|
||||
expect(out[0]!.dateMs).toBe(utc(2026, 2, 3));
|
||||
});
|
||||
|
||||
it('rolls a date more than 6 months before the anchor to next year', () => {
|
||||
// Anchor late in the year: "January 5" resolves ~10mo back -> roll to next.
|
||||
const novAnchor = Date.UTC(2026, 10, 1); // 2026-11-01
|
||||
const out = detectEvents([seg(0, 'deadline on January 5')], novAnchor);
|
||||
expect(out[0]!.dateMs).toBe(utc(2027, 0, 5));
|
||||
});
|
||||
|
||||
it('keeps an explicit year even if far past', () => {
|
||||
const out = detectEvents([seg(0, 'paper from March 3, 2020')], ANCHOR);
|
||||
expect(out[0]!.dateMs).toBe(utc(2020, 2, 3));
|
||||
});
|
||||
|
||||
it('resolves day-first "3 March"', () => {
|
||||
const out = detectEvents([seg(0, 'submission due 20 July')], ANCHOR);
|
||||
expect(out[0]!.dateMs).toBe(utc(2026, 6, 20));
|
||||
});
|
||||
|
||||
it('does NOT roll a date within the next 6 months', () => {
|
||||
const out = detectEvents([seg(0, 'exam on June 20')], ANCHOR);
|
||||
expect(out[0]!.dateMs).toBe(utc(2026, 5, 20));
|
||||
});
|
||||
|
||||
it('resolves "on the 14th" within the anchor month', () => {
|
||||
const out = detectEvents([seg(0, 'assignment due on the 14th')], ANCHOR);
|
||||
expect(out[0]!.dateMs).toBe(utc(2026, 5, 14));
|
||||
});
|
||||
|
||||
it('resolves relative "tomorrow" from the anchor', () => {
|
||||
const out = detectEvents([seg(0, 'quiz tomorrow')], ANCHOR);
|
||||
expect(out[0]!.dateMs).toBe(utc(2026, 5, 14));
|
||||
expect(out[0]!.confidence).toBeLessThan(0.9); // relative
|
||||
});
|
||||
|
||||
it('resolves "today" to the anchor day', () => {
|
||||
const out = detectEvents([seg(0, 'homework due today')], ANCHOR);
|
||||
expect(out[0]!.dateMs).toBe(utc(2026, 5, 13));
|
||||
});
|
||||
|
||||
it('resolves "next week" as +7 days', () => {
|
||||
const out = detectEvents([seg(0, 'the exam is next week')], ANCHOR);
|
||||
expect(out[0]!.dateMs).toBe(utc(2026, 5, 20));
|
||||
});
|
||||
|
||||
it('resolves "in 3 days" and "in 2 weeks"', () => {
|
||||
const a = detectEvents([seg(0, 'due in 3 days')], ANCHOR);
|
||||
expect(a[0]!.dateMs).toBe(utc(2026, 5, 16));
|
||||
const b = detectEvents([seg(0, 'due in 2 weeks')], ANCHOR);
|
||||
expect(b[0]!.dateMs).toBe(utc(2026, 5, 27));
|
||||
});
|
||||
|
||||
it('resolves "next Monday" to the Monday of the following week', () => {
|
||||
// Anchor Sat 2026-06-13. Next Monday = 2026-06-22 (skip the 15th).
|
||||
const out = detectEvents([seg(0, 'exam next Monday')], ANCHOR);
|
||||
expect(out[0]!.dateMs).toBe(utc(2026, 5, 22));
|
||||
});
|
||||
|
||||
it('captures a clock time and sets allDay=false', () => {
|
||||
const out = detectEvents([seg(0, 'exam on June 20 at 3pm')], ANCHOR);
|
||||
expect(out[0]!.allDay).toBe(false);
|
||||
expect(out[0]!.dateMs).toBe(utc(2026, 5, 20, 15, 0));
|
||||
});
|
||||
|
||||
it('parses 24h time "14:00"', () => {
|
||||
const out = detectEvents([seg(0, 'meeting 2026-06-20 14:00')], ANCHOR);
|
||||
expect(out[0]!.allDay).toBe(false);
|
||||
expect(out[0]!.dateMs).toBe(utc(2026, 5, 20, 14, 0));
|
||||
});
|
||||
|
||||
it('uses the surrounding clause as the title', () => {
|
||||
const out = detectEvents(
|
||||
[seg(0, 'Welcome everyone. The final exam is on June 20. See you.')],
|
||||
ANCHOR,
|
||||
);
|
||||
expect(out[0]!.title.toLowerCase()).toContain('final exam');
|
||||
});
|
||||
|
||||
it('boosts confidence when a deadline keyword is present', () => {
|
||||
const withKw = detectEvents([seg(0, 'the exam is on June 20')], ANCHOR)[0]!;
|
||||
const without = detectEvents([seg(0, 'we met on June 20')], ANCHOR)[0]!;
|
||||
expect(withKw.confidence).toBeGreaterThan(without.confidence);
|
||||
});
|
||||
|
||||
it('tags sourceSegmentId and start', () => {
|
||||
const out = detectEvents([seg(42, 'exam on June 20', 'segX')], ANCHOR);
|
||||
expect(out[0]!.sourceSegmentId).toBe('segX');
|
||||
expect(out[0]!.start).toBe(42);
|
||||
});
|
||||
|
||||
it('dedupes identical (dateMs + title)', () => {
|
||||
const out = detectEvents(
|
||||
[seg(0, 'exam on June 20'), seg(1, 'exam on June 20')],
|
||||
ANCHOR,
|
||||
);
|
||||
expect(out).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not read the wall clock (output depends only on the anchor)', () => {
|
||||
const a = detectEvents([seg(0, 'quiz tomorrow')], ANCHOR);
|
||||
const other = Date.UTC(2030, 0, 1);
|
||||
const b = detectEvents([seg(0, 'quiz tomorrow')], other);
|
||||
expect(a[0]!.dateMs).toBe(utc(2026, 5, 14));
|
||||
expect(b[0]!.dateMs).toBe(Date.UTC(2030, 0, 2));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,427 @@
|
||||
// Pure deadline/date detection over lecture segments. NO wall-clock reads:
|
||||
// every timestamp is derived from `anchorMs` (the lecture date) plus explicit
|
||||
// year/month/day/hour components found in the text. Constructing Date objects
|
||||
// from those explicit components is fine; calling Date.now() / new Date() with
|
||||
// no args is NOT.
|
||||
|
||||
import type { Segment } from '../types';
|
||||
import type { CandidateEvent } from './types';
|
||||
|
||||
const MONTHS: Record<string, number> = {
|
||||
january: 0,
|
||||
february: 1,
|
||||
march: 2,
|
||||
april: 3,
|
||||
may: 4,
|
||||
june: 5,
|
||||
july: 6,
|
||||
august: 7,
|
||||
september: 8,
|
||||
october: 9,
|
||||
november: 10,
|
||||
december: 11,
|
||||
jan: 0,
|
||||
feb: 1,
|
||||
mar: 2,
|
||||
apr: 3,
|
||||
jun: 5,
|
||||
jul: 6,
|
||||
aug: 7,
|
||||
sep: 8,
|
||||
sept: 8,
|
||||
oct: 9,
|
||||
nov: 10,
|
||||
dec: 11,
|
||||
};
|
||||
|
||||
const WEEKDAYS: Record<string, number> = {
|
||||
sunday: 0,
|
||||
monday: 1,
|
||||
tuesday: 2,
|
||||
wednesday: 3,
|
||||
thursday: 4,
|
||||
friday: 5,
|
||||
saturday: 6,
|
||||
};
|
||||
|
||||
const DEADLINE_WORDS = [
|
||||
'exam',
|
||||
'deadline',
|
||||
'due',
|
||||
'assignment',
|
||||
'midterm',
|
||||
'final',
|
||||
'quiz',
|
||||
'submit',
|
||||
'submission',
|
||||
'project',
|
||||
'homework',
|
||||
'test',
|
||||
];
|
||||
|
||||
const SIX_MONTHS_MS = 1000 * 60 * 60 * 24 * 30 * 6;
|
||||
|
||||
interface TimeOfDay {
|
||||
hour: number;
|
||||
minute: number;
|
||||
}
|
||||
|
||||
/** A partial match produced by a detector before titling/dating. */
|
||||
interface RawMatch {
|
||||
/** Absolute resolved instant in ms. */
|
||||
dateMs: number;
|
||||
/** Whether a clock time was found (=> not all-day). */
|
||||
hasTime: boolean;
|
||||
/** Matched substring (for `raw`). */
|
||||
matched: string;
|
||||
/** Index of the match within the segment text (for clause extraction). */
|
||||
index: number;
|
||||
/** Base confidence before deadline-keyword boosting. */
|
||||
baseConfidence: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect candidate dates/deadlines across all segments.
|
||||
*
|
||||
* @param segments lecture segments (text + timing).
|
||||
* @param anchorMs the lecture date in ms; relative dates anchor here and
|
||||
* absolute dates borrow this year when none is written.
|
||||
*/
|
||||
export function detectEvents(segments: Segment[], anchorMs: number): CandidateEvent[] {
|
||||
if (!segments.length || !Number.isFinite(anchorMs)) return [];
|
||||
|
||||
const out: CandidateEvent[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const seg of segments) {
|
||||
const text = seg.text ?? '';
|
||||
if (!text) continue;
|
||||
|
||||
const matches = [
|
||||
...detectIso(text, anchorMs),
|
||||
...detectMonthName(text, anchorMs),
|
||||
...detectOrdinalOnly(text, anchorMs),
|
||||
...detectRelative(text, anchorMs),
|
||||
];
|
||||
|
||||
for (const m of matches) {
|
||||
const title = clauseAround(text, m.index, m.matched.length);
|
||||
const hasDeadlineWord = DEADLINE_WORDS.some((w) =>
|
||||
title.toLowerCase().includes(w),
|
||||
);
|
||||
let confidence = m.baseConfidence;
|
||||
if (hasDeadlineWord) confidence = Math.min(1, confidence + 0.1);
|
||||
else confidence = Math.max(0, confidence - 0.2);
|
||||
|
||||
const key = `${m.dateMs}|${title.toLowerCase()}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
out.push({
|
||||
title,
|
||||
dateMs: m.dateMs,
|
||||
allDay: !m.hasTime,
|
||||
sourceSegmentId: seg.id,
|
||||
start: seg.start,
|
||||
raw: m.matched.trim(),
|
||||
confidence: Number(confidence.toFixed(2)),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Absolute detectors
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** ISO `YYYY-MM-DD`, optionally followed by a clock time. */
|
||||
function detectIso(text: string, anchorMs: number): RawMatch[] {
|
||||
const out: RawMatch[] = [];
|
||||
const re = /\b(\d{4})-(\d{2})-(\d{2})\b/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(text))) {
|
||||
const year = Number(m[1]);
|
||||
const month = Number(m[2]) - 1;
|
||||
const day = Number(m[3]);
|
||||
if (month < 0 || month > 11 || day < 1 || day > 31) continue;
|
||||
const time = findTimeNear(text, m.index + m[0].length);
|
||||
const dateMs = buildUtc(year, month, day, time);
|
||||
out.push({
|
||||
dateMs,
|
||||
hasTime: !!time,
|
||||
matched: m[0],
|
||||
index: m.index,
|
||||
baseConfidence: 0.9,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* "may" is also the (very common) modal verb, so a bare "may <number>" — e.g.
|
||||
* "you may 5 minutes", "you may submit 3 drafts" — must NOT be read as the month
|
||||
* May. Only accept it as a month with stronger context: an ordinal on the day
|
||||
* ("May 5th"), an explicit year ("May 5, 2026"), or a date preposition right
|
||||
* before it ("on/by/due/until/before/after May 5").
|
||||
*/
|
||||
function mayLooksLikeMonth(m: RegExpExecArray, text: string, hasYear: boolean): boolean {
|
||||
if (/\d\s*(?:st|nd|rd|th)\b/i.test(m[0])) return true;
|
||||
if (hasYear) return true;
|
||||
const before = text.slice(Math.max(0, m.index - 10), m.index).toLowerCase();
|
||||
return /\b(?:on|by|due|until|before|after|through)\s*$/.test(before);
|
||||
}
|
||||
|
||||
/** `March 3`, `March 3, 2026`, `3 March`, `on March 3rd`. */
|
||||
function detectMonthName(text: string, anchorMs: number): RawMatch[] {
|
||||
const out: RawMatch[] = [];
|
||||
const monthAlt = Object.keys(MONTHS).join('|');
|
||||
|
||||
// Month-first: "March 3", "March 3rd, 2026"
|
||||
const reMD = new RegExp(
|
||||
`\\b(${monthAlt})\\.?\\s+(\\d{1,2})(?:st|nd|rd|th)?(?:,?\\s+(\\d{4}))?`,
|
||||
'gi',
|
||||
);
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = reMD.exec(text))) {
|
||||
const month = MONTHS[m[1]!.toLowerCase()];
|
||||
const day = Number(m[2]);
|
||||
if (month === undefined || day < 1 || day > 31) continue;
|
||||
const explicitYear = m[3] ? Number(m[3]) : undefined;
|
||||
if (m[1]!.toLowerCase() === 'may' && !mayLooksLikeMonth(m, text, explicitYear !== undefined)) {
|
||||
continue;
|
||||
}
|
||||
pushResolved(out, text, m, month, day, explicitYear, anchorMs);
|
||||
}
|
||||
|
||||
// Day-first: "3 March", "3rd March 2026"
|
||||
const reDM = new RegExp(
|
||||
`\\b(\\d{1,2})(?:st|nd|rd|th)?\\s+(${monthAlt})\\.?(?:,?\\s+(\\d{4}))?`,
|
||||
'gi',
|
||||
);
|
||||
while ((m = reDM.exec(text))) {
|
||||
const day = Number(m[1]);
|
||||
const month = MONTHS[m[2]!.toLowerCase()];
|
||||
if (month === undefined || day < 1 || day > 31) continue;
|
||||
const explicitYear = m[3] ? Number(m[3]) : undefined;
|
||||
if (m[2]!.toLowerCase() === 'may' && !mayLooksLikeMonth(m, text, explicitYear !== undefined)) {
|
||||
continue;
|
||||
}
|
||||
pushResolved(out, text, m, month, day, explicitYear, anchorMs);
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
function pushResolved(
|
||||
out: RawMatch[],
|
||||
text: string,
|
||||
m: RegExpExecArray,
|
||||
month: number,
|
||||
day: number,
|
||||
explicitYear: number | undefined,
|
||||
anchorMs: number,
|
||||
): void {
|
||||
const time = findTimeNear(text, m.index + m[0].length);
|
||||
let year = explicitYear ?? utcYear(anchorMs);
|
||||
let dateMs = buildUtc(year, month, day, time);
|
||||
if (explicitYear === undefined && dateMs < anchorMs - SIX_MONTHS_MS) {
|
||||
year += 1;
|
||||
dateMs = buildUtc(year, month, day, time);
|
||||
}
|
||||
out.push({
|
||||
dateMs,
|
||||
hasTime: !!time,
|
||||
matched: m[0],
|
||||
index: m.index,
|
||||
baseConfidence: 0.9,
|
||||
});
|
||||
}
|
||||
|
||||
/** `on the 14th` — day-of-month only, month/year from the anchor. */
|
||||
function detectOrdinalOnly(text: string, anchorMs: number): RawMatch[] {
|
||||
const out: RawMatch[] = [];
|
||||
const re = /\bon the (\d{1,2})(?:st|nd|rd|th)\b/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(text))) {
|
||||
const day = Number(m[1]);
|
||||
if (day < 1 || day > 31) continue;
|
||||
const month = utcMonth(anchorMs);
|
||||
const year = utcYear(anchorMs);
|
||||
const time = findTimeNear(text, m.index + m[0].length);
|
||||
let dateMs = buildUtc(year, month, day, time);
|
||||
// "on the 14th" means the upcoming 14th; if already well past, roll a month.
|
||||
if (dateMs < anchorMs - SIX_MONTHS_MS) {
|
||||
dateMs = buildUtc(year + 1, month, day, time);
|
||||
}
|
||||
out.push({
|
||||
dateMs,
|
||||
hasTime: !!time,
|
||||
matched: m[0],
|
||||
index: m.index,
|
||||
baseConfidence: 0.85,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Relative detectors (anchored purely to anchorMs)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function detectRelative(text: string, anchorMs: number): RawMatch[] {
|
||||
const out: RawMatch[] = [];
|
||||
const dayMs = 1000 * 60 * 60 * 24;
|
||||
|
||||
const add = (offsetDays: number, m: RegExpExecArray) => {
|
||||
const time = findTimeNear(text, m.index + m[0].length);
|
||||
const dateMs = applyTime(anchorMs + offsetDays * dayMs, time);
|
||||
out.push({
|
||||
dateMs,
|
||||
hasTime: !!time,
|
||||
matched: m[0],
|
||||
index: m.index,
|
||||
baseConfidence: 0.6,
|
||||
});
|
||||
};
|
||||
|
||||
let m: RegExpExecArray | null;
|
||||
|
||||
const reToday = /\btoday\b/gi;
|
||||
while ((m = reToday.exec(text))) add(0, m);
|
||||
|
||||
const reTomorrow = /\btomorrow\b/gi;
|
||||
while ((m = reTomorrow.exec(text))) add(1, m);
|
||||
|
||||
const reNextWeek = /\bnext week\b/gi;
|
||||
while ((m = reNextWeek.exec(text))) add(7, m);
|
||||
|
||||
// "in N days" / "in N weeks"
|
||||
const reInN = /\bin (\d{1,3}) (day|days|week|weeks)\b/gi;
|
||||
while ((m = reInN.exec(text))) {
|
||||
const n = Number(m[1]);
|
||||
const unit = m[2]!.toLowerCase();
|
||||
const days = unit.startsWith('week') ? n * 7 : n;
|
||||
add(days, m);
|
||||
}
|
||||
|
||||
// "next Monday".."next Friday" (and weekend days)
|
||||
const reNextDow = /\bnext (sunday|monday|tuesday|wednesday|thursday|friday|saturday)\b/gi;
|
||||
while ((m = reNextDow.exec(text))) {
|
||||
const target = WEEKDAYS[m[1]!.toLowerCase()]!;
|
||||
const anchorDow = utcDow(anchorMs);
|
||||
// Days until the target weekday in the FOLLOWING week (1..7, never 0).
|
||||
let delta = (target - anchorDow + 7) % 7;
|
||||
if (delta === 0) delta = 7;
|
||||
delta += 7; // "next" => skip the current week's occurrence.
|
||||
add(delta, m);
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Time-of-day extraction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Look for a clock time shortly after the date phrase (within ~16 chars). */
|
||||
function findTimeNear(text: string, from: number): TimeOfDay | undefined {
|
||||
const window = text.slice(from, from + 18);
|
||||
return parseTime(window);
|
||||
}
|
||||
|
||||
function parseTime(s: string): TimeOfDay | undefined {
|
||||
// "at 3pm", "at 3:30 pm", "3pm"
|
||||
const re12 = /\b(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm)\b/i;
|
||||
const m12 = re12.exec(s);
|
||||
if (m12) {
|
||||
let hour = Number(m12[1]);
|
||||
const minute = m12[2] ? Number(m12[2]) : 0;
|
||||
const mer = m12[3]!.toLowerCase();
|
||||
if (hour === 12) hour = 0;
|
||||
if (mer === 'pm') hour += 12;
|
||||
if (hour >= 0 && hour <= 23 && minute >= 0 && minute <= 59) {
|
||||
return { hour, minute };
|
||||
}
|
||||
}
|
||||
// 24h "14:00", "at 09:30"
|
||||
const re24 = /\b(?:at\s+)?([01]?\d|2[0-3]):([0-5]\d)\b/;
|
||||
const m24 = re24.exec(s);
|
||||
if (m24) {
|
||||
return { hour: Number(m24[1]), minute: Number(m24[2]) };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pure UTC helpers (constructed only from explicit components / anchorMs)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function buildUtc(
|
||||
year: number,
|
||||
month: number,
|
||||
day: number,
|
||||
time: TimeOfDay | undefined,
|
||||
): number {
|
||||
return Date.UTC(year, month, day, time?.hour ?? 0, time?.minute ?? 0, 0, 0);
|
||||
}
|
||||
|
||||
/** Replace the time-of-day on a relative instant, keeping its UTC calendar day. */
|
||||
function applyTime(ms: number, time: TimeOfDay | undefined): number {
|
||||
if (!time) return ms;
|
||||
const d = new Date(ms);
|
||||
return Date.UTC(
|
||||
d.getUTCFullYear(),
|
||||
d.getUTCMonth(),
|
||||
d.getUTCDate(),
|
||||
time.hour,
|
||||
time.minute,
|
||||
0,
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
function utcYear(ms: number): number {
|
||||
return new Date(ms).getUTCFullYear();
|
||||
}
|
||||
function utcMonth(ms: number): number {
|
||||
return new Date(ms).getUTCMonth();
|
||||
}
|
||||
function utcDow(ms: number): number {
|
||||
return new Date(ms).getUTCDay();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Clause extraction for titles
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Extract a short clause around the matched date as the event title. */
|
||||
function clauseAround(text: string, index: number, length: number): string {
|
||||
// Bound to the surrounding sentence by splitting on sentence punctuation.
|
||||
const before = text.slice(0, index);
|
||||
const after = text.slice(index + length);
|
||||
|
||||
const startBreak = Math.max(
|
||||
before.lastIndexOf('. '),
|
||||
before.lastIndexOf('? '),
|
||||
before.lastIndexOf('! '),
|
||||
before.lastIndexOf('; '),
|
||||
before.lastIndexOf(', '),
|
||||
);
|
||||
const afterBreakRel = firstBreak(after);
|
||||
|
||||
const start = startBreak >= 0 ? startBreak + 2 : 0;
|
||||
const end =
|
||||
index + length + (afterBreakRel >= 0 ? afterBreakRel : after.length);
|
||||
|
||||
return text.slice(start, end).trim().replace(/\s+/g, ' ');
|
||||
}
|
||||
|
||||
function firstBreak(s: string): number {
|
||||
const idxs = ['. ', '? ', '! ', '; ', ', ']
|
||||
.map((p) => s.indexOf(p))
|
||||
.filter((i) => i >= 0);
|
||||
if (!idxs.length) return -1;
|
||||
return Math.min(...idxs);
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { toIcs } from './ics';
|
||||
|
||||
const utc = (y: number, mo: number, d: number, h = 0, mi = 0) =>
|
||||
Date.UTC(y, mo, d, h, mi, 0, 0);
|
||||
|
||||
describe('toIcs', () => {
|
||||
it('produces a valid empty calendar for []', () => {
|
||||
const ics = toIcs([]);
|
||||
expect(ics).toContain('BEGIN:VCALENDAR');
|
||||
expect(ics).toContain('VERSION:2.0');
|
||||
expect(ics).toContain('END:VCALENDAR');
|
||||
expect(ics).not.toContain('BEGIN:VEVENT');
|
||||
});
|
||||
|
||||
it('uses CRLF line endings', () => {
|
||||
const ics = toIcs([]);
|
||||
expect(ics).toContain('\r\n');
|
||||
expect(ics.endsWith('\r\n')).toBe(true);
|
||||
// No bare LF without a preceding CR.
|
||||
expect(/[^\r]\n/.test(ics)).toBe(false);
|
||||
});
|
||||
|
||||
it('emits an all-day VEVENT with VALUE=DATE', () => {
|
||||
const ics = toIcs([
|
||||
{ title: 'Final exam', dateMs: utc(2026, 5, 20), allDay: true },
|
||||
]);
|
||||
expect(ics).toContain('BEGIN:VEVENT');
|
||||
expect(ics).toContain('DTSTART;VALUE=DATE:20260620');
|
||||
expect(ics).toContain('SUMMARY:Final exam');
|
||||
});
|
||||
|
||||
it('emits a timed VEVENT with a UTC DTSTART', () => {
|
||||
const ics = toIcs([
|
||||
{ title: 'Meeting', dateMs: utc(2026, 5, 20, 14, 30), allDay: false },
|
||||
]);
|
||||
expect(ics).toContain('DTSTART:20260620T143000Z');
|
||||
});
|
||||
|
||||
it('derives DTSTAMP from the event dateMs', () => {
|
||||
const ics = toIcs([
|
||||
{ title: 'X', dateMs: utc(2026, 5, 20, 9, 0), allDay: true },
|
||||
]);
|
||||
expect(ics).toContain('DTSTAMP:20260620T090000Z');
|
||||
});
|
||||
|
||||
it('UID is deterministic (no randomness) and stable across calls', () => {
|
||||
const ev = { title: 'Exam', dateMs: utc(2026, 5, 20), allDay: true };
|
||||
const a = toIcs([ev]);
|
||||
const b = toIcs([ev]);
|
||||
expect(a).toBe(b);
|
||||
const uid = a.split('\r\n').find((l) => l.startsWith('UID:'));
|
||||
expect(uid).toBeDefined();
|
||||
expect(uid).toContain('@wisp');
|
||||
});
|
||||
|
||||
it('changes UID when title or date changes', () => {
|
||||
const a = toIcs([{ title: 'A', dateMs: utc(2026, 5, 20), allDay: true }]);
|
||||
const b = toIcs([{ title: 'B', dateMs: utc(2026, 5, 20), allDay: true }]);
|
||||
const uidA = a.split('\r\n').find((l) => l.startsWith('UID:'));
|
||||
const uidB = b.split('\r\n').find((l) => l.startsWith('UID:'));
|
||||
expect(uidA).not.toBe(uidB);
|
||||
});
|
||||
|
||||
it('escapes comma, semicolon, backslash and newline in SUMMARY', () => {
|
||||
const ics = toIcs([
|
||||
{
|
||||
title: 'a, b; c \\ d\ne',
|
||||
dateMs: utc(2026, 5, 20),
|
||||
allDay: true,
|
||||
},
|
||||
]);
|
||||
const line = ics
|
||||
.replace(/\r\n /g, '') // unfold first
|
||||
.split('\r\n')
|
||||
.find((l) => l.startsWith('SUMMARY:'));
|
||||
expect(line).toBe('SUMMARY:a\\, b\\; c \\\\ d\\ne');
|
||||
});
|
||||
|
||||
it('folds lines longer than 75 octets with a leading space', () => {
|
||||
const long = 'X'.repeat(200);
|
||||
const ics = toIcs([{ title: long, dateMs: utc(2026, 5, 20), allDay: true }]);
|
||||
for (const line of ics.split('\r\n')) {
|
||||
// Each physical line must be <=75 octets.
|
||||
expect(Buffer.byteLength(line, 'utf8')).toBeLessThanOrEqual(75);
|
||||
}
|
||||
// Unfolding restores the SUMMARY content.
|
||||
const unfolded = ics.replace(/\r\n /g, '');
|
||||
expect(unfolded).toContain(`SUMMARY:${long}`);
|
||||
});
|
||||
|
||||
it('emits one VEVENT per event', () => {
|
||||
const ics = toIcs([
|
||||
{ title: 'A', dateMs: utc(2026, 5, 20), allDay: true },
|
||||
{ title: 'B', dateMs: utc(2026, 5, 21), allDay: true },
|
||||
]);
|
||||
const count = ics.split('BEGIN:VEVENT').length - 1;
|
||||
expect(count).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,142 @@
|
||||
// Pure RFC 5545 iCalendar serialization. Deterministic (no randomness, no
|
||||
// wall-clock): UID and DTSTAMP are derived from the event itself, so the same
|
||||
// input always yields byte-identical output. CRLF line endings, 75-octet
|
||||
// folding.
|
||||
|
||||
interface IcsEvent {
|
||||
title: string;
|
||||
dateMs: number;
|
||||
allDay: boolean;
|
||||
}
|
||||
|
||||
/** Serialize events to a VCALENDAR string. Valid (and non-empty) even for []. */
|
||||
export function toIcs(events: IcsEvent[]): string {
|
||||
const lines: string[] = [
|
||||
'BEGIN:VCALENDAR',
|
||||
'VERSION:2.0',
|
||||
'PRODID:-//Wisp//Enrich//EN',
|
||||
'CALSCALE:GREGORIAN',
|
||||
];
|
||||
|
||||
for (const ev of events) {
|
||||
const stamp = formatUtc(ev.dateMs);
|
||||
lines.push('BEGIN:VEVENT');
|
||||
lines.push(`UID:${uidFor(ev)}`);
|
||||
lines.push(`DTSTAMP:${stamp}`);
|
||||
if (ev.allDay) {
|
||||
lines.push(`DTSTART;VALUE=DATE:${formatDate(ev.dateMs)}`);
|
||||
} else {
|
||||
lines.push(`DTSTART:${stamp}`);
|
||||
}
|
||||
lines.push(`SUMMARY:${escapeText(ev.title)}`);
|
||||
lines.push('END:VEVENT');
|
||||
}
|
||||
|
||||
lines.push('END:VCALENDAR');
|
||||
|
||||
return lines.map(foldLine).join('\r\n') + '\r\n';
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Deterministic UID derivation (FNV-1a hash of title + dateMs)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function uidFor(ev: IcsEvent): string {
|
||||
const h = fnv1a(`${ev.title}|${ev.dateMs}`);
|
||||
return `${h}@wisp`;
|
||||
}
|
||||
|
||||
function fnv1a(s: string): string {
|
||||
let hash = 0x811c9dc5;
|
||||
for (let i = 0; i < s.length; i++) {
|
||||
hash ^= s.charCodeAt(i);
|
||||
// 32-bit FNV prime multiply via shifts, kept unsigned.
|
||||
hash = Math.imul(hash, 0x01000193) >>> 0;
|
||||
}
|
||||
return hash.toString(16).padStart(8, '0');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Date formatting from dateMs only (UTC)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function formatDate(ms: number): string {
|
||||
const d = new Date(ms);
|
||||
return (
|
||||
pad(d.getUTCFullYear(), 4) + pad(d.getUTCMonth() + 1) + pad(d.getUTCDate())
|
||||
);
|
||||
}
|
||||
|
||||
function formatUtc(ms: number): string {
|
||||
const d = new Date(ms);
|
||||
return (
|
||||
pad(d.getUTCFullYear(), 4) +
|
||||
pad(d.getUTCMonth() + 1) +
|
||||
pad(d.getUTCDate()) +
|
||||
'T' +
|
||||
pad(d.getUTCHours()) +
|
||||
pad(d.getUTCMinutes()) +
|
||||
pad(d.getUTCSeconds()) +
|
||||
'Z'
|
||||
);
|
||||
}
|
||||
|
||||
function pad(n: number, width = 2): string {
|
||||
return String(n).padStart(width, '0');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// RFC 5545 text escaping + line folding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Escape backslash, semicolon, comma, and newlines per RFC 5545 §3.3.11. */
|
||||
function escapeText(s: string): string {
|
||||
return s
|
||||
.replace(/\\/g, '\\\\')
|
||||
.replace(/;/g, '\\;')
|
||||
.replace(/,/g, '\\,')
|
||||
.replace(/\r\n|\n|\r/g, '\\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold a content line to <=75 octets, continuation lines starting with a
|
||||
* single space. UTF-8 aware: never splits a multi-byte character across the
|
||||
* fold boundary, so every emitted line is valid UTF-8.
|
||||
*/
|
||||
function foldLine(line: string): string {
|
||||
if (utf8Len(line) <= 75) return line;
|
||||
|
||||
const chunks: string[] = [];
|
||||
// First line: 75 octets. Continuation lines: 74 octets (1 leading space).
|
||||
let limit = 75;
|
||||
let current = '';
|
||||
let currentBytes = 0;
|
||||
|
||||
for (const ch of line) {
|
||||
const w = utf8Len(ch);
|
||||
if (currentBytes + w > limit) {
|
||||
chunks.push(current);
|
||||
current = '';
|
||||
currentBytes = 0;
|
||||
limit = 74;
|
||||
}
|
||||
current += ch;
|
||||
currentBytes += w;
|
||||
}
|
||||
if (current) chunks.push(current);
|
||||
|
||||
return chunks.map((c, i) => (i === 0 ? c : ' ' + c)).join('\r\n');
|
||||
}
|
||||
|
||||
/** Number of UTF-8 octets a string encodes to. */
|
||||
function utf8Len(s: string): number {
|
||||
let n = 0;
|
||||
for (const ch of s) {
|
||||
const cp = ch.codePointAt(0)!;
|
||||
if (cp < 0x80) n += 1;
|
||||
else if (cp < 0x800) n += 2;
|
||||
else if (cp < 0x10000) n += 3;
|
||||
else n += 4;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Phase 4 enrichment barrel: pure detection/serialization + network lookups.
|
||||
// Pure modules contain NO transcript text in any outbound URL; the network
|
||||
// module sends only the bare identifier/topic. See each file's header.
|
||||
|
||||
export * from './types';
|
||||
export * from './dates';
|
||||
export * from './citations';
|
||||
export * from './ics';
|
||||
export * from './bib';
|
||||
export * from './links';
|
||||
export * from './lookup';
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { lookupLinksFor, bookLinksFor } from './links';
|
||||
import type { Citation, RefMetadata } from './types';
|
||||
|
||||
const cite = (kind: Citation['kind'], value: string): Citation => ({
|
||||
kind,
|
||||
value,
|
||||
raw: value,
|
||||
});
|
||||
|
||||
describe('lookupLinksFor', () => {
|
||||
it('builds a DOI resolver link', () => {
|
||||
const l = lookupLinksFor(cite('doi', '10.1038/nphys1170'));
|
||||
expect(l.doi).toBe('https://doi.org/10.1038%2Fnphys1170');
|
||||
});
|
||||
|
||||
it('builds an arXiv abstract link', () => {
|
||||
const l = lookupLinksFor(cite('arxiv', '1706.03762'));
|
||||
expect(l.arxiv).toBe('https://arxiv.org/abs/1706.03762');
|
||||
});
|
||||
|
||||
it('points openLibrary at the ISBN page for isbn citations', () => {
|
||||
const l = lookupLinksFor(cite('isbn', '9780134685991'));
|
||||
expect(l.openLibrary).toBe('https://openlibrary.org/isbn/9780134685991');
|
||||
});
|
||||
|
||||
it('always includes wikipedia/googleBooks search links', () => {
|
||||
const l = lookupLinksFor(cite('author-year', 'Vaswani (2017)'));
|
||||
expect(l.wikipedia).toContain(
|
||||
'https://en.wikipedia.org/wiki/Special:Search?search=',
|
||||
);
|
||||
expect(l.googleBooks).toContain('https://www.google.com/search?tbm=bks&q=');
|
||||
});
|
||||
|
||||
it('always includes annasArchive and libgen search links', () => {
|
||||
const l = lookupLinksFor(cite('doi', '10.1/x'));
|
||||
expect(l.annasArchive).toContain('https://annas-archive.org/search?q=');
|
||||
expect(l.libgen).toContain('https://libgen.is/search.php?req=');
|
||||
});
|
||||
|
||||
it('prefers resolved metadata title for the search query, URL-encoded', () => {
|
||||
const meta: RefMetadata = {
|
||||
title: 'Attention Is All You Need',
|
||||
source: 'Crossref',
|
||||
};
|
||||
const l = lookupLinksFor(cite('arxiv', '1706.03762'), meta);
|
||||
const q = encodeURIComponent('Attention Is All You Need');
|
||||
expect(l.wikipedia).toContain(q);
|
||||
expect(l.annasArchive).toContain(q);
|
||||
});
|
||||
|
||||
it('falls back to the citation value when no metadata title', () => {
|
||||
const l = lookupLinksFor(cite('author-year', 'Smith et al. (2021)'));
|
||||
const q = encodeURIComponent('Smith et al. (2021)');
|
||||
expect(l.googleBooks).toContain(q);
|
||||
});
|
||||
});
|
||||
|
||||
describe('bookLinksFor', () => {
|
||||
it('builds the full search set from a free-text query (no doi/arxiv)', () => {
|
||||
const l = bookLinksFor('Introduction to Algorithms');
|
||||
const q = encodeURIComponent('Introduction to Algorithms');
|
||||
expect(l.doi).toBeUndefined();
|
||||
expect(l.arxiv).toBeUndefined();
|
||||
expect(l.openLibrary).toBe(`https://openlibrary.org/search?q=${q}`);
|
||||
expect(l.wikipedia).toContain(q);
|
||||
expect(l.googleBooks).toContain(q);
|
||||
expect(l.annasArchive).toContain(q);
|
||||
expect(l.libgen).toContain(q);
|
||||
});
|
||||
|
||||
it('URL-encodes special characters', () => {
|
||||
const l = bookLinksFor('C&A: a/b test');
|
||||
expect(l.libgen).toContain(encodeURIComponent('C&A: a/b test'));
|
||||
expect(l.libgen).not.toContain(' ');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
// Pure outbound-link builders. NO network here — these only construct URLs.
|
||||
// Privacy: the only data placed in a URL is the extracted citation value or a
|
||||
// resolved title — never transcript text. Anna's Archive / LibGen appear as
|
||||
// SEARCH urls only; nothing in this app fetches or proxies them.
|
||||
|
||||
import type { Citation, LookupLinks, RefMetadata } from './types';
|
||||
|
||||
/** Build lookup links for a detected citation, optionally enriched by metadata. */
|
||||
export function lookupLinksFor(c: Citation, meta?: RefMetadata): LookupLinks {
|
||||
const query = (meta?.title || c.value).trim();
|
||||
|
||||
const links: LookupLinks = {
|
||||
...searchLinks(query),
|
||||
};
|
||||
|
||||
if (c.kind === 'doi') {
|
||||
links.doi = `https://doi.org/${encodeURIComponent(c.value)}`;
|
||||
}
|
||||
if (c.kind === 'arxiv') {
|
||||
links.arxiv = `https://arxiv.org/abs/${encodeURIComponent(c.value)}`;
|
||||
}
|
||||
if (c.kind === 'isbn') {
|
||||
// Direct ISBN page beats a free-text Open Library search.
|
||||
links.openLibrary = `https://openlibrary.org/isbn/${encodeURIComponent(
|
||||
c.value,
|
||||
)}`;
|
||||
}
|
||||
|
||||
return links;
|
||||
}
|
||||
|
||||
/** Build the same set of links from a free-text query (no DOI/arXiv). */
|
||||
export function bookLinksFor(query: string): LookupLinks {
|
||||
return searchLinks(query.trim());
|
||||
}
|
||||
|
||||
/** The query-driven search links shared by both entry points. */
|
||||
function searchLinks(query: string): LookupLinks {
|
||||
const q = encodeURIComponent(query);
|
||||
return {
|
||||
wikipedia: `https://en.wikipedia.org/wiki/Special:Search?search=${q}`,
|
||||
googleBooks: `https://www.google.com/search?tbm=bks&q=${q}`,
|
||||
openLibrary: `https://openlibrary.org/search?q=${q}`,
|
||||
annasArchive: `https://annas-archive.org/search?q=${q}`,
|
||||
libgen: `https://libgen.is/search.php?req=${q}`,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
// NETWORK metadata client for Phase 4 enrichment.
|
||||
//
|
||||
// PRIVACY: every request below sends ONLY the extracted citation id (DOI /
|
||||
// arXiv id / ISBN) or the topic string — never any transcript text. Detection
|
||||
// already happened locally; this module just resolves a bare identifier to
|
||||
// public metadata.
|
||||
//
|
||||
// All endpoints are CORS-friendly, legitimate, metadata-only APIs (Crossref,
|
||||
// OpenAlex, Open Library, Wikipedia REST). Their responses include the
|
||||
// `Access-Control-Allow-Origin` header, so they are readable even under our
|
||||
// COEP `require-corp` / COOP `same-origin` isolation (CORS responses satisfy
|
||||
// require-corp without needing CORP headers). We never fetch or proxy any
|
||||
// shadow-library content here — those are search-URL only and live in links.ts.
|
||||
//
|
||||
// Every function is defensive: ~8s AbortController timeout, try/catch that
|
||||
// resolves to `undefined`, and never throws. Results are memoised in an
|
||||
// in-memory Map for the session.
|
||||
|
||||
import type { Citation, RefMetadata } from './types';
|
||||
|
||||
/** Per-session metadata cache, keyed by `${kind}:${value}` / `wiki:${topic}`. */
|
||||
const cache = new Map<string, RefMetadata | undefined>();
|
||||
const wikiCache = new Map<
|
||||
string,
|
||||
{ title: string; extract: string; url: string } | undefined
|
||||
>();
|
||||
|
||||
const TIMEOUT_MS = 8000;
|
||||
|
||||
/** GET JSON with an abort timeout. Returns parsed JSON or undefined on any
|
||||
* failure (network, non-2xx, timeout, parse). Never throws. */
|
||||
async function fetchJson(url: string): Promise<unknown | undefined> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
method: 'GET',
|
||||
headers: { Accept: 'application/json' },
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!res.ok) return undefined;
|
||||
return (await res.json()) as unknown;
|
||||
} catch {
|
||||
return undefined;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
/** Narrow an unknown to a plain record for safe property access. */
|
||||
function asRecord(v: unknown): Record<string, unknown> | undefined {
|
||||
return typeof v === 'object' && v !== null
|
||||
? (v as Record<string, unknown>)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function asString(v: unknown): string | undefined {
|
||||
return typeof v === 'string' && v.length > 0 ? v : undefined;
|
||||
}
|
||||
|
||||
function asNumber(v: unknown): number | undefined {
|
||||
return typeof v === 'number' && Number.isFinite(v) ? v : undefined;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Crossref (DOI)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function parseCrossref(json: unknown, value: string): RefMetadata | undefined {
|
||||
const root = asRecord(json);
|
||||
const message = asRecord(root?.message);
|
||||
if (!message) return undefined;
|
||||
|
||||
const titleArr = Array.isArray(message.title) ? message.title : [];
|
||||
const title = asString(titleArr[0]);
|
||||
|
||||
const authors: string[] = [];
|
||||
if (Array.isArray(message.author)) {
|
||||
for (const a of message.author) {
|
||||
const rec = asRecord(a);
|
||||
if (!rec) continue;
|
||||
const given = asString(rec.given);
|
||||
const family = asString(rec.family);
|
||||
const name = [given, family].filter(Boolean).join(' ').trim();
|
||||
if (name) authors.push(name);
|
||||
}
|
||||
}
|
||||
|
||||
// Year: prefer published, then published-print/online, then issued.
|
||||
const year =
|
||||
crossrefYear(message.published) ??
|
||||
crossrefYear(message['published-print']) ??
|
||||
crossrefYear(message['published-online']) ??
|
||||
crossrefYear(message.issued);
|
||||
|
||||
return {
|
||||
title,
|
||||
authors: authors.length ? authors : undefined,
|
||||
year,
|
||||
source: 'Crossref',
|
||||
url: `https://doi.org/${value}`,
|
||||
};
|
||||
}
|
||||
|
||||
/** Crossref date parts live at `<field>['date-parts'][0][0]`. */
|
||||
function crossrefYear(field: unknown): number | undefined {
|
||||
const rec = asRecord(field);
|
||||
const parts = rec?.['date-parts'];
|
||||
if (!Array.isArray(parts)) return undefined;
|
||||
const first = parts[0];
|
||||
if (!Array.isArray(first)) return undefined;
|
||||
return asNumber(first[0]);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// OpenAlex (arXiv, and DOI fallback)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function parseOpenAlex(
|
||||
json: unknown,
|
||||
source: string,
|
||||
url: string,
|
||||
): RefMetadata | undefined {
|
||||
const work = asRecord(json);
|
||||
if (!work) return undefined;
|
||||
|
||||
const title = asString(work.title) ?? asString(work.display_name);
|
||||
|
||||
const authors: string[] = [];
|
||||
if (Array.isArray(work.authorships)) {
|
||||
for (const a of work.authorships) {
|
||||
const rec = asRecord(a);
|
||||
const author = asRecord(rec?.author);
|
||||
const name = asString(author?.display_name);
|
||||
if (name) authors.push(name);
|
||||
}
|
||||
}
|
||||
|
||||
const year = asNumber(work.publication_year);
|
||||
|
||||
// OpenAlex returns an error object (with no title) for unknown ids.
|
||||
if (!title && !authors.length && year === undefined) return undefined;
|
||||
|
||||
return {
|
||||
title,
|
||||
authors: authors.length ? authors : undefined,
|
||||
year,
|
||||
source,
|
||||
url,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Open Library (ISBN)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function parseOpenLibrary(
|
||||
json: unknown,
|
||||
value: string,
|
||||
): RefMetadata | undefined {
|
||||
const book = asRecord(json);
|
||||
if (!book) return undefined;
|
||||
|
||||
const title = asString(book.title);
|
||||
|
||||
// Open Library's /isbn/{}.json gives `by_statement` (a free-text credit)
|
||||
// rather than structured authors; surface it as a single author when present.
|
||||
const byStatement = asString(book.by_statement);
|
||||
const authors = byStatement ? [byStatement] : undefined;
|
||||
|
||||
// publish_date is free-text (e.g. "March 2004"); pull a 4-digit year out.
|
||||
let year: number | undefined;
|
||||
const publishDate = asString(book.publish_date);
|
||||
if (publishDate) {
|
||||
const m = publishDate.match(/\b(\d{4})\b/);
|
||||
if (m) year = asNumber(Number(m[1]));
|
||||
}
|
||||
|
||||
if (!title && !authors && year === undefined) return undefined;
|
||||
|
||||
return {
|
||||
title,
|
||||
authors,
|
||||
year,
|
||||
source: 'Open Library',
|
||||
url: `https://openlibrary.org/isbn/${value}`,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Resolve a detected citation to public metadata.
|
||||
*
|
||||
* Sends ONLY the citation's identifier (DOI / arXiv id / ISBN) to a
|
||||
* metadata-only API — never any transcript text. Returns `undefined` on any
|
||||
* failure or for `author-year` citations (the UI offers search links instead).
|
||||
* Never throws.
|
||||
*/
|
||||
export async function resolveCitation(
|
||||
c: Citation,
|
||||
): Promise<RefMetadata | undefined> {
|
||||
const key = `${c.kind}:${c.value}`;
|
||||
if (cache.has(key)) return cache.get(key);
|
||||
|
||||
let result: RefMetadata | undefined;
|
||||
|
||||
switch (c.kind) {
|
||||
case 'doi': {
|
||||
// Crossref is the authoritative DOI registry and is CORS-enabled.
|
||||
const json = await fetchJson(
|
||||
`https://api.crossref.org/works/${encodeURIComponent(c.value)}`,
|
||||
);
|
||||
result = parseCrossref(json, c.value);
|
||||
// Fallback to OpenAlex if Crossref had nothing (e.g. DataCite DOI).
|
||||
if (!result) {
|
||||
const alt = await fetchJson(
|
||||
`https://api.openalex.org/works/doi:${encodeURIComponent(c.value)}`,
|
||||
);
|
||||
result = parseOpenAlex(alt, 'OpenAlex', `https://doi.org/${c.value}`);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'arxiv': {
|
||||
// OpenAlex resolves arXiv ids over CORS, avoiding arXiv's
|
||||
// non-CORS Atom export API.
|
||||
const json = await fetchJson(
|
||||
`https://api.openalex.org/works/arxiv:${encodeURIComponent(c.value)}`,
|
||||
);
|
||||
result = parseOpenAlex(
|
||||
json,
|
||||
'OpenAlex',
|
||||
`https://arxiv.org/abs/${c.value}`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'isbn': {
|
||||
const json = await fetchJson(
|
||||
`https://openlibrary.org/isbn/${encodeURIComponent(c.value)}.json`,
|
||||
);
|
||||
result = parseOpenLibrary(json, c.value);
|
||||
break;
|
||||
}
|
||||
case 'author-year':
|
||||
// No reliable id to resolve; UI falls back to search links.
|
||||
result = undefined;
|
||||
break;
|
||||
}
|
||||
|
||||
cache.set(key, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a one-paragraph Wikipedia summary for a topic.
|
||||
*
|
||||
* Sends ONLY the topic string to Wikipedia's REST summary endpoint (CORS-
|
||||
* enabled) — never any transcript text. Returns `undefined` on 404 / any
|
||||
* failure. Never throws.
|
||||
*/
|
||||
export async function wikipediaSummary(
|
||||
topic: string,
|
||||
): Promise<{ title: string; extract: string; url: string } | undefined> {
|
||||
const key = `wiki:${topic}`;
|
||||
if (wikiCache.has(key)) return wikiCache.get(key);
|
||||
|
||||
let result: { title: string; extract: string; url: string } | undefined;
|
||||
|
||||
const json = await fetchJson(
|
||||
`https://en.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(
|
||||
topic,
|
||||
)}`,
|
||||
);
|
||||
const root = asRecord(json);
|
||||
if (root) {
|
||||
const title = asString(root.title);
|
||||
const extract = asString(root.extract);
|
||||
const contentUrls = asRecord(root.content_urls);
|
||||
const desktop = asRecord(contentUrls?.desktop);
|
||||
const url = asString(desktop?.page);
|
||||
if (title && extract && url) {
|
||||
result = { title, extract, url };
|
||||
}
|
||||
}
|
||||
|
||||
wikiCache.set(key, result);
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
// Shared types for Phase 4 enrichment: dates -> calendar, references -> lookups.
|
||||
// All detection is pure over Segment text; network lookups send only the
|
||||
// extracted topic/DOI/ISBN string — never the transcript itself.
|
||||
|
||||
/** A detected date/deadline the user can confirm into their calendar. */
|
||||
export interface CandidateEvent {
|
||||
/** Short event title (the surrounding clause, trimmed). */
|
||||
title: string;
|
||||
/** Resolved absolute time, ms since epoch. */
|
||||
dateMs: number;
|
||||
/** All-day if no specific time was detected. */
|
||||
allDay: boolean;
|
||||
/** Source segment for jump-to-lecture. */
|
||||
sourceSegmentId?: string;
|
||||
/** Audio start time of the source segment. */
|
||||
start?: number;
|
||||
/** The matched text. */
|
||||
raw: string;
|
||||
/** 0..1 detection confidence (relative dates are lower). */
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
/** A detected citation/reference in the lecture. */
|
||||
export interface Citation {
|
||||
kind: 'doi' | 'arxiv' | 'isbn' | 'author-year';
|
||||
/** Normalized id (DOI, arXiv id, ISBN-13/10) or the raw author-year string. */
|
||||
value: string;
|
||||
sourceSegmentId?: string;
|
||||
start?: number;
|
||||
raw: string;
|
||||
}
|
||||
|
||||
/** Resolved metadata for a citation, from a legitimate metadata API. */
|
||||
export interface RefMetadata {
|
||||
title?: string;
|
||||
authors?: string[];
|
||||
year?: number;
|
||||
/** Which API answered, e.g. 'Crossref' | 'OpenAlex' | 'Open Library'. */
|
||||
source: string;
|
||||
/** Canonical URL (DOI resolver, work page, etc.). */
|
||||
url?: string;
|
||||
}
|
||||
|
||||
/** Outbound links for finding a reference/book. Legit sources first; the
|
||||
* shadow-library links (Anna's Archive / LibGen) are search URLs only. */
|
||||
export interface LookupLinks {
|
||||
doi?: string;
|
||||
arxiv?: string;
|
||||
openAlex?: string;
|
||||
openLibrary?: string;
|
||||
googleBooks?: string;
|
||||
wikipedia?: string;
|
||||
annasArchive?: string;
|
||||
libgen?: string;
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
// Extract a human-readable message from an unknown thrown value.
|
||||
//
|
||||
// `String(err)` on a non-Error object yields the useless "[object Object]",
|
||||
// which is exactly what surfaced when native modules (whisper.rn, the audio
|
||||
// decoder, file system) reject with a plain `{ message, code }`-shaped object
|
||||
// instead of an `Error`. This pulls the most useful text out of whatever was
|
||||
// thrown so the UI shows something actionable.
|
||||
export function errorMessage(err: unknown): string {
|
||||
if (err instanceof Error) return err.message;
|
||||
if (typeof err === 'string') return err;
|
||||
if (err && typeof err === 'object') {
|
||||
const o = err as { message?: unknown; reason?: unknown; code?: unknown };
|
||||
const parts = [o.message, o.reason, o.code]
|
||||
.filter((x) => x != null && x !== '')
|
||||
.map(String);
|
||||
if (parts.length > 0) return parts.join(' — ');
|
||||
try {
|
||||
return JSON.stringify(err);
|
||||
} catch {
|
||||
/* fall through to String() for circular/unserializable objects */
|
||||
}
|
||||
}
|
||||
return String(err);
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
// Bring-your-own-key cloud generation engine (OpenAI-compatible chat
|
||||
// completions). Used when the user has supplied a CloudConfig (baseUrl, apiKey,
|
||||
// model). Works against OpenAI itself or any OpenAI-compatible endpoint.
|
||||
//
|
||||
// PRIVACY: this engine sends ONLY the system prompt + the user's question +
|
||||
// the retrieved lecture snippets (the caller assembles these via prompt.ts).
|
||||
// Raw audio and full transcripts are NEVER sent — that is a hard Phase 5 rule.
|
||||
//
|
||||
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||
|
||||
import type { CloudConfig, GenerationEngine, GenOptions } from './engine';
|
||||
|
||||
interface ChatMessage {
|
||||
role: 'system' | 'user';
|
||||
content: string;
|
||||
}
|
||||
|
||||
interface ChatCompletionResponse {
|
||||
choices?: { message?: { content?: string } }[];
|
||||
}
|
||||
|
||||
export function createCloudEngine(cfg: CloudConfig): GenerationEngine {
|
||||
// Normalize: drop a trailing slash so we can append the path cleanly.
|
||||
const base = cfg.baseUrl.replace(/\/+$/, '');
|
||||
|
||||
return {
|
||||
kind: 'cloud',
|
||||
label: 'Cloud (your key)',
|
||||
|
||||
async isAvailable(): Promise<boolean> {
|
||||
return !!cfg.apiKey;
|
||||
},
|
||||
|
||||
// Cloud models need no local download; nothing to load.
|
||||
async loadModel(): Promise<void> {
|
||||
/* no-op */
|
||||
},
|
||||
|
||||
isLoaded(): boolean {
|
||||
return true;
|
||||
},
|
||||
|
||||
async generate(prompt: string, opts?: GenOptions): Promise<string> {
|
||||
if (!cfg.apiKey) {
|
||||
throw new Error('Cloud generation requires an API key.');
|
||||
}
|
||||
|
||||
const messages: ChatMessage[] = [];
|
||||
if (opts?.system) messages.push({ role: 'system', content: opts.system });
|
||||
messages.push({ role: 'user', content: prompt });
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(`${base}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${cfg.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: cfg.model,
|
||||
messages,
|
||||
max_tokens: opts?.maxTokens ?? 512,
|
||||
}),
|
||||
signal: opts?.signal,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof DOMException && err.name === 'AbortError') throw err;
|
||||
throw new Error(
|
||||
`Cloud request failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
const detail = await res.text().catch(() => '');
|
||||
throw new Error(
|
||||
`Cloud request failed (${res.status} ${res.statusText})${detail ? `: ${detail}` : ''}`,
|
||||
);
|
||||
}
|
||||
|
||||
let data: ChatCompletionResponse;
|
||||
try {
|
||||
data = (await res.json()) as ChatCompletionResponse;
|
||||
} catch {
|
||||
throw new Error('Cloud response was not valid JSON.');
|
||||
}
|
||||
|
||||
const content = data.choices?.[0]?.message?.content;
|
||||
if (typeof content !== 'string') {
|
||||
throw new Error('Cloud response did not contain a message.');
|
||||
}
|
||||
return content;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
// Optional generative layer (ROADMAP Phase 5). STRICTLY opt-in and gated: a
|
||||
// local WebLLM model (WebGPU) or a bring-your-own-key cloud model. When neither
|
||||
// is available the app falls back to the deterministic Phase 1/3 features — the
|
||||
// LLM is never a hard dependency, and raw audio/transcripts are NEVER sent to a
|
||||
// cloud: only the user's question + the retrieved snippets go out (BYO-key path).
|
||||
|
||||
export interface GenOptions {
|
||||
system?: string;
|
||||
maxTokens?: number;
|
||||
signal?: AbortSignal;
|
||||
/** Streamed token callback (optional). */
|
||||
onToken?: (delta: string) => void;
|
||||
}
|
||||
|
||||
export interface GenerationEngine {
|
||||
/** Which backend this is. */
|
||||
readonly kind: 'webllm' | 'cloud' | 'none';
|
||||
/** Human label, e.g. 'On-device (Qwen2.5-1.5B)' or 'Cloud (your key)'. */
|
||||
readonly label: string;
|
||||
/** Whether this engine can run here right now (WebGPU present / key set). */
|
||||
isAvailable(): Promise<boolean>;
|
||||
/** Load/prepare the model. onProgress in [0,1] (for the WebLLM download). */
|
||||
loadModel(onProgress?: (p: number) => void): Promise<void>;
|
||||
isLoaded(): boolean;
|
||||
/** Generate a completion for `prompt`. */
|
||||
generate(prompt: string, opts?: GenOptions): Promise<string>;
|
||||
}
|
||||
|
||||
/** Cloud (BYO-key) configuration — OpenAI-compatible chat completions. */
|
||||
export interface CloudConfig {
|
||||
/** e.g. https://api.openai.com/v1 (or any OpenAI-compatible base). */
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
/** Default local WebLLM model (small instruct, q4) — pulled from the CDN. */
|
||||
export const WEBLLM_MODEL = 'Qwen2.5-1.5B-Instruct-q4f16_1-MLC';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// RAG ("ask your lectures") result shape — produced by rag.ts.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface RagCitation {
|
||||
transcriptId: string;
|
||||
segmentId?: string;
|
||||
start: number;
|
||||
text: string;
|
||||
/** Retrieval score from Phase 1 search. */
|
||||
score: number;
|
||||
}
|
||||
|
||||
export interface RagAnswer {
|
||||
/** The generated answer (grounded in the citations). */
|
||||
answer: string;
|
||||
/** The lecture moments the answer is based on (always shown verbatim). */
|
||||
citations: RagCitation[];
|
||||
/** False when no relevant lecture content was retrieved (we refuse to answer). */
|
||||
grounded: boolean;
|
||||
/** True when produced by an LLM; false when this is the search-only fallback. */
|
||||
generated: boolean;
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
// NATIVE on-device generation stub. Running a local LLM on device (e.g. via
|
||||
// llama.rn / MLC's native runtime) is a Phase 5 follow-up; until then the native
|
||||
// webllm engine reports unavailable so callers transparently fall back to the
|
||||
// search-only path (or a BYO-key cloud model). It NEVER produces a fake answer.
|
||||
//
|
||||
// This module is selected by Metro for native (.native.ts) and is never imported
|
||||
// by any vitest test.
|
||||
|
||||
import type { GenerationEngine } from './engine';
|
||||
|
||||
const NOT_AVAILABLE = 'On-device generation is not available on native yet';
|
||||
|
||||
export const webllm: GenerationEngine = {
|
||||
kind: 'webllm',
|
||||
label: 'On-device (Qwen2.5-1.5B)',
|
||||
|
||||
async isAvailable(): Promise<boolean> {
|
||||
return false;
|
||||
},
|
||||
|
||||
async loadModel(): Promise<void> {
|
||||
throw new Error(NOT_AVAILABLE);
|
||||
},
|
||||
|
||||
isLoaded(): boolean {
|
||||
return false;
|
||||
},
|
||||
|
||||
async generate(): Promise<string> {
|
||||
throw new Error(NOT_AVAILABLE);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,3 @@
|
||||
// Base resolver for TypeScript. Metro picks engineImpl.web.ts / engineImpl.native.ts
|
||||
// by platform extension at build time; tsc resolves this file (defaults to web).
|
||||
export { webllm } from './engineImpl.web';
|
||||
@@ -0,0 +1,139 @@
|
||||
// WEB-ONLY on-device generation engine, backed by @mlc-ai/web-llm running a
|
||||
// small instruct model (Qwen2.5-1.5B, q4) entirely in the browser via WebGPU.
|
||||
//
|
||||
// WHY WE LOAD IT FROM A CDN AT RUNTIME (not a static import):
|
||||
// web-llm ships WASM and uses dynamic imports that Metro (Expo's web bundler)
|
||||
// cannot statically bundle. As with transcription/engineImpl.web.ts, we never
|
||||
// let Metro see the package: we load the ESM build from a CDN at runtime via a
|
||||
// dynamic import hidden behind `new Function` (so Metro's static analyzer can't
|
||||
// trip over it). The browser resolves it natively. This keeps the JS bundle
|
||||
// small — the model itself is a ~1GB download, cached by the browser after the
|
||||
// first run — and means web-llm is NEVER bundled.
|
||||
//
|
||||
// Requires WebGPU. On a device without a WebGPU adapter, isAvailable() is false
|
||||
// and callers fall back to the search-only path (no fake answers).
|
||||
//
|
||||
// This module is web-only and is NEVER imported by any vitest test.
|
||||
|
||||
import { WEBLLM_MODEL } from './engine';
|
||||
import type { GenerationEngine, GenOptions } from './engine';
|
||||
|
||||
// Pin the web-llm ESM build we load at runtime.
|
||||
const WEBLLM_CDN = 'https://esm.run/@mlc-ai/web-llm';
|
||||
|
||||
// `new Function` hides the dynamic import() specifier from Metro's bundler so it
|
||||
// never tries to resolve/transform web-llm or its WASM.
|
||||
const runtimeImport = new Function('u', 'return import(u)') as (u: string) => Promise<WebLlmModule>;
|
||||
|
||||
// Minimal structural types for the bits of web-llm we use.
|
||||
interface InitProgressReport {
|
||||
progress: number;
|
||||
text?: string;
|
||||
}
|
||||
interface ChatMessage {
|
||||
role: 'system' | 'user' | 'assistant';
|
||||
content: string;
|
||||
}
|
||||
interface CompletionChunk {
|
||||
choices: { delta: { content?: string } }[];
|
||||
}
|
||||
interface CompletionResponse {
|
||||
choices: { message: { content: string } }[];
|
||||
}
|
||||
interface ChatCompletions {
|
||||
create(req: {
|
||||
messages: ChatMessage[];
|
||||
stream: false;
|
||||
max_tokens: number;
|
||||
}): Promise<CompletionResponse>;
|
||||
create(req: {
|
||||
messages: ChatMessage[];
|
||||
stream: true;
|
||||
max_tokens: number;
|
||||
}): Promise<AsyncIterable<CompletionChunk>>;
|
||||
}
|
||||
interface MLCEngine {
|
||||
chat: { completions: ChatCompletions };
|
||||
}
|
||||
interface WebLlmModule {
|
||||
CreateMLCEngine(
|
||||
model: string,
|
||||
opts?: { initProgressCallback?: (p: InitProgressReport) => void },
|
||||
): Promise<MLCEngine>;
|
||||
}
|
||||
|
||||
let libPromise: Promise<WebLlmModule> | null = null;
|
||||
function lib(): Promise<WebLlmModule> {
|
||||
if (!libPromise) libPromise = runtimeImport(WEBLLM_CDN);
|
||||
return libPromise;
|
||||
}
|
||||
|
||||
let engineInstance: MLCEngine | null = null;
|
||||
|
||||
async function hasWebGpu(): Promise<boolean> {
|
||||
try {
|
||||
if (typeof navigator === 'undefined' || !('gpu' in navigator)) return false;
|
||||
const gpu = (navigator as { gpu?: { requestAdapter(): Promise<unknown> } }).gpu;
|
||||
const adapter = await gpu?.requestAdapter();
|
||||
return adapter != null;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export const webllm: GenerationEngine = {
|
||||
kind: 'webllm',
|
||||
label: 'On-device (Qwen2.5-1.5B)',
|
||||
|
||||
async isAvailable(): Promise<boolean> {
|
||||
return hasWebGpu();
|
||||
},
|
||||
|
||||
async loadModel(onProgress?: (p: number) => void): Promise<void> {
|
||||
if (engineInstance) return; // idempotent
|
||||
const { CreateMLCEngine } = await lib();
|
||||
engineInstance = await CreateMLCEngine(WEBLLM_MODEL, {
|
||||
initProgressCallback: (p) => onProgress?.(p.progress),
|
||||
});
|
||||
},
|
||||
|
||||
isLoaded(): boolean {
|
||||
return engineInstance != null;
|
||||
},
|
||||
|
||||
async generate(prompt: string, opts?: GenOptions): Promise<string> {
|
||||
if (!engineInstance) {
|
||||
throw new Error('On-device model is not loaded; call loadModel() first.');
|
||||
}
|
||||
const messages: ChatMessage[] = [];
|
||||
if (opts?.system) messages.push({ role: 'system', content: opts.system });
|
||||
messages.push({ role: 'user', content: prompt });
|
||||
|
||||
const maxTokens = opts?.maxTokens ?? 512;
|
||||
|
||||
if (opts?.onToken) {
|
||||
// Stream: surface each delta and accumulate the full text to return.
|
||||
const stream = await engineInstance.chat.completions.create({
|
||||
messages,
|
||||
stream: true,
|
||||
max_tokens: maxTokens,
|
||||
});
|
||||
let full = '';
|
||||
for await (const chunk of stream) {
|
||||
const delta = chunk.choices[0]?.delta?.content ?? '';
|
||||
if (delta) {
|
||||
full += delta;
|
||||
opts.onToken(delta);
|
||||
}
|
||||
}
|
||||
return full;
|
||||
}
|
||||
|
||||
const res = await engineInstance.chat.completions.create({
|
||||
messages,
|
||||
stream: false,
|
||||
max_tokens: maxTokens,
|
||||
});
|
||||
return res.choices[0]?.message?.content ?? '';
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,51 @@
|
||||
// Public entry point for the optional generative ("ask your lectures") layer.
|
||||
//
|
||||
// `./engineImpl` is resolved by Metro to engineImpl.web.ts or
|
||||
// engineImpl.native.ts by platform extension; the base engineImpl.ts re-export
|
||||
// (web) is what TypeScript resolves for typechecking. Consumers call
|
||||
// getGenerationEngine() and stay platform-agnostic.
|
||||
//
|
||||
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||
|
||||
import type { CloudConfig, GenerationEngine } from './engine';
|
||||
import { createCloudEngine } from './cloud';
|
||||
import { webllm } from './engineImpl';
|
||||
|
||||
/**
|
||||
* An engine that does nothing useful — represents "no generation backend here".
|
||||
* isAvailable() is always false; generate()/loadModel() throw. Callers should
|
||||
* check isAvailable() and fall back to the search-only path (no fake answers).
|
||||
*/
|
||||
export const noneEngine: GenerationEngine = {
|
||||
kind: 'none',
|
||||
label: 'No model',
|
||||
async isAvailable(): Promise<boolean> {
|
||||
return false;
|
||||
},
|
||||
async loadModel(): Promise<void> {
|
||||
throw new Error('No generation engine is available.');
|
||||
},
|
||||
isLoaded(): boolean {
|
||||
return false;
|
||||
},
|
||||
async generate(): Promise<string> {
|
||||
throw new Error('No generation engine is available.');
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Pick the generation engine.
|
||||
*
|
||||
* - If a cloud config WITH an apiKey is given, use the BYO-key cloud engine.
|
||||
* - Otherwise return the platform on-device (webllm) engine. On a device without
|
||||
* WebGPU (web) or on native, that engine's isAvailable() resolves false, which
|
||||
* is how the "none" state is represented in practice — callers MUST check
|
||||
* isAvailable() before loading/generating, and fall back to search-only.
|
||||
*/
|
||||
export function getGenerationEngine(cloud?: CloudConfig): GenerationEngine {
|
||||
if (cloud?.apiKey) return createCloudEngine(cloud);
|
||||
return webllm;
|
||||
}
|
||||
|
||||
export { createCloudEngine, webllm };
|
||||
export * from './engine';
|
||||
@@ -0,0 +1,45 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { buildRagPrompt } from './prompt';
|
||||
|
||||
describe('buildRagPrompt', () => {
|
||||
it('numbers excerpts 1-based with formatted times and includes the question', () => {
|
||||
const { system, prompt } = buildRagPrompt('What is entropy?', [
|
||||
{ text: 'Entropy measures disorder.', start: 65, transcriptId: 't1' },
|
||||
{ text: 'It always increases in a closed system.', start: 130, transcriptId: 't1' },
|
||||
]);
|
||||
|
||||
// Excerpts are numbered and carry a mm:ss timecode from `start`.
|
||||
expect(prompt).toContain('[1] (1:05) Entropy measures disorder.');
|
||||
expect(prompt).toContain('[2] (2:10) It always increases in a closed system.');
|
||||
// Question is present.
|
||||
expect(prompt).toContain('Question: What is entropy?');
|
||||
// System prompt enforces grounding + citations.
|
||||
expect(system).toMatch(/only/i);
|
||||
expect(system).toMatch(/\[1\]|square brackets/i);
|
||||
});
|
||||
|
||||
it('trims the question', () => {
|
||||
const { prompt } = buildRagPrompt(' hello? ', [
|
||||
{ text: 'hi', start: 0, transcriptId: 't1' },
|
||||
]);
|
||||
expect(prompt).toContain('Question: hello?');
|
||||
expect(prompt).not.toContain('Question: hello?');
|
||||
});
|
||||
|
||||
it('is deterministic for identical inputs', () => {
|
||||
const snips = [{ text: 'a', start: 5, transcriptId: 't1' }];
|
||||
const a = buildRagPrompt('q', snips);
|
||||
const b = buildRagPrompt('q', snips);
|
||||
expect(a).toEqual(b);
|
||||
});
|
||||
|
||||
it('yields a refusal-style prompt with no snippets (no excerpts -> admit not found)', () => {
|
||||
const { system, prompt } = buildRagPrompt('Anything?', []);
|
||||
expect(prompt).toContain('(no lecture excerpts were found)');
|
||||
expect(prompt).toContain('Question: Anything?');
|
||||
// Still must not contain a fabricated [1] excerpt.
|
||||
expect(prompt).not.toMatch(/^\[1\]/m);
|
||||
// The system rule that drives the refusal is present.
|
||||
expect(system).toMatch(/could not find it in the lectures/i);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
// PURE, deterministic RAG prompt builder for the optional "ask your lectures"
|
||||
// generative layer (Phase 5). No I/O, no platform deps — just string assembly,
|
||||
// so it is fully unit-testable and produces byte-identical output for identical
|
||||
// inputs.
|
||||
//
|
||||
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||
|
||||
import { formatClock } from '../format';
|
||||
|
||||
/** A retrieved lecture snippet to ground the answer in. */
|
||||
export interface PromptSnippet {
|
||||
text: string;
|
||||
/** Snippet start, in seconds (rendered as mm:ss in the prompt). */
|
||||
start: number;
|
||||
transcriptId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the system + user messages for a grounded, cited RAG answer.
|
||||
*
|
||||
* The system prompt forces the model to answer ONLY from the supplied excerpts,
|
||||
* to admit when the excerpts don't cover the question (no hallucinated answers),
|
||||
* to stay concise, and to cite excerpt numbers like [1], [2].
|
||||
*
|
||||
* The user prompt lists the excerpts as `[i] (mm:ss) text` (1-based), then the
|
||||
* question. With zero snippets the excerpt block is replaced by an explicit
|
||||
* "(no lecture excerpts were found)" marker, which — combined with the grounding
|
||||
* rule — steers the model to a refusal rather than an invented answer.
|
||||
*
|
||||
* Deterministic: same inputs -> same output.
|
||||
*/
|
||||
export function buildRagPrompt(
|
||||
question: string,
|
||||
snippets: PromptSnippet[],
|
||||
): { system: string; prompt: string } {
|
||||
const system = [
|
||||
'You are a study assistant that answers questions about a student\'s lecture recordings.',
|
||||
'Answer ONLY using the numbered lecture excerpts provided below.',
|
||||
'If the excerpts do not contain the answer, say you could not find it in the lectures — do not use outside knowledge and do not guess.',
|
||||
'Be concise.',
|
||||
'Cite the excerpts you used by their number in square brackets, like [1] or [2][3].',
|
||||
].join(' ');
|
||||
|
||||
const q = question.trim();
|
||||
|
||||
const excerptBlock =
|
||||
snippets.length === 0
|
||||
? '(no lecture excerpts were found)'
|
||||
: snippets
|
||||
.map((s, i) => `[${i + 1}] (${formatClock(s.start)}) ${s.text.trim()}`)
|
||||
.join('\n');
|
||||
|
||||
const prompt = `Lecture excerpts:\n${excerptBlock}\n\nQuestion: ${q}`;
|
||||
|
||||
return { system, prompt };
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
// RAG orchestrator for the optional "ask your lectures" feature (Phase 5).
|
||||
//
|
||||
// Flow: retrieve the most relevant lecture moments with the existing on-device
|
||||
// semantic search, then (only if a generation engine is available) ask the model
|
||||
// to write a grounded, cited answer from those moments. The retrieved moments are
|
||||
// ALWAYS returned as citations so the UI can show them verbatim.
|
||||
//
|
||||
// Hard rules honoured here:
|
||||
// - Opt-in + gated: if no engine is available we return the search hits as a
|
||||
// "search-only" fallback (generated:false) — never a fake/ungrounded answer.
|
||||
// - Privacy: we only ever pass the question + retrieved snippets to the engine
|
||||
// (the cloud path is OpenAI-compatible BYO-key); raw audio/transcripts never
|
||||
// leave the device.
|
||||
// - Defensive: any generation failure degrades to the search-only fallback so
|
||||
// the user still gets the moments.
|
||||
//
|
||||
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||
|
||||
import { searchLectures } from '../search/search';
|
||||
import type { GenerationEngine, RagAnswer, RagCitation } from './engine';
|
||||
import { buildRagPrompt } from './prompt';
|
||||
|
||||
/** How many lecture moments to retrieve and feed the model. */
|
||||
const RAG_LIMIT = 6;
|
||||
|
||||
const EMPTY: RagAnswer = {
|
||||
answer: '',
|
||||
citations: [],
|
||||
grounded: false,
|
||||
generated: false,
|
||||
};
|
||||
|
||||
/**
|
||||
* Answer `question` from the user's lectures, grounded in retrieved moments.
|
||||
*
|
||||
* - Empty/whitespace question -> empty, ungrounded result.
|
||||
* - No relevant moments retrieved -> empty, ungrounded result (we refuse to
|
||||
* answer with nothing to cite).
|
||||
* - Engine unavailable / generation fails -> the retrieved moments as a
|
||||
* search-only fallback (grounded:true, generated:false).
|
||||
* - Otherwise -> the model's grounded, cited answer (generated:true).
|
||||
*/
|
||||
export async function askLectures(
|
||||
question: string,
|
||||
engine: GenerationEngine,
|
||||
opts?: { courseId?: string | null; signal?: AbortSignal },
|
||||
): Promise<RagAnswer> {
|
||||
const q = question.trim();
|
||||
if (q.length === 0) return EMPTY;
|
||||
|
||||
const hits = await searchLectures(q, {
|
||||
courseId: opts?.courseId,
|
||||
limit: RAG_LIMIT,
|
||||
});
|
||||
if (hits.length === 0) return EMPTY;
|
||||
|
||||
// The retrieved moments are shown verbatim regardless of whether we generate.
|
||||
const citations: RagCitation[] = hits.map((h) => ({
|
||||
transcriptId: h.transcriptId,
|
||||
segmentId: h.segmentId,
|
||||
start: h.start,
|
||||
text: h.text,
|
||||
score: h.score,
|
||||
}));
|
||||
|
||||
// Search-only fallback: no engine here right now (no WebGPU / no key set).
|
||||
if (!(await engine.isAvailable())) {
|
||||
return { answer: '', citations, grounded: true, generated: false };
|
||||
}
|
||||
|
||||
try {
|
||||
if (!engine.isLoaded()) await engine.loadModel();
|
||||
const { system, prompt } = buildRagPrompt(
|
||||
q,
|
||||
hits.map((h) => ({
|
||||
text: h.text,
|
||||
start: h.start,
|
||||
transcriptId: h.transcriptId,
|
||||
})),
|
||||
);
|
||||
const answer = await engine.generate(prompt, {
|
||||
system,
|
||||
maxTokens: 512,
|
||||
signal: opts?.signal,
|
||||
});
|
||||
return { answer, citations, grounded: true, generated: true };
|
||||
} catch {
|
||||
// Any failure (model load, generation, abort) -> still give the moments.
|
||||
return { answer: '', citations, grounded: true, generated: false };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { cardsFromGlossary } from './flashcards';
|
||||
import type { GlossaryEntry } from './glossary';
|
||||
|
||||
describe('cardsFromGlossary', () => {
|
||||
it('maps one card per entry and carries provenance', () => {
|
||||
const entries: GlossaryEntry[] = [
|
||||
{
|
||||
term: 'Photosynthesis',
|
||||
definition: 'A process used by plants to make food.',
|
||||
start: 7,
|
||||
segmentId: 'segX',
|
||||
},
|
||||
];
|
||||
const cards = cardsFromGlossary(entries);
|
||||
expect(cards.length).toBe(1);
|
||||
expect(cards[0]!.back).toBe('A process used by plants to make food.');
|
||||
expect(cards[0]!.start).toBe(7);
|
||||
expect(cards[0]!.segmentId).toBe('segX');
|
||||
});
|
||||
|
||||
it('uses a "What is X?" front when the term is absent from the definition', () => {
|
||||
const entries: GlossaryEntry[] = [
|
||||
{ term: 'Photosynthesis', definition: 'A process used by plants.', start: 0 },
|
||||
];
|
||||
const [card] = cardsFromGlossary(entries);
|
||||
expect(card!.front).toBe('What is Photosynthesis?');
|
||||
});
|
||||
|
||||
it('makes a cloze front when the term appears in the definition', () => {
|
||||
const entries: GlossaryEntry[] = [
|
||||
{
|
||||
term: 'Recursion',
|
||||
definition: 'Recursion is when a function calls itself.',
|
||||
start: 0,
|
||||
},
|
||||
];
|
||||
const [card] = cardsFromGlossary(entries);
|
||||
expect(card!.front).toBe('_____ is when a function calls itself.');
|
||||
expect(card!.front).not.toContain('Recursion');
|
||||
});
|
||||
|
||||
it('returns [] for no entries', () => {
|
||||
expect(cardsFromGlossary([])).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
// Turn glossary entries into flashcard seeds. Deterministic, no model.
|
||||
// A seed is a UI-friendly front/back pair carrying its source provenance; the
|
||||
// repo later wraps it into a FlashcardDraft + SRS state.
|
||||
//
|
||||
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||
|
||||
import type { GlossaryEntry } from './glossary';
|
||||
|
||||
/** A front/back pair for a flashcard, with optional source anchor. */
|
||||
export interface CardSeed {
|
||||
front: string;
|
||||
back: string;
|
||||
/** Source segment id, for click-to-seek. */
|
||||
segmentId?: string;
|
||||
/** Source start time (seconds). */
|
||||
start?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build one {@link CardSeed} per glossary entry.
|
||||
*
|
||||
* Front: if the term appears verbatim (case-insensitively) inside the
|
||||
* definition, we make a cloze deletion — blanking the term in the definition so
|
||||
* the learner recalls it in context. Otherwise we fall back to the plain
|
||||
* "What is {term}?" prompt. Back is always the full definition.
|
||||
*/
|
||||
export function cardsFromGlossary(entries: GlossaryEntry[]): CardSeed[] {
|
||||
return entries.map((entry) => {
|
||||
const front = makeFront(entry.term, entry.definition);
|
||||
return {
|
||||
front,
|
||||
back: entry.definition,
|
||||
segmentId: entry.segmentId,
|
||||
start: entry.start,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** "_____" cloze if the term occurs in the definition, else a Q prompt. */
|
||||
function makeFront(term: string, definition: string): string {
|
||||
const re = new RegExp(escapeRegExp(term), 'i');
|
||||
if (re.test(definition)) {
|
||||
return definition.replace(re, '_____');
|
||||
}
|
||||
return `What is ${term}?`;
|
||||
}
|
||||
|
||||
/** Escape a string for safe use inside a RegExp. */
|
||||
function escapeRegExp(s: string): string {
|
||||
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { glossary } from './glossary';
|
||||
import type { Segment } from '../types';
|
||||
|
||||
const seg = (start: number, text: string, id?: string): Segment => ({
|
||||
id,
|
||||
start,
|
||||
end: start + 1,
|
||||
text,
|
||||
});
|
||||
|
||||
describe('glossary', () => {
|
||||
it('returns [] for no segments', () => {
|
||||
expect(glossary([])).toEqual([]);
|
||||
});
|
||||
|
||||
it('captures an "X is Y" definition pattern', () => {
|
||||
const segs = [
|
||||
seg(3, 'Backpropagation is an algorithm for training neural networks.', 's1'),
|
||||
];
|
||||
const out = glossary(segs);
|
||||
const entry = out.find((e) => e.term.toLowerCase() === 'backpropagation');
|
||||
expect(entry).toBeDefined();
|
||||
expect(entry!.definition).toBe(
|
||||
'Backpropagation is an algorithm for training neural networks.',
|
||||
);
|
||||
expect(entry!.start).toBe(3);
|
||||
expect(entry!.segmentId).toBe('s1');
|
||||
});
|
||||
|
||||
it('recognizes means / refers to / is defined as', () => {
|
||||
const segs = [
|
||||
seg(0, 'Entropy means the amount of disorder in a system.'),
|
||||
seg(1, 'A token refers to a unit of text.'),
|
||||
seg(2, 'Latency is defined as the delay before a transfer begins.'),
|
||||
];
|
||||
const terms = glossary(segs).map((e) => e.term.toLowerCase());
|
||||
expect(terms).toContain('entropy');
|
||||
expect(terms).toContain('a token');
|
||||
expect(terms).toContain('latency');
|
||||
});
|
||||
|
||||
it('dedupes by lowercased term, preferring the explicit definition', () => {
|
||||
const segs = [
|
||||
seg(0, 'Recursion is fun. Recursion appears again here as Recursion.'),
|
||||
seg(1, 'Recursion is when a function calls itself.', 'def'),
|
||||
];
|
||||
const out = glossary(segs);
|
||||
const recursion = out.filter((e) => e.term.toLowerCase() === 'recursion');
|
||||
expect(recursion.length).toBe(1);
|
||||
expect(recursion[0]!.definition).toBe(
|
||||
'Recursion is when a function calls itself.',
|
||||
);
|
||||
});
|
||||
|
||||
it('respects the max cap', () => {
|
||||
const segs = [
|
||||
seg(0, 'Apple is a fruit.'),
|
||||
seg(1, 'Banana is a fruit.'),
|
||||
seg(2, 'Cherry is a fruit.'),
|
||||
];
|
||||
expect(glossary(segs, { max: 2 }).length).toBeLessThanOrEqual(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,185 @@
|
||||
// Deterministic glossary extraction: surface candidate terms + definitions from
|
||||
// a lecture transcript with NO model. Two complementary strategies:
|
||||
// (a) explicit definition patterns: "X is/are/means/refers to/is defined as Y"
|
||||
// (b) frequent Capitalized phrases / frequent content nouns as a fallback,
|
||||
// using their first containing sentence as a stand-in definition.
|
||||
//
|
||||
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||
|
||||
import type { Segment } from '../types';
|
||||
import { splitSentences, tokenizeWords, STOPWORDS } from './tokenize';
|
||||
|
||||
/** One glossary row: a term, a definition, and where it came from. */
|
||||
export interface GlossaryEntry {
|
||||
term: string;
|
||||
definition: string;
|
||||
/** Start time (seconds) of the source segment. */
|
||||
start: number;
|
||||
/** Source segment id, when the segment had one. */
|
||||
segmentId?: string;
|
||||
}
|
||||
|
||||
/** Internal accumulator while collecting candidates. */
|
||||
interface Candidate extends GlossaryEntry {
|
||||
/** How "important"/frequent this candidate is, for ranking. */
|
||||
frequency: number;
|
||||
/** Whether it came from an explicit definition pattern (preferred). */
|
||||
explicit: boolean;
|
||||
/** Original discovery order, for stable tie-breaking. */
|
||||
order: number;
|
||||
}
|
||||
|
||||
// "X is/are/means/refers to/is defined as Y" — capture the subject (term) and
|
||||
// keep the whole sentence as the definition. The subject is the run of words
|
||||
// before the connective; we cap its length so we don't swallow half a clause.
|
||||
const DEFINITION_RE =
|
||||
/^(.{2,60}?)\s+(?:is|are|means|refers? to|is defined as)\s+(.+)$/i;
|
||||
|
||||
// A run of Capitalized words (proper-noun-ish phrase), e.g. "Hidden Markov Model".
|
||||
const CAPITALIZED_PHRASE_RE = /\b([A-Z][a-z0-9]+(?:\s+[A-Z][a-z0-9]+)*)\b/g;
|
||||
|
||||
/**
|
||||
* Build a glossary of at most `max` entries from `segments`.
|
||||
*
|
||||
* Candidates are deduped by lowercased term (explicit definitions win over
|
||||
* fallbacks; otherwise the first/most-frequent occurrence is kept), then sorted
|
||||
* by frequency descending (explicit entries ranked ahead on ties), and capped.
|
||||
*
|
||||
* Empty input -> [].
|
||||
*/
|
||||
export function glossary(
|
||||
segments: Segment[],
|
||||
opts?: { max?: number },
|
||||
): GlossaryEntry[] {
|
||||
const max = opts?.max ?? 12;
|
||||
if (segments.length === 0 || max <= 0) return [];
|
||||
|
||||
// Document-wide content-word frequencies, used to rank fallback candidates.
|
||||
const wordFreq = new Map<string, number>();
|
||||
for (const seg of segments) {
|
||||
for (const tok of tokenizeWords(seg.text)) {
|
||||
wordFreq.set(tok, (wordFreq.get(tok) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
// Frequency of each Capitalized phrase (by lowercased key) across the doc.
|
||||
const phraseFreq = new Map<string, number>();
|
||||
for (const seg of segments) {
|
||||
for (const m of seg.text.matchAll(CAPITALIZED_PHRASE_RE)) {
|
||||
const phrase = m[1];
|
||||
if (!phrase) continue;
|
||||
const key = phrase.toLowerCase();
|
||||
phraseFreq.set(key, (phraseFreq.get(key) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
// Dedupe by lowercased term. Map preserves first-seen insertion order.
|
||||
const byTerm = new Map<string, Candidate>();
|
||||
let order = 0;
|
||||
|
||||
const add = (cand: Omit<Candidate, 'order'>) => {
|
||||
const key = cand.term.toLowerCase().trim();
|
||||
if (key.length === 0) return;
|
||||
const existing = byTerm.get(key);
|
||||
if (!existing) {
|
||||
byTerm.set(key, { ...cand, order: order++ });
|
||||
return;
|
||||
}
|
||||
// An explicit definition supersedes a fallback for the same term. When both
|
||||
// are explicit, keep the more informative (longer) definition; this lets a
|
||||
// substantive "X is when ..." win over a throwaway "X is fun." mention.
|
||||
const upgrade =
|
||||
(cand.explicit && !existing.explicit) ||
|
||||
(cand.explicit === existing.explicit &&
|
||||
cand.definition.length > existing.definition.length);
|
||||
if (upgrade) {
|
||||
byTerm.set(key, { ...cand, order: existing.order });
|
||||
}
|
||||
};
|
||||
|
||||
// Pass 1: explicit "X is Y" definitions (highest quality).
|
||||
for (const seg of segments) {
|
||||
for (const sentence of splitSentences(seg.text)) {
|
||||
const m = sentence.match(DEFINITION_RE);
|
||||
if (!m || !m[1]) continue;
|
||||
const term = m[1].trim();
|
||||
// Skip junk subjects: a term with no content words (pure stopwords) OR one
|
||||
// that's too short. (Was `&&`, which only skipped when BOTH held, so an
|
||||
// all-stopword term like "there" still became a glossary entry.)
|
||||
const termWords = tokenizeWords(term);
|
||||
if (termWords.length === 0 || term.length < 3) continue;
|
||||
add({
|
||||
term,
|
||||
definition: sentence,
|
||||
start: seg.start,
|
||||
segmentId: seg.id,
|
||||
// Frequency from the phrase/word tables, boosted so explicit wins.
|
||||
frequency:
|
||||
(phraseFreq.get(term.toLowerCase()) ?? 0) +
|
||||
(wordFreq.get(term.toLowerCase()) ?? 0) +
|
||||
1,
|
||||
explicit: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 2a: frequent multi-word Capitalized phrases as fallback candidates.
|
||||
for (const seg of segments) {
|
||||
const sentences = splitSentences(seg.text);
|
||||
for (const [key, freq] of phraseFreq) {
|
||||
if (freq < 1) continue;
|
||||
// Only multi-word phrases here; single Capitalized words are noisy.
|
||||
if (!key.includes(' ')) continue;
|
||||
// First sentence in this segment that contains the phrase.
|
||||
const containing = sentences.find((s) => s.toLowerCase().includes(key));
|
||||
if (!containing) continue;
|
||||
// Reconstruct a Title-Cased display term from the key.
|
||||
const term = key.replace(/\b\w/g, (c) => c.toUpperCase());
|
||||
add({
|
||||
term,
|
||||
definition: containing,
|
||||
start: seg.start,
|
||||
segmentId: seg.id,
|
||||
frequency: freq,
|
||||
explicit: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 2b: frequent single content nouns as a last-resort fallback.
|
||||
for (const seg of segments) {
|
||||
const sentences = splitSentences(seg.text);
|
||||
for (const tok of tokenizeWords(seg.text)) {
|
||||
if (STOPWORDS.has(tok)) continue;
|
||||
const freq = wordFreq.get(tok) ?? 0;
|
||||
if (freq < 2) continue; // require some recurrence to qualify
|
||||
const containing = sentences.find((s) =>
|
||||
s.toLowerCase().includes(tok),
|
||||
);
|
||||
if (!containing) continue;
|
||||
add({
|
||||
term: tok,
|
||||
definition: containing,
|
||||
start: seg.start,
|
||||
segmentId: seg.id,
|
||||
frequency: freq,
|
||||
explicit: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Rank: explicit first, then frequency desc, then discovery order (stable).
|
||||
const ranked = [...byTerm.values()].sort(
|
||||
(a, b) =>
|
||||
Number(b.explicit) - Number(a.explicit) ||
|
||||
b.frequency - a.frequency ||
|
||||
a.order - b.order,
|
||||
);
|
||||
|
||||
return ranked.slice(0, max).map(({ term, definition, start, segmentId }) => ({
|
||||
term,
|
||||
definition,
|
||||
start,
|
||||
segmentId,
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// Public surface of the deterministic study helpers (Phase 3). All pure: no
|
||||
// React/Expo/Node-builtins, so the UI and tests can import freely.
|
||||
//
|
||||
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||
|
||||
export { splitSentences, tokenizeWords, STOPWORDS } from './tokenize';
|
||||
export { summarize } from './summary';
|
||||
export type { SourcedSentence } from './summary';
|
||||
export { glossary } from './glossary';
|
||||
export type { GlossaryEntry } from './glossary';
|
||||
export { cardsFromGlossary } from './flashcards';
|
||||
export type { CardSeed } from './flashcards';
|
||||
export { initialSrs, review } from './srs';
|
||||
export { generateQuiz } from './quiz';
|
||||
export type { QuizQuestion } from './quiz';
|
||||
@@ -0,0 +1,66 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { generateQuiz } from './quiz';
|
||||
import type { GlossaryEntry } from './glossary';
|
||||
|
||||
const entry = (term: string, definition: string, start = 0): GlossaryEntry => ({
|
||||
term,
|
||||
definition,
|
||||
start,
|
||||
segmentId: `seg-${term}`,
|
||||
});
|
||||
|
||||
const FOUR: GlossaryEntry[] = [
|
||||
entry('Alpha', 'the first letter'),
|
||||
entry('Beta', 'the second letter'),
|
||||
entry('Gamma', 'the third letter'),
|
||||
entry('Delta', 'the fourth letter'),
|
||||
];
|
||||
|
||||
describe('generateQuiz', () => {
|
||||
it('returns [] when there are fewer than 4 terms', () => {
|
||||
expect(generateQuiz(FOUR.slice(0, 3))).toEqual([]);
|
||||
expect(generateQuiz([])).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns [] when count <= 0', () => {
|
||||
expect(generateQuiz(FOUR, { count: 0 })).toEqual([]);
|
||||
});
|
||||
|
||||
it('each question has 4 options with the answer at answerIndex', () => {
|
||||
const quiz = generateQuiz(FOUR);
|
||||
expect(quiz.length).toBe(4);
|
||||
for (let i = 0; i < quiz.length; i++) {
|
||||
const q = quiz[i]!;
|
||||
const src = FOUR[i]!;
|
||||
expect(q.options.length).toBe(4);
|
||||
// The option at answerIndex is the correct term for that question.
|
||||
expect(q.options[q.answerIndex]).toBe(src.term);
|
||||
// Question wraps the correct definition.
|
||||
expect(q.question).toBe(`Which term means: "${src.definition}"?`);
|
||||
}
|
||||
});
|
||||
|
||||
it('uses deterministic answer placement at index i % 4', () => {
|
||||
const quiz = generateQuiz(FOUR);
|
||||
expect(quiz.map((q) => q.answerIndex)).toEqual([0, 1, 2, 3]);
|
||||
});
|
||||
|
||||
it('options are distinct and include 3 distractors from other entries', () => {
|
||||
const quiz = generateQuiz(FOUR);
|
||||
for (const q of quiz) {
|
||||
const unique = new Set(q.options.map((o) => o.toLowerCase()));
|
||||
expect(unique.size).toBe(4);
|
||||
}
|
||||
});
|
||||
|
||||
it('carries start and segmentId from the correct entry', () => {
|
||||
const quiz = generateQuiz(FOUR);
|
||||
expect(quiz[0]!.segmentId).toBe('seg-Alpha');
|
||||
expect(quiz[0]!.start).toBe(0);
|
||||
});
|
||||
|
||||
it('respects the count cap', () => {
|
||||
const quiz = generateQuiz(FOUR, { count: 2 });
|
||||
expect(quiz.length).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,88 @@
|
||||
// Deterministic multiple-choice quiz generation from glossary entries. No model,
|
||||
// NO randomness — given the same entries, the same quiz is produced every time
|
||||
// (important for reproducible tests and stable UI across re-renders).
|
||||
//
|
||||
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||
|
||||
import type { GlossaryEntry } from './glossary';
|
||||
|
||||
/** One MCQ: a definition prompt, term options, and the correct option index. */
|
||||
export interface QuizQuestion {
|
||||
question: string;
|
||||
options: string[];
|
||||
answerIndex: number;
|
||||
/** Source start time (seconds). */
|
||||
start?: number;
|
||||
/** Source segment id. */
|
||||
segmentId?: string;
|
||||
}
|
||||
|
||||
const OPTION_COUNT = 4;
|
||||
|
||||
/**
|
||||
* Generate up to `count` questions from `entries`.
|
||||
*
|
||||
* Requires at least {@link OPTION_COUNT} (4) distinct terms so every question
|
||||
* can have one correct answer + 3 distinct distractors; with fewer entries we
|
||||
* return [].
|
||||
*
|
||||
* Determinism: for question `i`, the correct answer is placed at index
|
||||
* `i % OPTION_COUNT`, and the remaining slots are filled with distractor terms
|
||||
* drawn from the OTHER entries by rotating through the list (offset by `i`), so
|
||||
* the layout is a pure function of the input order.
|
||||
*/
|
||||
export function generateQuiz(
|
||||
entries: GlossaryEntry[],
|
||||
opts?: { count?: number },
|
||||
): QuizQuestion[] {
|
||||
const count = opts?.count ?? 8;
|
||||
if (entries.length < OPTION_COUNT || count <= 0) return [];
|
||||
|
||||
const questions: QuizQuestion[] = [];
|
||||
const n = entries.length;
|
||||
const limit = Math.min(count, n);
|
||||
|
||||
for (let i = 0; i < limit; i++) {
|
||||
const correct = entries[i];
|
||||
if (!correct) continue; // unreachable (i < limit <= n), satisfies the checker
|
||||
|
||||
// Collect 3 distinct distractor terms from other entries, rotating the
|
||||
// start point by `i` so different questions get different wrong answers.
|
||||
const distractors: string[] = [];
|
||||
const seen = new Set<string>([correct.term.toLowerCase()]);
|
||||
let j = 1;
|
||||
while (distractors.length < OPTION_COUNT - 1 && j <= n) {
|
||||
const cand = entries[(i + j) % n];
|
||||
j++;
|
||||
if (!cand) continue;
|
||||
const key = cand.term.toLowerCase();
|
||||
if (!seen.has(key)) {
|
||||
seen.add(key);
|
||||
distractors.push(cand.term);
|
||||
}
|
||||
}
|
||||
|
||||
// Defensive: if duplicate terms left us short, skip this question rather
|
||||
// than emit one with fewer than 4 options. (glossary() dedupes, so this is
|
||||
// effectively unreachable for real input.)
|
||||
if (distractors.length < OPTION_COUNT - 1) continue;
|
||||
|
||||
// Place the answer at a deterministic index; fill the rest with distractors.
|
||||
const answerIndex = i % OPTION_COUNT;
|
||||
const options: string[] = [];
|
||||
let d = 0;
|
||||
for (let slot = 0; slot < OPTION_COUNT; slot++) {
|
||||
options.push(slot === answerIndex ? correct.term : (distractors[d++] as string));
|
||||
}
|
||||
|
||||
questions.push({
|
||||
question: `Which term means: "${correct.definition}"?`,
|
||||
options,
|
||||
answerIndex,
|
||||
start: correct.start,
|
||||
segmentId: correct.segmentId,
|
||||
});
|
||||
}
|
||||
|
||||
return questions;
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { initialSrs, review } from './srs';
|
||||
|
||||
const MS_PER_DAY = 86_400_000;
|
||||
const NOW = 1_700_000_000_000;
|
||||
|
||||
describe('initialSrs', () => {
|
||||
it('creates a neutral, immediately-due card', () => {
|
||||
const s = initialSrs(NOW);
|
||||
expect(s).toEqual({
|
||||
ease: 2.5,
|
||||
intervalDays: 0,
|
||||
reps: 0,
|
||||
lapses: 0,
|
||||
due: NOW,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('review', () => {
|
||||
it('does not mutate the input state', () => {
|
||||
const s = initialSrs(NOW);
|
||||
const frozen = { ...s };
|
||||
review(s, 2, NOW);
|
||||
expect(s).toEqual(frozen);
|
||||
});
|
||||
|
||||
it('produces monotonically increasing intervals on repeated "good"', () => {
|
||||
let s = initialSrs(NOW);
|
||||
const intervals: number[] = [];
|
||||
let t = NOW;
|
||||
for (let i = 0; i < 5; i++) {
|
||||
s = review(s, 2, t);
|
||||
intervals.push(s.intervalDays);
|
||||
t += s.intervalDays * MS_PER_DAY;
|
||||
}
|
||||
// 1 day, 6 days, then *ease each time -> strictly increasing.
|
||||
for (let i = 1; i < intervals.length; i++) {
|
||||
expect(intervals[i]!).toBeGreaterThan(intervals[i - 1]!);
|
||||
}
|
||||
expect(intervals[0]).toBe(1);
|
||||
expect(intervals[1]).toBe(6);
|
||||
});
|
||||
|
||||
it('increments reps on passing grades and resets on "again"', () => {
|
||||
let s = initialSrs(NOW);
|
||||
s = review(s, 2, NOW); // good
|
||||
s = review(s, 2, NOW); // good
|
||||
expect(s.reps).toBe(2);
|
||||
s = review(s, 0, NOW); // again
|
||||
expect(s.reps).toBe(0);
|
||||
expect(s.lapses).toBe(1);
|
||||
});
|
||||
|
||||
it('floors ease at 1.3 even after many "again" reviews', () => {
|
||||
let s = initialSrs(NOW);
|
||||
for (let i = 0; i < 20; i++) s = review(s, 0, NOW);
|
||||
expect(s.ease).toBe(1.3);
|
||||
});
|
||||
|
||||
it('lowers ease on hard, raises it on easy', () => {
|
||||
const good = review(initialSrs(NOW), 2, NOW);
|
||||
expect(good.ease).toBe(2.5);
|
||||
const hard = review(initialSrs(NOW), 1, NOW);
|
||||
expect(hard.ease).toBeCloseTo(2.35, 5);
|
||||
const easy = review(initialSrs(NOW), 3, NOW);
|
||||
expect(easy.ease).toBeCloseTo(2.65, 5);
|
||||
});
|
||||
|
||||
it('computes due = now + intervalDays * 86400000 and sets lastReviewed', () => {
|
||||
const s = review(initialSrs(NOW), 2, NOW);
|
||||
expect(s.intervalDays).toBe(1);
|
||||
expect(s.due).toBe(NOW + 1 * MS_PER_DAY);
|
||||
expect(s.lastReviewed).toBe(NOW);
|
||||
});
|
||||
|
||||
it('gives "easy" a longer interval than "good" at the same step', () => {
|
||||
const good = review(initialSrs(NOW), 2, NOW);
|
||||
const easy = review(initialSrs(NOW), 3, NOW);
|
||||
expect(easy.intervalDays).toBeGreaterThan(good.intervalDays);
|
||||
});
|
||||
|
||||
it('keeps "hard" shorter than "good" for a reviewed card (reps >= 2)', () => {
|
||||
// Bring a card to reps=2 with two Goods (interval 6, ease 2.5).
|
||||
let s = initialSrs(NOW);
|
||||
s = review(s, 2, NOW);
|
||||
s = review(s, 2, NOW);
|
||||
expect(s.reps).toBe(2);
|
||||
const hard = review(s, 1, NOW);
|
||||
const good = review(s, 2, NOW);
|
||||
expect(hard.intervalDays).toBeLessThan(good.intervalDays);
|
||||
// And Hard still grows the interval (it's a pass, not a lapse).
|
||||
expect(hard.intervalDays).toBeGreaterThan(s.intervalDays);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,124 @@
|
||||
// Spaced repetition scheduling — a faithful SM-2 variant adapted to a 4-button
|
||||
// grading UI (Again / Hard / Good / Easy). Pure & deterministic.
|
||||
//
|
||||
// The classic SM-2 algorithm grades recall on 0..5 and recomputes an ease
|
||||
// factor (EF) plus an interval. We collapse the UI to four buttons and map them
|
||||
// to behaviors that match learner expectations on Anki-style apps:
|
||||
//
|
||||
// grade 0 = Again : failed recall. Reset the rep streak, count a lapse, drop
|
||||
// ease by 0.2, and re-show soon (a short sub-day interval).
|
||||
// grade 1 = Hard : recalled with difficulty. Grow the interval gently (x1.2)
|
||||
// and drop ease by 0.15.
|
||||
// grade 2 = Good : normal recall. Use the SM-2 schedule:
|
||||
// rep 1 -> 1 day, rep 2 -> 6 days, then interval *= ease.
|
||||
// grade 3 = Easy : effortless recall. Like Good but with an easy bonus
|
||||
// (~x1.3) and ease bumped up by 0.15.
|
||||
//
|
||||
// Invariants:
|
||||
// - ease is floored at 1.3 (SM-2's classic minimum).
|
||||
// - reps increments only on a passing grade (>= 1); Again resets it to 0.
|
||||
// - due = now + intervalDays * MS_PER_DAY.
|
||||
// - lastReviewed is set to `now` on every review.
|
||||
//
|
||||
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||
|
||||
import type { SrsState } from '../db/schema';
|
||||
|
||||
const MS_PER_DAY = 86_400_000;
|
||||
const MIN_EASE = 1.3;
|
||||
/** Sub-day interval (in days) used when a card is failed and must re-show soon. */
|
||||
const AGAIN_INTERVAL_DAYS = 1 / 1440; // ~1 minute, expressed in days
|
||||
const HARD_MULTIPLIER = 1.2;
|
||||
const EASY_BONUS = 1.3;
|
||||
|
||||
/** A brand-new card: due immediately, neutral ease, nothing learned yet. */
|
||||
export function initialSrs(now: number): SrsState {
|
||||
return {
|
||||
ease: 2.5,
|
||||
intervalDays: 0,
|
||||
reps: 0,
|
||||
lapses: 0,
|
||||
due: now,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a review `grade` to `srs` at time `now`, returning the next state.
|
||||
* Does not mutate the input.
|
||||
*/
|
||||
export function review(
|
||||
srs: SrsState,
|
||||
grade: 0 | 1 | 2 | 3,
|
||||
now: number,
|
||||
): SrsState {
|
||||
let { ease, intervalDays, reps, lapses } = srs;
|
||||
|
||||
if (grade === 0) {
|
||||
// Again: the card was forgotten.
|
||||
ease = floorEase(ease - 0.2);
|
||||
reps = 0;
|
||||
lapses += 1;
|
||||
intervalDays = AGAIN_INTERVAL_DAYS;
|
||||
} else if (grade === 1) {
|
||||
// Hard: passed with difficulty. Lower ease and grow the interval only
|
||||
// GENTLY (x1.2) off the PREVIOUS interval — crucially WITHOUT compounding
|
||||
// the ease factor. Multiplying by ease as well would push Hard past Good for
|
||||
// reviewed cards (e.g. 2.35*1.2 = 2.82 > 2.5), the opposite of what a
|
||||
// struggling learner expects.
|
||||
ease = floorEase(ease - 0.15);
|
||||
reps += 1;
|
||||
intervalDays = hardInterval(srs);
|
||||
} else if (grade === 2) {
|
||||
// Good: standard SM-2 progression, ease unchanged.
|
||||
reps += 1;
|
||||
intervalDays = nextInterval(srs, ease, 1);
|
||||
} else {
|
||||
// Easy: like Good with an easy bonus and an ease bump.
|
||||
ease = floorEase(ease + 0.15);
|
||||
reps += 1;
|
||||
intervalDays = nextInterval(srs, ease, EASY_BONUS);
|
||||
}
|
||||
|
||||
return {
|
||||
ease,
|
||||
intervalDays,
|
||||
reps,
|
||||
lapses,
|
||||
due: now + intervalDays * MS_PER_DAY,
|
||||
lastReviewed: now,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* SM-2 interval progression for a passing grade.
|
||||
* - first successful rep (the card had reps 0) -> 1 day
|
||||
* - second successful rep (had reps 1) -> 6 days
|
||||
* - thereafter -> previous interval * ease
|
||||
* The result is scaled by `multiplier` (Hard < 1-ish via lower ease, Easy bonus).
|
||||
*/
|
||||
function nextInterval(prev: SrsState, ease: number, multiplier: number): number {
|
||||
let base: number;
|
||||
if (prev.reps <= 0) {
|
||||
base = 1;
|
||||
} else if (prev.reps === 1) {
|
||||
base = 6;
|
||||
} else {
|
||||
base = prev.intervalDays * ease;
|
||||
}
|
||||
return base * multiplier;
|
||||
}
|
||||
|
||||
/**
|
||||
* Interval for a "Hard" pass: a gentle x1.2 growth off the previous interval,
|
||||
* never compounded by ease, so Hard stays strictly shorter than Good (which
|
||||
* multiplies by ease) at every stage. A brand-new/unseen card gets a 1-day step.
|
||||
*/
|
||||
function hardInterval(prev: SrsState): number {
|
||||
if (prev.reps <= 0 || prev.intervalDays <= 0) return 1;
|
||||
return prev.intervalDays * HARD_MULTIPLIER;
|
||||
}
|
||||
|
||||
/** Clamp an ease factor to SM-2's minimum of 1.3. */
|
||||
function floorEase(ease: number): number {
|
||||
return ease < MIN_EASE ? MIN_EASE : ease;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { summarize } from './summary';
|
||||
import type { Segment } from '../types';
|
||||
|
||||
const seg = (start: number, text: string, id?: string): Segment => ({
|
||||
id,
|
||||
start,
|
||||
end: start + 1,
|
||||
text,
|
||||
});
|
||||
|
||||
describe('summarize', () => {
|
||||
it('returns [] for no segments', () => {
|
||||
expect(summarize([])).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns [] when maxSentences <= 0', () => {
|
||||
expect(summarize([seg(0, 'Neural networks learn.')], { maxSentences: 0 })).toEqual(
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
it('picks the top-N highest-frequency sentences', () => {
|
||||
// "neural" + "networks" dominate; the off-topic sentence should be dropped.
|
||||
const segs = [
|
||||
seg(0, 'Neural networks process data.'),
|
||||
seg(1, 'Neural networks learn from neural networks.'),
|
||||
seg(2, 'The weather today is sunny.'),
|
||||
];
|
||||
const out = summarize(segs, { maxSentences: 2 });
|
||||
expect(out.length).toBe(2);
|
||||
const texts = out.map((s) => s.text);
|
||||
expect(texts).not.toContain('The weather today is sunny.');
|
||||
});
|
||||
|
||||
it('returns sentences in chronological order with the right start + id', () => {
|
||||
const segs = [
|
||||
seg(10, 'Gradient descent optimizes the gradient.', 'segB'),
|
||||
seg(2, 'Gradient descent uses the gradient slope.', 'segA'),
|
||||
];
|
||||
const out = summarize(segs, { maxSentences: 2 });
|
||||
// Both kept; chronological order means start 2 comes before start 10.
|
||||
expect(out.map((s) => s.start)).toEqual([2, 10]);
|
||||
expect(out[0]!.segmentId).toBe('segA');
|
||||
expect(out[1]!.segmentId).toBe('segB');
|
||||
});
|
||||
|
||||
it('flattens multi-sentence segments and tags them with the segment start', () => {
|
||||
const segs = [seg(5, 'Alpha beta gamma. Alpha beta delta.', 'multi')];
|
||||
const out = summarize(segs, { maxSentences: 5 });
|
||||
expect(out.length).toBe(2);
|
||||
for (const s of out) {
|
||||
expect(s.start).toBe(5);
|
||||
expect(s.segmentId).toBe('multi');
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
// Deterministic extractive summary: pick the most "central" sentences of a
|
||||
// lecture by term-frequency scoring, keeping them in chronological order so the
|
||||
// summary still reads like a condensed version of the talk. No model.
|
||||
//
|
||||
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||
|
||||
import type { Segment } from '../types';
|
||||
import { splitSentences, tokenizeWords } from './tokenize';
|
||||
|
||||
/** A summary sentence tagged with where it came from (for click-to-seek). */
|
||||
export interface SourcedSentence {
|
||||
text: string;
|
||||
/** Start time (seconds) of the source segment. */
|
||||
start: number;
|
||||
/** Source segment id, when the segment had one. */
|
||||
segmentId?: string;
|
||||
}
|
||||
|
||||
/** Internal: a sentence carrying its source + original ordering keys. */
|
||||
interface ScoredSentence extends SourcedSentence {
|
||||
/** Original position across the flattened doc, for stable chronological sort. */
|
||||
position: number;
|
||||
/** Term-frequency score, normalized by sentence length. */
|
||||
score: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Summarize `segments` into at most `maxSentences` sentences.
|
||||
*
|
||||
* Algorithm (fully deterministic):
|
||||
* 1. Build a document term-frequency table over `tokenizeWords` of all text.
|
||||
* 2. Flatten every segment into sentences, each tagged with that segment's
|
||||
* start + id, and a global position index.
|
||||
* 3. Score each sentence as the sum of its content words' document frequencies,
|
||||
* normalized by the number of content words (so long sentences don't win by
|
||||
* length alone). Sentences with no content words score 0.
|
||||
* 4. Take the top `maxSentences` by score (ties broken by earlier position),
|
||||
* then RETURN them re-sorted into chronological order (start, then position).
|
||||
*
|
||||
* Empty input -> [].
|
||||
*/
|
||||
export function summarize(
|
||||
segments: Segment[],
|
||||
opts?: { maxSentences?: number },
|
||||
): SourcedSentence[] {
|
||||
const maxSentences = opts?.maxSentences ?? 5;
|
||||
if (segments.length === 0 || maxSentences <= 0) return [];
|
||||
|
||||
// 1. Document term frequencies.
|
||||
const freq = new Map<string, number>();
|
||||
for (const seg of segments) {
|
||||
for (const tok of tokenizeWords(seg.text)) {
|
||||
freq.set(tok, (freq.get(tok) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Flatten into sentences with provenance + position.
|
||||
const sentences: ScoredSentence[] = [];
|
||||
let position = 0;
|
||||
for (const seg of segments) {
|
||||
for (const text of splitSentences(seg.text)) {
|
||||
// 3. Score this sentence.
|
||||
const tokens = tokenizeWords(text);
|
||||
let sum = 0;
|
||||
for (const tok of tokens) sum += freq.get(tok) ?? 0;
|
||||
const score = tokens.length > 0 ? sum / tokens.length : 0;
|
||||
|
||||
sentences.push({
|
||||
text,
|
||||
start: seg.start,
|
||||
segmentId: seg.id,
|
||||
position: position++,
|
||||
score,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (sentences.length === 0) return [];
|
||||
|
||||
// 4a. Pick top-N by score, ties broken by earlier position (deterministic).
|
||||
const ranked = [...sentences].sort(
|
||||
(a, b) => b.score - a.score || a.position - b.position,
|
||||
);
|
||||
const picked = ranked.slice(0, maxSentences);
|
||||
|
||||
// 4b. Restore chronological order: by start, then original position.
|
||||
picked.sort((a, b) => a.start - b.start || a.position - b.position);
|
||||
|
||||
return picked.map(({ text, start, segmentId }) => ({ text, start, segmentId }));
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { splitSentences, tokenizeWords, STOPWORDS } from './tokenize';
|
||||
|
||||
describe('splitSentences', () => {
|
||||
it('splits on . ? and ! and trims', () => {
|
||||
const out = splitSentences('Hello world. How are you? Great!');
|
||||
expect(out).toEqual(['Hello world.', 'How are you?', 'Great!']);
|
||||
});
|
||||
|
||||
it('splits on newlines too', () => {
|
||||
const out = splitSentences('first line\nsecond line\r\nthird line');
|
||||
expect(out).toEqual(['first line', 'second line', 'third line']);
|
||||
});
|
||||
|
||||
it('drops empty fragments and whitespace-only pieces', () => {
|
||||
expect(splitSentences(' ')).toEqual([]);
|
||||
expect(splitSentences('A. . B.')).toEqual(['A.', '.', 'B.']);
|
||||
expect(splitSentences('')).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps the terminating punctuation with its sentence', () => {
|
||||
expect(splitSentences('One. Two.')).toEqual(['One.', 'Two.']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('tokenizeWords', () => {
|
||||
it('lowercases and splits on non-word runs', () => {
|
||||
expect(tokenizeWords('Neural-Networks rock')).toEqual([
|
||||
'neural',
|
||||
'networks',
|
||||
'rock',
|
||||
]);
|
||||
});
|
||||
|
||||
it('drops empties, stopwords, and tokens shorter than 3 chars', () => {
|
||||
// "is", "a", "of" are short/stop; "the" is a stopword.
|
||||
const out = tokenizeWords('The cat is on a mat of fur');
|
||||
expect(out).toEqual(['cat', 'mat', 'fur']);
|
||||
});
|
||||
|
||||
it('uses the shared STOPWORD set', () => {
|
||||
expect(STOPWORDS.has('the')).toBe(true);
|
||||
expect(tokenizeWords('the and for')).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns [] for empty / symbol-only input', () => {
|
||||
expect(tokenizeWords('')).toEqual([]);
|
||||
expect(tokenizeWords('!!! ??? ...')).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,127 @@
|
||||
// Text tokenization for the deterministic study helpers (summary, glossary,
|
||||
// flashcards, quiz). No model, no I/O, no platform deps — pure & testable.
|
||||
//
|
||||
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||
|
||||
/**
|
||||
* A small, language-agnostic-leaning English stopword set. Kept intentionally
|
||||
* compact: enough to stop "the/and/of" from dominating term frequency, without
|
||||
* stripping domain words. Used by both summary scoring and glossary candidate
|
||||
* selection.
|
||||
*/
|
||||
export const STOPWORDS: ReadonlySet<string> = new Set([
|
||||
'the',
|
||||
'and',
|
||||
'for',
|
||||
'are',
|
||||
'but',
|
||||
'not',
|
||||
'you',
|
||||
'all',
|
||||
'any',
|
||||
'can',
|
||||
'had',
|
||||
'her',
|
||||
'was',
|
||||
'one',
|
||||
'our',
|
||||
'out',
|
||||
'has',
|
||||
'his',
|
||||
'how',
|
||||
'its',
|
||||
'may',
|
||||
'new',
|
||||
'now',
|
||||
'old',
|
||||
'see',
|
||||
'two',
|
||||
'way',
|
||||
'who',
|
||||
'did',
|
||||
'get',
|
||||
'him',
|
||||
'use',
|
||||
'this',
|
||||
'that',
|
||||
'with',
|
||||
'from',
|
||||
'they',
|
||||
'will',
|
||||
'what',
|
||||
'when',
|
||||
'were',
|
||||
'been',
|
||||
'have',
|
||||
'into',
|
||||
'than',
|
||||
'them',
|
||||
'then',
|
||||
'some',
|
||||
'such',
|
||||
'also',
|
||||
'just',
|
||||
'like',
|
||||
'over',
|
||||
'only',
|
||||
'most',
|
||||
'much',
|
||||
'very',
|
||||
'each',
|
||||
'because',
|
||||
'about',
|
||||
'which',
|
||||
'their',
|
||||
'there',
|
||||
'these',
|
||||
'those',
|
||||
'would',
|
||||
'could',
|
||||
'should',
|
||||
'where',
|
||||
'while',
|
||||
'being',
|
||||
'between',
|
||||
'here',
|
||||
'does',
|
||||
'doing',
|
||||
'done',
|
||||
'made',
|
||||
'make',
|
||||
'used',
|
||||
'using',
|
||||
'both',
|
||||
'more',
|
||||
'less',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Split text into sentences on sentence-ending punctuation (`.`, `?`, `!`) and
|
||||
* newlines. Each result is trimmed; empties are dropped.
|
||||
*
|
||||
* This is intentionally simple (no abbreviation handling) — lecture transcripts
|
||||
* are conversational and rarely contain "Dr." / "e.g." style edge cases, and a
|
||||
* mis-split sentence only marginally affects summary/glossary scoring.
|
||||
*/
|
||||
export function splitSentences(text: string): string[] {
|
||||
return text
|
||||
// Break after any run of .?! (keep the run with the sentence it ends), and
|
||||
// also break on newlines so transcript line breaks become boundaries.
|
||||
.split(/(?<=[.?!])\s+|[\r\n]+/)
|
||||
.map((s) => s.trim())
|
||||
.filter((s) => s.length > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenize text into "content words": lowercase, split on non-word runs,
|
||||
* dropping empties, stopwords, and very short tokens (length < 3).
|
||||
*
|
||||
* Used for term-frequency scoring; short tokens and stopwords carry little
|
||||
* topical signal and would otherwise dominate the frequency counts.
|
||||
*/
|
||||
export function tokenizeWords(text: string): string[] {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.split(/\W+/)
|
||||
.filter((t) => t.length >= 3 && !STOPWORDS.has(t));
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { lexicalRank } from './lexical';
|
||||
|
||||
describe('lexicalRank', () => {
|
||||
it('returns [] for an empty query', () => {
|
||||
const rows = [{ id: 'a', text: 'hello world' }];
|
||||
expect(lexicalRank(rows, '')).toEqual([]);
|
||||
expect(lexicalRank(rows, ' ')).toEqual([]);
|
||||
expect(lexicalRank(rows, '!!! ???')).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns [] when no row matches', () => {
|
||||
const rows = [
|
||||
{ id: 'a', text: 'the quick brown fox' },
|
||||
{ id: 'b', text: 'lazy dog' },
|
||||
];
|
||||
expect(lexicalRank(rows, 'elephant')).toEqual([]);
|
||||
});
|
||||
|
||||
it('scores by total term-occurrence count', () => {
|
||||
const rows = [
|
||||
{ id: 'a', text: 'cat cat cat' },
|
||||
{ id: 'b', text: 'cat dog' },
|
||||
];
|
||||
const out = lexicalRank(rows, 'cat');
|
||||
expect(out).toEqual([
|
||||
{ id: 'a', score: 3 },
|
||||
{ id: 'b', score: 1 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('is case-insensitive and tokenizes on non-word chars', () => {
|
||||
const rows = [{ id: 'a', text: 'Neural-Networks, neural networks!' }];
|
||||
// "neural" appears twice, "networks" appears twice => 4.
|
||||
expect(lexicalRank(rows, 'Neural networks')).toEqual([{ id: 'a', score: 4 }]);
|
||||
});
|
||||
|
||||
it('sums distinct query terms and drops zero-score rows', () => {
|
||||
const rows = [
|
||||
{ id: 'a', text: 'gradient descent and gradient ascent' }, // gradient x2
|
||||
{ id: 'b', text: 'gradient boosting' }, // gradient x1
|
||||
{ id: 'c', text: 'random forest' }, // 0 -> dropped
|
||||
];
|
||||
const out = lexicalRank(rows, 'gradient descent');
|
||||
// a: gradient(2)+descent(1)=3 ; b: gradient(1)=1 ; c dropped
|
||||
expect(out).toEqual([
|
||||
{ id: 'a', score: 3 },
|
||||
{ id: 'b', score: 1 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('sorts by score descending', () => {
|
||||
const rows = [
|
||||
{ id: 'low', text: 'alpha' },
|
||||
{ id: 'high', text: 'alpha alpha alpha' },
|
||||
{ id: 'mid', text: 'alpha alpha' },
|
||||
];
|
||||
expect(lexicalRank(rows, 'alpha').map((r) => r.id)).toEqual(['high', 'mid', 'low']);
|
||||
});
|
||||
|
||||
it('skips rows with empty text', () => {
|
||||
const rows = [
|
||||
{ id: 'empty', text: '' },
|
||||
{ id: 'a', text: 'match match' },
|
||||
];
|
||||
expect(lexicalRank(rows, 'match')).toEqual([{ id: 'a', score: 2 }]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,44 @@
|
||||
// Pure lexical (keyword) ranking. Complements semantic search by catching exact
|
||||
// term matches the embedding model might under-weight (names, acronyms, codes).
|
||||
// No I/O, no platform deps — fully unit-testable.
|
||||
|
||||
/** Lowercase + split on non-word chars, dropping empties. */
|
||||
function tokenize(s: string): string[] {
|
||||
return s
|
||||
.toLowerCase()
|
||||
.split(/\W+/)
|
||||
.filter((t) => t.length > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Rank `rows` by how often the query's terms occur in each row's text.
|
||||
*
|
||||
* Scoring: sum, over each query term, of the number of times that term appears
|
||||
* as a token in the row's text. Rows scoring zero are dropped. Result is sorted
|
||||
* by score descending (ties keep input order, since Array.sort is stable).
|
||||
*/
|
||||
export function lexicalRank(
|
||||
rows: { id: string; text: string }[],
|
||||
query: string,
|
||||
): { id: string; score: number }[] {
|
||||
const terms = tokenize(query);
|
||||
if (terms.length === 0) return [];
|
||||
|
||||
const out: { id: string; score: number }[] = [];
|
||||
for (const row of rows) {
|
||||
const tokens = tokenize(row.text);
|
||||
if (tokens.length === 0) continue;
|
||||
|
||||
// Count occurrences per token once, then sum the query terms' counts.
|
||||
const counts = new Map<string, number>();
|
||||
for (const tok of tokens) counts.set(tok, (counts.get(tok) ?? 0) + 1);
|
||||
|
||||
let score = 0;
|
||||
for (const term of terms) score += counts.get(term) ?? 0;
|
||||
|
||||
if (score > 0) out.push({ id: row.id, score });
|
||||
}
|
||||
|
||||
out.sort((a, b) => b.score - a.score);
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { rrf } from './rrf';
|
||||
|
||||
describe('rrf', () => {
|
||||
it('returns [] for no lists / empty lists', () => {
|
||||
expect(rrf([])).toEqual([]);
|
||||
expect(rrf([[], []])).toEqual([]);
|
||||
});
|
||||
|
||||
it('ranks a single list by its existing order', () => {
|
||||
const out = rrf([['a', 'b', 'c']]);
|
||||
expect(out.map((r) => r.id)).toEqual(['a', 'b', 'c']);
|
||||
// rank-1 score = 1/(60+1)
|
||||
expect(out[0]!.score).toBeCloseTo(1 / 61, 10);
|
||||
expect(out[1]!.score).toBeCloseTo(1 / 62, 10);
|
||||
});
|
||||
|
||||
it('sums contributions for ids in multiple lists', () => {
|
||||
// 'a' is rank 1 in list1 and rank 1 in list2 => 2/(k+1).
|
||||
const out = rrf([
|
||||
['a', 'b'],
|
||||
['a', 'c'],
|
||||
]);
|
||||
const a = out.find((r) => r.id === 'a')!;
|
||||
expect(a.score).toBeCloseTo(2 / 61, 10);
|
||||
// 'a' should rank above 'b' and 'c' (each only 1/(k+2)).
|
||||
expect(out[0]!.id).toBe('a');
|
||||
});
|
||||
|
||||
it('an item ranked high in both lists beats one ranked high in only one', () => {
|
||||
const semantic = ['x', 'y', 'z'];
|
||||
const lexical = ['y', 'x', 'w'];
|
||||
const out = rrf([semantic, lexical]);
|
||||
// y: 1/(60+2) + 1/(60+1) ; x: 1/(60+1) + 1/(60+2) -> tie, but both top.
|
||||
expect(new Set([out[0]!.id, out[1]!.id])).toEqual(new Set(['x', 'y']));
|
||||
// z and w each appear once, so they rank lower.
|
||||
expect(out.slice(2).map((r) => r.id).sort()).toEqual(['w', 'z']);
|
||||
});
|
||||
|
||||
it('respects a custom k', () => {
|
||||
const out = rrf([['a']], 0);
|
||||
// rank 1 with k=0 => 1/1 = 1.
|
||||
expect(out[0]!.score).toBeCloseTo(1, 10);
|
||||
});
|
||||
|
||||
it('sorts by fused score descending', () => {
|
||||
const out = rrf([
|
||||
['a', 'b', 'c'],
|
||||
['a', 'b', 'c'],
|
||||
]);
|
||||
expect(out.map((r) => r.id)).toEqual(['a', 'b', 'c']);
|
||||
expect(out[0]!.score).toBeGreaterThan(out[1]!.score);
|
||||
expect(out[1]!.score).toBeGreaterThan(out[2]!.score);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
// Pure Reciprocal Rank Fusion (RRF). Combines several ranked id lists into one
|
||||
// ranking by summing 1/(k + rank) for each id across the lists it appears in
|
||||
// (rank is 1-based; k dampens the contribution of low ranks). This is a robust,
|
||||
// score-free way to fuse heterogeneous signals (semantic + lexical) — only the
|
||||
// ORDER of each input list matters, not its raw scores.
|
||||
//
|
||||
// No I/O, no platform deps — fully unit-testable.
|
||||
|
||||
/**
|
||||
* Fuse ordered id lists into a single ranking, highest fused score first.
|
||||
*
|
||||
* @param lists ordered id lists (rank 1 = first element of each list).
|
||||
* @param k RRF damping constant (default 60, the canonical value).
|
||||
*/
|
||||
export function rrf(lists: string[][], k = 60): { id: string; score: number }[] {
|
||||
const scores = new Map<string, number>();
|
||||
|
||||
for (const list of lists) {
|
||||
for (let i = 0; i < list.length; i++) {
|
||||
const id = list[i];
|
||||
if (id === undefined) continue;
|
||||
const rank = i + 1; // 1-based
|
||||
scores.set(id, (scores.get(id) ?? 0) + 1 / (k + rank));
|
||||
}
|
||||
}
|
||||
|
||||
return [...scores.entries()]
|
||||
.map(([id, score]) => ({ id, score }))
|
||||
.sort((a, b) => b.score - a.score);
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
// Cross-lecture search orchestrator (Phase 1). Embeds the query on-device,
|
||||
// pulls semantic candidates from the vector store, re-ranks them lexically, and
|
||||
// fuses the two signals with Reciprocal Rank Fusion. Returns segment-level hits
|
||||
// (with the matching signal tagged) that the UI can jump to. 100% on-device.
|
||||
//
|
||||
// IMPORTANT: relative imports only inside src/lib (vitest has no '@/*' alias).
|
||||
|
||||
import { getRepo } from '../db';
|
||||
import type { VectorHit } from '../db/repo';
|
||||
import { getEmbeddingEngine } from '../embedding';
|
||||
import { lexicalRank } from './lexical';
|
||||
import { rrf } from './rrf';
|
||||
import type { SearchHit, SearchOptions } from './types';
|
||||
|
||||
/** How many semantic candidates to pull before re-ranking/fusing. */
|
||||
const CANDIDATE_LIMIT = 50;
|
||||
/** Default number of hits returned to the caller. */
|
||||
const DEFAULT_LIMIT = 30;
|
||||
|
||||
/**
|
||||
* Search the user's lectures for `query`, returning fused semantic + lexical
|
||||
* hits at segment granularity (best first). Empty/whitespace queries return [].
|
||||
*/
|
||||
export async function searchLectures(
|
||||
query: string,
|
||||
opts?: SearchOptions,
|
||||
): Promise<SearchHit[]> {
|
||||
const q = query.trim();
|
||||
if (q.length === 0) return [];
|
||||
|
||||
// 1) Embed the query on-device (lazy-load the model on first use).
|
||||
const engine = getEmbeddingEngine();
|
||||
if (!engine.isLoaded()) await engine.loadModel();
|
||||
const [vec] = await engine.embed([q], 'query');
|
||||
if (!vec) return [];
|
||||
|
||||
// 2) Semantic candidates from the vector store (cosine, top CANDIDATE_LIMIT).
|
||||
const cand = await getRepo().searchVectors(vec, {
|
||||
courseId: opts?.courseId,
|
||||
limit: CANDIDATE_LIMIT,
|
||||
});
|
||||
if (cand.length === 0) return [];
|
||||
|
||||
// O(1) candidate lookup by segmentId for the fusion map-back step.
|
||||
const bySegment = new Map<string, VectorHit>();
|
||||
for (const c of cand) bySegment.set(c.segmentId, c);
|
||||
|
||||
// 3) Two ranked id lists over the SAME candidate set.
|
||||
// - semantic: candidates are already cosine-desc from the repo.
|
||||
const semanticIds = cand.map((c) => c.segmentId);
|
||||
const semanticSet = new Set(semanticIds);
|
||||
// - lexical: keyword re-rank of just these candidates' texts.
|
||||
const lexicalRanked = lexicalRank(
|
||||
cand.map((c) => ({ id: c.segmentId, text: c.text })),
|
||||
q,
|
||||
);
|
||||
const lexicalIds = lexicalRanked.map((r) => r.id);
|
||||
const lexicalSet = new Set(lexicalIds);
|
||||
|
||||
// 4) Fuse the two orderings.
|
||||
const fused = rrf([semanticIds, lexicalIds]);
|
||||
|
||||
// 5) Map fused ids back to their VectorHit and tag the matching signal.
|
||||
const limit = opts?.limit ?? DEFAULT_LIMIT;
|
||||
const hits: SearchHit[] = [];
|
||||
for (const { id } of fused) {
|
||||
const hit = bySegment.get(id);
|
||||
if (!hit) continue;
|
||||
const inSemantic = semanticSet.has(id);
|
||||
const inLexical = lexicalSet.has(id);
|
||||
const via: SearchHit['via'] = inSemantic && inLexical ? 'both' : inLexical ? 'lexical' : 'semantic';
|
||||
hits.push({ ...hit, via });
|
||||
if (hits.length >= limit) break;
|
||||
}
|
||||
|
||||
return hits;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
// Shared types for cross-lecture search (Phase 1).
|
||||
import type { VectorHit } from '../db/repo';
|
||||
|
||||
/** A search result at segment granularity, with which signal(s) matched. */
|
||||
export interface SearchHit extends VectorHit {
|
||||
via: 'semantic' | 'lexical' | 'both';
|
||||
}
|
||||
|
||||
export interface SearchOptions {
|
||||
/** Scope to a course (null = Unsorted); omit for all. */
|
||||
courseId?: string | null;
|
||||
/** Max hits to return. */
|
||||
limit?: number;
|
||||
}
|
||||
@@ -28,7 +28,10 @@
|
||||
|
||||
import { initWhisper } from 'whisper.rn';
|
||||
import type { WhisperContext } from 'whisper.rn';
|
||||
import { Paths, File } from 'expo-file-system';
|
||||
import { Paths, File, Directory } from 'expo-file-system';
|
||||
// Progress-reporting downloads aren't in the new OO File API yet, so we use the
|
||||
// (still supported) legacy resumable downloader for the model fetch.
|
||||
import { createDownloadResumable } from 'expo-file-system/legacy';
|
||||
import type {
|
||||
ModelId,
|
||||
PcmAudio,
|
||||
@@ -36,33 +39,95 @@ import type {
|
||||
TranscribeOptions,
|
||||
} from '../types';
|
||||
import { MODELS, recommendModel } from '../models/catalog';
|
||||
import { errorMessage } from '../errorMessage';
|
||||
import type { EngineCapabilities, TranscriptionEngine } from './engine';
|
||||
|
||||
/** Loaded whisper.cpp contexts, keyed by model id. */
|
||||
const loaded = new Map<ModelId, WhisperContext>();
|
||||
|
||||
/**
|
||||
* Resolve the on-device path of a model's ggml file.
|
||||
* Where the official whisper.cpp ggml models are hosted. The catalog's `ggml`
|
||||
* field is the exact filename in this repo (e.g. 'ggml-tiny.en.bin').
|
||||
*/
|
||||
const GGML_BASE_URL = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main';
|
||||
|
||||
/**
|
||||
* The on-device file handle for a model's ggml binary.
|
||||
*
|
||||
* Models live under `<documentDirectory>/models/<ggml-filename>`. The document
|
||||
* directory survives app restarts and OS storage pressure, which is what we want
|
||||
* for multi-hundred-MB model files.
|
||||
*
|
||||
* TODO: the model DOWNLOAD UI / fetcher is NOT built yet. This function only
|
||||
* computes where the file *should* be; the .bin must already exist there
|
||||
* (e.g. side-loaded during development) or loadModel() will reject. Wire a
|
||||
* downloader (File.downloadFileAsync into Paths.document/'models') before
|
||||
* shipping.
|
||||
*
|
||||
* whisper.rn's initWhisper expects a plain filesystem path. Expo's File `.uri`
|
||||
* is a `file://` URI, so we strip the scheme.
|
||||
*/
|
||||
function modelPathFor(id: ModelId): string {
|
||||
const file = new File(Paths.document, 'models', MODELS[id].ggml);
|
||||
// e.g. 'file:///data/.../Documents/models/ggml-tiny.en.bin' -> '/data/.../ggml-tiny.en.bin'
|
||||
function modelFile(id: ModelId): File {
|
||||
return new File(Paths.document, 'models', MODELS[id].ggml);
|
||||
}
|
||||
|
||||
/** whisper.rn's initWhisper wants a plain path; Expo's `.uri` is `file://…`. */
|
||||
function plainPath(file: File): string {
|
||||
return file.uri.replace(/^file:\/\//, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a model's ggml file is present on disk, downloading it from Hugging
|
||||
* Face if needed. Reports 0..1 download progress via `onProgress`. Downloads to
|
||||
* a `.part` file and renames on success, so an interrupted download never leaves
|
||||
* a truncated file that a later run would mistake for a complete model.
|
||||
*/
|
||||
async function ensureModelDownloaded(
|
||||
id: ModelId,
|
||||
onProgress?: (p: number) => void,
|
||||
): Promise<File> {
|
||||
const dest = modelFile(id);
|
||||
if (dest.exists) return dest;
|
||||
|
||||
// Make sure <documentDirectory>/models/ exists.
|
||||
const dir = new Directory(Paths.document, 'models');
|
||||
if (!dir.exists) dir.create({ intermediates: true, idempotent: true });
|
||||
|
||||
const url = `${GGML_BASE_URL}/${MODELS[id].ggml}`;
|
||||
const partUri = `${dest.uri}.part`;
|
||||
|
||||
// Clean up any leftover .part from a previously-aborted attempt.
|
||||
const part = new File(partUri);
|
||||
if (part.exists) {
|
||||
try {
|
||||
part.delete();
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
}
|
||||
|
||||
const task = createDownloadResumable(url, partUri, {}, (p) => {
|
||||
if (p.totalBytesExpectedToWrite > 0) {
|
||||
onProgress?.(p.totalBytesWritten / p.totalBytesExpectedToWrite);
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await task.downloadAsync();
|
||||
if (!result || result.status !== 200) {
|
||||
throw new Error(
|
||||
`server returned ${result?.status ?? 'no response'}`,
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
try {
|
||||
if (part.exists) part.delete();
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
throw new Error(
|
||||
`Couldn't download the ${MODELS[id].label} model (~${MODELS[id].approxMB} MB): ` +
|
||||
`${errorMessage(e)}. Check your connection and try again.`,
|
||||
);
|
||||
}
|
||||
|
||||
// Atomically put the completed file in place.
|
||||
part.move(dest);
|
||||
onProgress?.(1);
|
||||
return dest;
|
||||
}
|
||||
|
||||
export const engine: TranscriptionEngine = {
|
||||
platform: 'native',
|
||||
|
||||
@@ -81,13 +146,31 @@ export const engine: TranscriptionEngine = {
|
||||
};
|
||||
},
|
||||
|
||||
async loadModel(modelId: ModelId): Promise<void> {
|
||||
async loadModel(
|
||||
modelId: ModelId,
|
||||
onProgress?: (p: number) => void,
|
||||
): Promise<void> {
|
||||
if (loaded.has(modelId)) return; // idempotent
|
||||
|
||||
const filePath = modelPathFor(modelId);
|
||||
// initWhisper rejects if the file is missing/corrupt. See the TODO in
|
||||
// modelPathFor: until the download UI exists, the .bin must already be there.
|
||||
const ctx = await initWhisper({ filePath });
|
||||
// Fetch the ggml file on first use (with download progress), then load it.
|
||||
const file = await ensureModelDownloaded(modelId, onProgress);
|
||||
|
||||
let ctx: WhisperContext;
|
||||
try {
|
||||
ctx = await initWhisper({ filePath: plainPath(file) });
|
||||
} catch (e) {
|
||||
// The file is present but unusable (corrupt / truncated). Delete it so the
|
||||
// next attempt re-downloads a clean copy rather than failing forever.
|
||||
try {
|
||||
if (file.exists) file.delete();
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
throw new Error(
|
||||
`The ${MODELS[modelId].label} model couldn't be loaded and was cleared — ` +
|
||||
`try again. (${errorMessage(e)})`,
|
||||
);
|
||||
}
|
||||
loaded.set(modelId, ctx);
|
||||
},
|
||||
|
||||
@@ -113,11 +196,15 @@ export const engine: TranscriptionEngine = {
|
||||
// Float32Array whose `.buffer` is exactly these samples.
|
||||
const pcm = audio.samples.slice();
|
||||
|
||||
// English-only models (".en") only do English — only pass language/translate
|
||||
// for multilingual models (mirrors the web engine; keeps whisper.cpp from
|
||||
// being asked to do something a .en model can't).
|
||||
const multilingual = MODELS[opts.modelId].multilingual;
|
||||
// transcribeData returns { stop, promise }; await the promise.
|
||||
const { promise } = ctx.transcribeData(pcm.buffer, {
|
||||
// language undefined => whisper.rn auto-detects ('auto').
|
||||
language: opts.language,
|
||||
translate: opts.translate ?? false,
|
||||
...(multilingual && opts.language ? { language: opts.language } : {}),
|
||||
...(multilingual ? { translate: opts.translate ?? false } : {}),
|
||||
});
|
||||
const result = await promise;
|
||||
|
||||
|
||||
@@ -40,7 +40,9 @@ interface AsrOutput {
|
||||
type AsrPipeline = (audio: Float32Array, opts: Record<string, unknown>) => Promise<AsrOutput>;
|
||||
interface PipelineOptions {
|
||||
device?: string;
|
||||
dtype?: string;
|
||||
// A single dtype for all sub-models, or a per-file map (encoder_model,
|
||||
// decoder_model_merged, …) — Whisper's decoder needs special handling on WASM.
|
||||
dtype?: string | Record<string, string>;
|
||||
progress_callback?: (e: { status?: string; progress?: number }) => void;
|
||||
}
|
||||
interface TransformersModule {
|
||||
@@ -64,8 +66,21 @@ async function lib(): Promise<TransformersModule> {
|
||||
const loaded = new Map<ModelId, AsrPipeline>();
|
||||
let cachedBackend: Backend | undefined;
|
||||
|
||||
/**
|
||||
* Mobile browsers expose WebGPU but their drivers are flaky for sustained ML
|
||||
* inference — fp16 Whisper tends to crash the GPU process after a chunk or two.
|
||||
* On mobile we deliberately use (cross-origin-isolated, multi-threaded) WASM,
|
||||
* which is slower but reliable. Desktop keeps WebGPU.
|
||||
*/
|
||||
function isMobileWeb(): boolean {
|
||||
if (typeof navigator === 'undefined') return false;
|
||||
const ua = navigator.userAgent || '';
|
||||
return /Android|iPhone|iPad|iPod|Mobile|Silk|Kindle/i.test(ua);
|
||||
}
|
||||
|
||||
async function detectWebGpu(): Promise<boolean> {
|
||||
try {
|
||||
if (isMobileWeb()) return false;
|
||||
if (typeof navigator === 'undefined' || !('gpu' in navigator)) return false;
|
||||
const gpu = (navigator as { gpu?: { requestAdapter(): Promise<unknown> } }).gpu;
|
||||
const adapter = await gpu?.requestAdapter();
|
||||
@@ -98,10 +113,17 @@ export const engine: TranscriptionEngine = {
|
||||
const { pipeline } = await lib();
|
||||
const webgpu = (await resolveBackend()) === 'webgpu';
|
||||
const asr = await pipeline('automatic-speech-recognition', MODELS[modelId].webRepo, {
|
||||
// WebGPU + fp16 when available; otherwise 8-bit weights on WASM, which
|
||||
// stays small to download and runs acceptably on a plain CPU.
|
||||
device: webgpu ? 'webgpu' : 'wasm',
|
||||
dtype: webgpu ? 'fp16' : 'q8',
|
||||
// WebGPU: fp16 (fast, small). WASM: an explicit fp32 decoder. The default
|
||||
// quantized decoders (q8/q4) use `MatMulNBits` ops that onnxruntime-web
|
||||
// (transformers.js 4.2.0) FAILS to load on the WASM backend with
|
||||
// "Missing required scale" — verified across Xenova + onnx-community
|
||||
// repos. fp32 avoids those ops and actually runs on any CPU (the WASM
|
||||
// path is what mobile and no-WebGPU devices use). Larger download, but it
|
||||
// works; q8 simply does not load on WASM here.
|
||||
dtype: webgpu
|
||||
? 'fp16'
|
||||
: { encoder_model: 'fp32', decoder_model_merged: 'fp32' },
|
||||
progress_callback: (e) => {
|
||||
if (e.status === 'progress' && e.progress != null) onProgress?.(e.progress / 100);
|
||||
},
|
||||
@@ -117,21 +139,43 @@ export const engine: TranscriptionEngine = {
|
||||
const asr = loaded.get(opts.modelId);
|
||||
if (!asr) throw new Error(`Model "${opts.modelId}" is not loaded; call loadModel() first.`);
|
||||
|
||||
const out = await asr(audio.samples, {
|
||||
// English-only Whisper models (".en") reject `language`/`task` — transformers.js
|
||||
// throws "Cannot specify `task` or `language` for an English-only model" if
|
||||
// either is passed. Only multilingual models accept them (translation also
|
||||
// requires a multilingual model). So we add those keys ONLY when multilingual,
|
||||
// and never pass an explicit `undefined` language (which trips the same check).
|
||||
const genOpts: Record<string, unknown> = {
|
||||
return_timestamps: true,
|
||||
// One window at a time; 30s matches Whisper's frame so it won't re-chunk.
|
||||
chunk_length_s: 30,
|
||||
language: opts.language,
|
||||
task: opts.translate ? 'translate' : 'transcribe',
|
||||
});
|
||||
};
|
||||
if (MODELS[opts.modelId].multilingual) {
|
||||
genOpts.task = opts.translate ? 'translate' : 'transcribe';
|
||||
if (opts.language) genOpts.language = opts.language;
|
||||
}
|
||||
|
||||
const out = await asr(audio.samples, genOpts);
|
||||
|
||||
// Whisper sometimes omits the END timestamp of the last utterance in a
|
||||
// window (transformers.js yields timestamp[1] === null). Don't collapse that
|
||||
// to a zero-length [t, t] segment (it breaks stitch seams + citation spans):
|
||||
// estimate the end from the next chunk's start, else the window's duration.
|
||||
const chunks = out.chunks ?? [];
|
||||
const chunkDurationSec = audio.samples.length / audio.sampleRate;
|
||||
const segments: Segment[] = [];
|
||||
for (const c of out.chunks ?? []) {
|
||||
const [start, end] = c.timestamp;
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const c = chunks[i]!;
|
||||
const [start, rawEnd] = c.timestamp;
|
||||
if (start == null) continue;
|
||||
const text = c.text.trim();
|
||||
if (text.length === 0) continue;
|
||||
segments.push({ start, end: end ?? start, text });
|
||||
let end = rawEnd;
|
||||
if (end == null) {
|
||||
const nextStart = chunks[i + 1]?.timestamp[0];
|
||||
end = nextStart != null && nextStart > start ? nextStart : chunkDurationSec;
|
||||
}
|
||||
if (end < start) end = chunkDurationSec;
|
||||
segments.push({ start, end, text });
|
||||
}
|
||||
return segments;
|
||||
},
|
||||
|
||||
@@ -127,16 +127,21 @@ describe('transcribe pipeline', () => {
|
||||
const transcribing = events.filter((e) => e.stage === 'transcribing');
|
||||
expect(transcribing.length).toBeGreaterThan(0);
|
||||
|
||||
// Monotonically non-decreasing, strictly bounded in (0, 1].
|
||||
for (let i = 0; i < transcribing.length; i++) {
|
||||
const e = transcribing[i]!;
|
||||
expect(e.progress).toBeGreaterThan(0);
|
||||
// The first transcribing event is a kickoff fired BEFORE the first chunk so
|
||||
// the UI leaves the "loading" stage immediately: progress 0, no partial yet.
|
||||
expect(transcribing[0]!.progress).toBe(0);
|
||||
expect(transcribing[0]!.partial).toEqual([]);
|
||||
|
||||
// Every event carries the total chunk count; progress stays within [0, 1].
|
||||
for (const e of transcribing) {
|
||||
expect(e.chunkCount).toBeGreaterThan(0);
|
||||
expect(e.progress).toBeGreaterThanOrEqual(0);
|
||||
expect(e.progress).toBeLessThanOrEqual(1);
|
||||
if (i > 0) {
|
||||
expect(e.progress).toBeGreaterThan(transcribing[i - 1]!.progress);
|
||||
}
|
||||
}
|
||||
// Final transcribing event hits exactly 1.
|
||||
// After the kickoff, per-chunk progress strictly increases and ends at 1.
|
||||
for (let i = 2; i < transcribing.length; i++) {
|
||||
expect(transcribing[i]!.progress).toBeGreaterThan(transcribing[i - 1]!.progress);
|
||||
}
|
||||
expect(transcribing[transcribing.length - 1]!.progress).toBe(1);
|
||||
|
||||
// The growing partial should reach the final segment count on the last tick.
|
||||
|
||||
@@ -26,6 +26,10 @@ export interface TranscribeProgress {
|
||||
* grows after each chunk during 'transcribing'.
|
||||
*/
|
||||
partial: Segment[];
|
||||
/** Index of the chunk currently being / just transcribed (0-based). */
|
||||
chunkIndex?: number;
|
||||
/** Total number of chunks planned for this audio. */
|
||||
chunkCount?: number;
|
||||
}
|
||||
|
||||
export interface TranscribeParams {
|
||||
@@ -80,6 +84,18 @@ export async function transcribe(params: TranscribeParams): Promise<Segment[]> {
|
||||
// 3. Transcribe each window. `perChunk[i]` holds chunk-local segments for
|
||||
// plan[i]; the stitcher offsets and merges them.
|
||||
const perChunk: Segment[][] = [];
|
||||
|
||||
// Flip to the 'transcribing' stage immediately, BEFORE the first (often slow)
|
||||
// chunk runs — otherwise the UI sits at "Loading model… 100%" for the whole
|
||||
// first inference and looks frozen/crashed. progress 0/N, no partial yet.
|
||||
onProgress?.({
|
||||
stage: 'transcribing',
|
||||
progress: 0,
|
||||
partial: [],
|
||||
chunkIndex: 0,
|
||||
chunkCount: plan.length,
|
||||
});
|
||||
|
||||
for (let i = 0; i < plan.length; i++) {
|
||||
// Cooperative cancellation: check before each (potentially long) chunk.
|
||||
if (signal?.aborted) throw new DOMException('Aborted', 'AbortError');
|
||||
@@ -92,6 +108,13 @@ export async function transcribe(params: TranscribeParams): Promise<Segment[]> {
|
||||
};
|
||||
|
||||
const segs = await engine.transcribeChunk(chunkAudio, options);
|
||||
|
||||
// Re-check AFTER the (long) inference: the top-of-loop check can't catch a
|
||||
// cancel that happens DURING transcribeChunk — and for single-chunk audio
|
||||
// the loop body runs once, so without this a late Cancel would still fall
|
||||
// through and save the transcript.
|
||||
if (signal?.aborted) throw new DOMException('Aborted', 'AbortError');
|
||||
|
||||
perChunk.push(segs);
|
||||
|
||||
// Emit a growing absolute-time partial: stitch everything decoded so far.
|
||||
@@ -99,6 +122,8 @@ export async function transcribe(params: TranscribeParams): Promise<Segment[]> {
|
||||
stage: 'transcribing',
|
||||
progress: (i + 1) / plan.length,
|
||||
partial: stitchSegments(perChunk, plan.slice(0, i + 1), { overlapSec }),
|
||||
chunkIndex: i + 1,
|
||||
chunkCount: plan.length,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
// Session/UI state for the optional generative ("ask your lectures") layer
|
||||
// (Phase 5). Holds the BYO-key cloud config (if any), the on-device model load
|
||||
// status, and which backend is selected, and exposes a single ask() that wires
|
||||
// retrieval -> generation via askLectures.
|
||||
//
|
||||
// The feature is GATED BY AVAILABILITY, not a manual switch: `enabled` defaults
|
||||
// true, but ask() still degrades to the search-only fallback when no engine is
|
||||
// available (no WebGPU and no cloud key). Raw audio/transcripts never leave the
|
||||
// device — only the question + retrieved snippets do (cloud path).
|
||||
//
|
||||
// Persistence: on web the cloud config is saved to localStorage under
|
||||
// 'wisp:ai'; native keeps it in-memory (no localStorage). Everything here is
|
||||
// defensive — refreshAvailability never throws.
|
||||
import { create } from 'zustand';
|
||||
|
||||
import {
|
||||
getGenerationEngine,
|
||||
type CloudConfig,
|
||||
type GenerationEngine,
|
||||
type RagAnswer,
|
||||
} from '@/lib/generation';
|
||||
import { askLectures } from '@/lib/generation/rag';
|
||||
|
||||
const STORAGE_KEY = 'wisp:ai';
|
||||
|
||||
type ModelStatus = 'idle' | 'loading' | 'ready';
|
||||
type EngineKind = 'webllm' | 'cloud' | 'none';
|
||||
|
||||
interface PersistedState {
|
||||
enabled: boolean;
|
||||
cloud?: CloudConfig;
|
||||
}
|
||||
|
||||
/** Web-only localStorage handle (undefined on native / SSR). */
|
||||
function ls(): Storage | undefined {
|
||||
try {
|
||||
return typeof localStorage !== 'undefined' ? localStorage : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Read persisted enabled + cloud config from localStorage (web only). */
|
||||
function loadPersisted(): PersistedState {
|
||||
const store = ls();
|
||||
if (!store) return { enabled: true };
|
||||
try {
|
||||
const raw = store.getItem(STORAGE_KEY);
|
||||
if (!raw) return { enabled: true };
|
||||
const parsed = JSON.parse(raw) as Partial<PersistedState>;
|
||||
return {
|
||||
enabled: parsed.enabled ?? true,
|
||||
cloud: parsed.cloud,
|
||||
};
|
||||
} catch {
|
||||
return { enabled: true };
|
||||
}
|
||||
}
|
||||
|
||||
/** Persist (or clear) the cloud config + enabled flag on web. No-op on native. */
|
||||
function persist(state: PersistedState): void {
|
||||
const store = ls();
|
||||
if (!store) return;
|
||||
try {
|
||||
store.setItem(STORAGE_KEY, JSON.stringify(state));
|
||||
} catch {
|
||||
/* quota / private mode — ignore, just stay in-memory */
|
||||
}
|
||||
}
|
||||
|
||||
const initial = loadPersisted();
|
||||
|
||||
interface AiState {
|
||||
/** Feature flag — defaults true; real gating is via availability. */
|
||||
enabled: boolean;
|
||||
/** BYO-key cloud config, if the user supplied one. */
|
||||
cloud?: CloudConfig;
|
||||
/** On-device model load status (WebLLM). */
|
||||
modelStatus: ModelStatus;
|
||||
/** WebLLM download progress in [0,1]. */
|
||||
progress: number;
|
||||
/** Which backend the current config resolves to. */
|
||||
engineKind: EngineKind;
|
||||
|
||||
/** Set (or clear, with no arg) the cloud config; persists on web. */
|
||||
setCloud: (cfg?: CloudConfig) => void;
|
||||
/** Recompute engineKind and probe availability. Never throws. */
|
||||
refreshAvailability: () => Promise<void>;
|
||||
/** Resolve an engine, loading the on-device model if needed. */
|
||||
ensureReady: () => Promise<GenerationEngine>;
|
||||
/** Retrieve + (optionally) generate a grounded, cited answer. */
|
||||
ask: (
|
||||
question: string,
|
||||
opts?: { courseId?: string | null; signal?: AbortSignal },
|
||||
) => Promise<RagAnswer>;
|
||||
}
|
||||
|
||||
export const useAi = create<AiState>((set, get) => ({
|
||||
enabled: initial.enabled,
|
||||
cloud: initial.cloud,
|
||||
modelStatus: 'idle',
|
||||
progress: 0,
|
||||
// Resolve the initial backend synchronously from the persisted config.
|
||||
engineKind: getGenerationEngine(initial.cloud).kind,
|
||||
|
||||
setCloud: (cfg) => {
|
||||
persist({ enabled: get().enabled, cloud: cfg });
|
||||
// A cloud config swap invalidates any in-progress on-device load state.
|
||||
set({
|
||||
cloud: cfg,
|
||||
engineKind: getGenerationEngine(cfg).kind,
|
||||
modelStatus: 'idle',
|
||||
progress: 0,
|
||||
});
|
||||
},
|
||||
|
||||
refreshAvailability: async () => {
|
||||
try {
|
||||
const engine = getGenerationEngine(get().cloud);
|
||||
set({ engineKind: engine.kind });
|
||||
// Probe availability (WebGPU adapter / configured key). The result isn't
|
||||
// stored beyond engineKind; ask()/ensureReady() re-check at call time.
|
||||
await engine.isAvailable();
|
||||
} catch {
|
||||
// Stay defensive: never throw out of availability refresh.
|
||||
set({ engineKind: 'none' });
|
||||
}
|
||||
},
|
||||
|
||||
ensureReady: async () => {
|
||||
const engine = getGenerationEngine(get().cloud);
|
||||
set({ engineKind: engine.kind });
|
||||
// Only the on-device WebLLM engine has a model to download — and only when
|
||||
// it can actually run here (WebGPU). On a machine without WebGPU we skip the
|
||||
// (doomed, ~1GB) load and let askLectures degrade to search-only.
|
||||
if (
|
||||
engine.kind === 'webllm' &&
|
||||
!engine.isLoaded() &&
|
||||
(await engine.isAvailable())
|
||||
) {
|
||||
set({ modelStatus: 'loading', progress: 0 });
|
||||
try {
|
||||
await engine.loadModel((p) => set({ progress: p }));
|
||||
set({ modelStatus: 'ready', progress: 1 });
|
||||
} catch (err) {
|
||||
set({ modelStatus: 'idle', progress: 0 });
|
||||
throw err;
|
||||
}
|
||||
} else {
|
||||
set({ modelStatus: 'ready' });
|
||||
}
|
||||
return engine;
|
||||
},
|
||||
|
||||
ask: async (question, opts) => {
|
||||
const engine = await get().ensureReady();
|
||||
return askLectures(question, engine, opts);
|
||||
},
|
||||
}));
|
||||
@@ -0,0 +1,129 @@
|
||||
// Drives on-device indexing for semantic search (Phase 1): loads the embedding
|
||||
// model, embeds each transcript's segments into unit vectors, and persists them
|
||||
// to the StorageRepo's vector store. UI/session state only — the repo is the
|
||||
// record of truth for what's been embedded.
|
||||
import { create } from 'zustand';
|
||||
|
||||
import { getRepo } from '@/lib/db';
|
||||
import { EMBED_MODEL, getEmbeddingEngine } from '@/lib/embedding';
|
||||
|
||||
type Status = 'idle' | 'indexing' | 'ready';
|
||||
|
||||
interface EmbeddingState {
|
||||
status: Status;
|
||||
/** Build progress in [0,1] (0.2 = model load, 0.8 = embedding). */
|
||||
progress: number;
|
||||
/** Count of transcripts with no vectors for the active model. */
|
||||
pending: number;
|
||||
|
||||
/** Refresh `pending` from the repo (cheap; no model load). */
|
||||
refreshPending: () => Promise<void>;
|
||||
/** Embed every un-embedded transcript. Loads the model first. */
|
||||
buildIndex: () => Promise<void>;
|
||||
/** Embed a single transcript (used right after a new transcription). */
|
||||
embedOne: (transcriptId: string) => Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Embed and persist vectors for one transcript. Replaces any existing vectors
|
||||
* for it (upsertVectors semantics). No-op if the transcript or its segments are
|
||||
* gone. Caller is responsible for model loading and status/progress bookkeeping.
|
||||
*/
|
||||
async function embedTranscript(transcriptId: string): Promise<void> {
|
||||
const repo = getRepo();
|
||||
const t = await repo.get(transcriptId);
|
||||
if (!t) return;
|
||||
if (t.segments.length === 0) {
|
||||
// Nothing to embed, but record an (empty) vector set so it stops counting
|
||||
// as "unembedded" for this model.
|
||||
await repo.upsertVectors(transcriptId, EMBED_MODEL, []);
|
||||
return;
|
||||
}
|
||||
const eng = getEmbeddingEngine();
|
||||
// Embed in small batches, yielding to the event loop between them. The WASM
|
||||
// backend runs synchronously on the MAIN thread, so embedding a whole lecture
|
||||
// (100s of segments) in one call froze the UI for seconds — which, right after
|
||||
// a transcription, looked like the transcript screen was "stuck loading" and
|
||||
// starved any follow-up transcription. Per-text vectors are identical
|
||||
// regardless of batch size (e5 pools each text independently).
|
||||
const texts = t.segments.map((s) => s.text);
|
||||
const BATCH = 16;
|
||||
const vecs: Float32Array[] = [];
|
||||
for (let i = 0; i < texts.length; i += BATCH) {
|
||||
const out = await eng.embed(texts.slice(i, i + BATCH), 'passage');
|
||||
for (const v of out) vecs.push(v);
|
||||
// Hand the thread back so navigation, rendering and other work can run.
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
await repo.upsertVectors(
|
||||
transcriptId,
|
||||
EMBED_MODEL,
|
||||
t.segments.map((s, j) => ({
|
||||
segmentId: s.id ?? String(j),
|
||||
start: s.start,
|
||||
end: s.end,
|
||||
text: s.text,
|
||||
vector: vecs[j] ?? new Float32Array(eng.dim),
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
export const useEmbedding = create<EmbeddingState>((set, get) => ({
|
||||
status: 'idle',
|
||||
progress: 0,
|
||||
pending: 0,
|
||||
|
||||
refreshPending: async () => {
|
||||
const ids = await getRepo().unembeddedIds(EMBED_MODEL);
|
||||
set({ pending: ids.length });
|
||||
},
|
||||
|
||||
buildIndex: async () => {
|
||||
// Guard against concurrent runs.
|
||||
if (get().status === 'indexing') return;
|
||||
set({ status: 'indexing', progress: 0 });
|
||||
try {
|
||||
const eng = getEmbeddingEngine();
|
||||
// Model load is the first 20% of the bar.
|
||||
await eng.loadModel((p) => set({ progress: p * 0.2 }));
|
||||
|
||||
const ids = await getRepo().unembeddedIds(EMBED_MODEL);
|
||||
if (ids.length === 0) {
|
||||
set({ status: 'ready', progress: 1, pending: 0 });
|
||||
return;
|
||||
}
|
||||
|
||||
for (let i = 0; i < ids.length; i++) {
|
||||
const id = ids[i];
|
||||
if (id === undefined) continue;
|
||||
await embedTranscript(id);
|
||||
// Remaining 80% spread across the transcripts.
|
||||
set({ progress: 0.2 + 0.8 * ((i + 1) / ids.length) });
|
||||
}
|
||||
|
||||
set({ status: 'ready', progress: 1, pending: 0 });
|
||||
} catch (err) {
|
||||
// Surface progress reset but don't crash callers; leave pending intact so
|
||||
// a retry can pick up where it left off.
|
||||
set({ status: 'idle', progress: 0 });
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
|
||||
embedOne: async (transcriptId) => {
|
||||
// Guard against clobbering an in-flight full build.
|
||||
if (get().status === 'indexing') return;
|
||||
set({ status: 'indexing' });
|
||||
try {
|
||||
const eng = getEmbeddingEngine();
|
||||
// Lazy model load (no progress weighting for a single transcript).
|
||||
if (!eng.isLoaded()) await eng.loadModel();
|
||||
await embedTranscript(transcriptId);
|
||||
const pending = (await getRepo().unembeddedIds(EMBED_MODEL)).length;
|
||||
set({ status: 'ready', pending });
|
||||
} catch (err) {
|
||||
set({ status: 'idle' });
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
}));
|
||||
@@ -6,10 +6,12 @@ import { create } from 'zustand';
|
||||
|
||||
import { getDecoder, type AudioFileInput } from '@/lib/audio';
|
||||
import { getRepo } from '@/lib/db';
|
||||
import { errorMessage } from '@/lib/errorMessage';
|
||||
import { DEFAULT_MODEL } from '@/lib/models/catalog';
|
||||
import { getEngine } from '@/lib/transcription';
|
||||
import { transcribe } from '@/lib/transcription/pipeline';
|
||||
import type { ModelId, Segment } from '@/lib/types';
|
||||
import { useEmbedding } from './embeddingStore';
|
||||
|
||||
type Status = 'idle' | 'loading' | 'transcribing' | 'done' | 'error';
|
||||
|
||||
@@ -26,6 +28,9 @@ interface TranscribeState {
|
||||
stage?: 'loading' | 'transcribing';
|
||||
progress: number;
|
||||
partial: Segment[];
|
||||
/** During 'transcribing': how many chunks are done and how many total. */
|
||||
chunkIndex?: number;
|
||||
chunkCount?: number;
|
||||
error?: string;
|
||||
modelId: ModelId;
|
||||
lastTranscriptId?: string;
|
||||
@@ -65,6 +70,8 @@ export const useTranscribe = create<TranscribeState>((set, get) => ({
|
||||
stage: 'loading',
|
||||
progress: 0,
|
||||
partial: [],
|
||||
chunkIndex: undefined,
|
||||
chunkCount: undefined,
|
||||
error: undefined,
|
||||
lastTranscriptId: undefined,
|
||||
audioUrl: makeAudioUrl(input),
|
||||
@@ -80,7 +87,14 @@ export const useTranscribe = create<TranscribeState>((set, get) => ({
|
||||
engine: getEngine(),
|
||||
signal: abort.signal,
|
||||
onProgress: (p) =>
|
||||
set({ stage: p.stage, progress: p.progress, partial: p.partial, status: p.stage }),
|
||||
set({
|
||||
stage: p.stage,
|
||||
progress: p.progress,
|
||||
partial: p.partial,
|
||||
status: p.stage,
|
||||
chunkIndex: p.chunkIndex,
|
||||
chunkCount: p.chunkCount,
|
||||
}),
|
||||
});
|
||||
|
||||
const durationSec = pcm.samples.length / pcm.sampleRate;
|
||||
@@ -106,6 +120,11 @@ export const useTranscribe = create<TranscribeState>((set, get) => ({
|
||||
console.warn('[wisp] could not persist source audio:', e);
|
||||
}
|
||||
|
||||
// Index the new transcript for semantic search in the background. Lazy
|
||||
// model load happens inside embedOne; this must never block or fail the
|
||||
// transcription itself, so it's fire-and-forget with a swallowed error.
|
||||
void useEmbedding.getState().embedOne(saved.id).catch(() => {});
|
||||
|
||||
set({ status: 'done', progress: 1, partial: segments, lastTranscriptId: saved.id });
|
||||
return saved.id;
|
||||
} catch (err) {
|
||||
@@ -113,7 +132,7 @@ export const useTranscribe = create<TranscribeState>((set, get) => ({
|
||||
set({ status: 'idle', progress: 0 });
|
||||
return undefined;
|
||||
}
|
||||
set({ status: 'error', error: err instanceof Error ? err.message : String(err) });
|
||||
set({ status: 'error', error: errorMessage(err) });
|
||||
return undefined;
|
||||
}
|
||||
},
|
||||
|
||||
@@ -7,6 +7,11 @@ import { getRepo, type TranscriptMeta } from '@/lib/db';
|
||||
/** 'all' = every transcript · null = Unsorted · string = a specific course id. */
|
||||
export type CourseFilter = 'all' | null | string;
|
||||
|
||||
// Monotonic token so overlapping refreshes (focus + typing + filter toggles all
|
||||
// call refresh()) can't resolve out of order and clobber the list with a stale
|
||||
// query/filter's results. Only the latest-issued refresh is allowed to commit.
|
||||
let refreshToken = 0;
|
||||
|
||||
interface TranscriptsState {
|
||||
items: TranscriptMeta[];
|
||||
loading: boolean;
|
||||
@@ -27,6 +32,7 @@ export const useTranscripts = create<TranscriptsState>((set, get) => ({
|
||||
courseFilter: 'all',
|
||||
|
||||
refresh: async () => {
|
||||
const my = ++refreshToken;
|
||||
set({ loading: true });
|
||||
try {
|
||||
const repo = getRepo();
|
||||
@@ -40,9 +46,13 @@ export const useTranscripts = create<TranscriptsState>((set, get) => ({
|
||||
} else {
|
||||
items = cf === 'all' ? await repo.list() : await repo.listByCourse(cf);
|
||||
}
|
||||
// A newer refresh was issued while we awaited — discard our stale result.
|
||||
if (my !== refreshToken) return;
|
||||
set({ items });
|
||||
} finally {
|
||||
set({ loading: false });
|
||||
// Only the latest refresh clears the spinner, so it doesn't flicker off
|
||||
// while a newer in-flight refresh is still running.
|
||||
if (my === refreshToken) set({ loading: false });
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
Reference in New Issue
Block a user