Native: download whisper model on demand + surface real errors
CI / test (push) Successful in 18s
CI / deploy-web (push) Successful in 33s
CI / build-apk (push) Successful in 6m40s

Two native-only fixes behind the "transcription failed [object Object]"
report on the APK:

1. Model download. engineImpl.native.loadModel assumed the ggml .bin was
   already on disk (the downloader was a TODO), so on a real device it
   rejected — and since in-app recording is web-only, no model was ever
   present. Now loadModel fetches the model from Hugging Face
   (ggerganov/whisper.cpp) into <documentDirectory>/models on first use,
   reporting 0..1 progress through the existing "Loading model… X%" UI.
   Downloads to a .part file and renames on success so an interrupted
   download can't leave a truncated model; if initWhisper still rejects,
   the file is deleted so the next try re-downloads cleanly. Model URLs
   verified (tiny.en 77.7MB / base.en 148MB / small.en 488MB).

2. Error surfacing. transcribeStore did `String(err)`, which renders a
   non-Error native rejection (whisper.rn / file system throw plain
   objects) as "[object Object]". New shared errorMessage() pulls
   message/reason/code (or JSON) out of whatever was thrown, so failures
   are actionable instead of opaque.

Progress downloads aren't in the new expo-file-system OO API yet, so the
fetch uses the still-supported expo-file-system/legacy resumable
downloader; native-only, not bundled on web.

Validated: tsc clean, 279 tests pass, web export unaffected (legacy not
in the web bundle), arm64 APK builds and the native JS bundle resolves
the legacy import. On-device download + transcribe still needs a real
phone to confirm end to end.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-15 09:21:34 +02:00
parent 7aa8b05971
commit 06519faea2
3 changed files with 128 additions and 20 deletions
+24
View File
@@ -0,0 +1,24 @@
// Extract a human-readable message from an unknown thrown value.
//
// `String(err)` on a non-Error object yields the useless "[object Object]",
// which is exactly what surfaced when native modules (whisper.rn, the audio
// decoder, file system) reject with a plain `{ message, code }`-shaped object
// instead of an `Error`. This pulls the most useful text out of whatever was
// thrown so the UI shows something actionable.
export function errorMessage(err: unknown): string {
if (err instanceof Error) return err.message;
if (typeof err === 'string') return err;
if (err && typeof err === 'object') {
const o = err as { message?: unknown; reason?: unknown; code?: unknown };
const parts = [o.message, o.reason, o.code]
.filter((x) => x != null && x !== '')
.map(String);
if (parts.length > 0) return parts.join(' — ');
try {
return JSON.stringify(err);
} catch {
/* fall through to String() for circular/unserializable objects */
}
}
return String(err);
}
+102 -19
View File
@@ -28,7 +28,10 @@
import { initWhisper } from 'whisper.rn'; import { initWhisper } from 'whisper.rn';
import type { WhisperContext } from 'whisper.rn'; import type { WhisperContext } from 'whisper.rn';
import { Paths, File } from 'expo-file-system'; import { Paths, File, Directory } from 'expo-file-system';
// Progress-reporting downloads aren't in the new OO File API yet, so we use the
// (still supported) legacy resumable downloader for the model fetch.
import { createDownloadResumable } from 'expo-file-system/legacy';
import type { import type {
ModelId, ModelId,
PcmAudio, PcmAudio,
@@ -36,33 +39,95 @@ import type {
TranscribeOptions, TranscribeOptions,
} from '../types'; } from '../types';
import { MODELS, recommendModel } from '../models/catalog'; import { MODELS, recommendModel } from '../models/catalog';
import { errorMessage } from '../errorMessage';
import type { EngineCapabilities, TranscriptionEngine } from './engine'; import type { EngineCapabilities, TranscriptionEngine } from './engine';
/** Loaded whisper.cpp contexts, keyed by model id. */ /** Loaded whisper.cpp contexts, keyed by model id. */
const loaded = new Map<ModelId, WhisperContext>(); const loaded = new Map<ModelId, WhisperContext>();
/** /**
* Resolve the on-device path of a model's ggml file. * Where the official whisper.cpp ggml models are hosted. The catalog's `ggml`
* field is the exact filename in this repo (e.g. 'ggml-tiny.en.bin').
*/
const GGML_BASE_URL = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main';
/**
* The on-device file handle for a model's ggml binary.
* *
* Models live under `<documentDirectory>/models/<ggml-filename>`. The document * Models live under `<documentDirectory>/models/<ggml-filename>`. The document
* directory survives app restarts and OS storage pressure, which is what we want * directory survives app restarts and OS storage pressure, which is what we want
* for multi-hundred-MB model files. * for multi-hundred-MB model files.
*
* TODO: the model DOWNLOAD UI / fetcher is NOT built yet. This function only
* computes where the file *should* be; the .bin must already exist there
* (e.g. side-loaded during development) or loadModel() will reject. Wire a
* downloader (File.downloadFileAsync into Paths.document/'models') before
* shipping.
*
* whisper.rn's initWhisper expects a plain filesystem path. Expo's File `.uri`
* is a `file://` URI, so we strip the scheme.
*/ */
function modelPathFor(id: ModelId): string { function modelFile(id: ModelId): File {
const file = new File(Paths.document, 'models', MODELS[id].ggml); return new File(Paths.document, 'models', MODELS[id].ggml);
// e.g. 'file:///data/.../Documents/models/ggml-tiny.en.bin' -> '/data/.../ggml-tiny.en.bin' }
/** whisper.rn's initWhisper wants a plain path; Expo's `.uri` is `file://…`. */
function plainPath(file: File): string {
return file.uri.replace(/^file:\/\//, ''); return file.uri.replace(/^file:\/\//, '');
} }
/**
* Ensure a model's ggml file is present on disk, downloading it from Hugging
* Face if needed. Reports 0..1 download progress via `onProgress`. Downloads to
* a `.part` file and renames on success, so an interrupted download never leaves
* a truncated file that a later run would mistake for a complete model.
*/
async function ensureModelDownloaded(
id: ModelId,
onProgress?: (p: number) => void,
): Promise<File> {
const dest = modelFile(id);
if (dest.exists) return dest;
// Make sure <documentDirectory>/models/ exists.
const dir = new Directory(Paths.document, 'models');
if (!dir.exists) dir.create({ intermediates: true, idempotent: true });
const url = `${GGML_BASE_URL}/${MODELS[id].ggml}`;
const partUri = `${dest.uri}.part`;
// Clean up any leftover .part from a previously-aborted attempt.
const part = new File(partUri);
if (part.exists) {
try {
part.delete();
} catch {
/* best effort */
}
}
const task = createDownloadResumable(url, partUri, {}, (p) => {
if (p.totalBytesExpectedToWrite > 0) {
onProgress?.(p.totalBytesWritten / p.totalBytesExpectedToWrite);
}
});
try {
const result = await task.downloadAsync();
if (!result || result.status !== 200) {
throw new Error(
`server returned ${result?.status ?? 'no response'}`,
);
}
} catch (e) {
try {
if (part.exists) part.delete();
} catch {
/* best effort */
}
throw new Error(
`Couldn't download the ${MODELS[id].label} model (~${MODELS[id].approxMB} MB): ` +
`${errorMessage(e)}. Check your connection and try again.`,
);
}
// Atomically put the completed file in place.
part.move(dest);
onProgress?.(1);
return dest;
}
export const engine: TranscriptionEngine = { export const engine: TranscriptionEngine = {
platform: 'native', platform: 'native',
@@ -81,13 +146,31 @@ export const engine: TranscriptionEngine = {
}; };
}, },
async loadModel(modelId: ModelId): Promise<void> { async loadModel(
modelId: ModelId,
onProgress?: (p: number) => void,
): Promise<void> {
if (loaded.has(modelId)) return; // idempotent if (loaded.has(modelId)) return; // idempotent
const filePath = modelPathFor(modelId); // Fetch the ggml file on first use (with download progress), then load it.
// initWhisper rejects if the file is missing/corrupt. See the TODO in const file = await ensureModelDownloaded(modelId, onProgress);
// modelPathFor: until the download UI exists, the .bin must already be there.
const ctx = await initWhisper({ filePath }); let ctx: WhisperContext;
try {
ctx = await initWhisper({ filePath: plainPath(file) });
} catch (e) {
// The file is present but unusable (corrupt / truncated). Delete it so the
// next attempt re-downloads a clean copy rather than failing forever.
try {
if (file.exists) file.delete();
} catch {
/* best effort */
}
throw new Error(
`The ${MODELS[modelId].label} model couldn't be loaded and was cleared — ` +
`try again. (${errorMessage(e)})`,
);
}
loaded.set(modelId, ctx); loaded.set(modelId, ctx);
}, },
+2 -1
View File
@@ -6,6 +6,7 @@ import { create } from 'zustand';
import { getDecoder, type AudioFileInput } from '@/lib/audio'; import { getDecoder, type AudioFileInput } from '@/lib/audio';
import { getRepo } from '@/lib/db'; import { getRepo } from '@/lib/db';
import { errorMessage } from '@/lib/errorMessage';
import { DEFAULT_MODEL } from '@/lib/models/catalog'; import { DEFAULT_MODEL } from '@/lib/models/catalog';
import { getEngine } from '@/lib/transcription'; import { getEngine } from '@/lib/transcription';
import { transcribe } from '@/lib/transcription/pipeline'; import { transcribe } from '@/lib/transcription/pipeline';
@@ -119,7 +120,7 @@ export const useTranscribe = create<TranscribeState>((set, get) => ({
set({ status: 'idle', progress: 0 }); set({ status: 'idle', progress: 0 });
return undefined; return undefined;
} }
set({ status: 'error', error: err instanceof Error ? err.message : String(err) }); set({ status: 'error', error: errorMessage(err) });
return undefined; return undefined;
} }
}, },