Native: download whisper model on demand + surface real errors
Two native-only fixes behind the "transcription failed [object Object]" report on the APK: 1. Model download. engineImpl.native.loadModel assumed the ggml .bin was already on disk (the downloader was a TODO), so on a real device it rejected — and since in-app recording is web-only, no model was ever present. Now loadModel fetches the model from Hugging Face (ggerganov/whisper.cpp) into <documentDirectory>/models on first use, reporting 0..1 progress through the existing "Loading model… X%" UI. Downloads to a .part file and renames on success so an interrupted download can't leave a truncated model; if initWhisper still rejects, the file is deleted so the next try re-downloads cleanly. Model URLs verified (tiny.en 77.7MB / base.en 148MB / small.en 488MB). 2. Error surfacing. transcribeStore did `String(err)`, which renders a non-Error native rejection (whisper.rn / file system throw plain objects) as "[object Object]". New shared errorMessage() pulls message/reason/code (or JSON) out of whatever was thrown, so failures are actionable instead of opaque. Progress downloads aren't in the new expo-file-system OO API yet, so the fetch uses the still-supported expo-file-system/legacy resumable downloader; native-only, not bundled on web. Validated: tsc clean, 279 tests pass, web export unaffected (legacy not in the web bundle), arm64 APK builds and the native JS bundle resolves the legacy import. On-device download + transcribe still needs a real phone to confirm end to end. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
// Extract a human-readable message from an unknown thrown value.
|
||||
//
|
||||
// `String(err)` on a non-Error object yields the useless "[object Object]",
|
||||
// which is exactly what surfaced when native modules (whisper.rn, the audio
|
||||
// decoder, file system) reject with a plain `{ message, code }`-shaped object
|
||||
// instead of an `Error`. This pulls the most useful text out of whatever was
|
||||
// thrown so the UI shows something actionable.
|
||||
export function errorMessage(err: unknown): string {
|
||||
if (err instanceof Error) return err.message;
|
||||
if (typeof err === 'string') return err;
|
||||
if (err && typeof err === 'object') {
|
||||
const o = err as { message?: unknown; reason?: unknown; code?: unknown };
|
||||
const parts = [o.message, o.reason, o.code]
|
||||
.filter((x) => x != null && x !== '')
|
||||
.map(String);
|
||||
if (parts.length > 0) return parts.join(' — ');
|
||||
try {
|
||||
return JSON.stringify(err);
|
||||
} catch {
|
||||
/* fall through to String() for circular/unserializable objects */
|
||||
}
|
||||
}
|
||||
return String(err);
|
||||
}
|
||||
@@ -28,7 +28,10 @@
|
||||
|
||||
import { initWhisper } from 'whisper.rn';
|
||||
import type { WhisperContext } from 'whisper.rn';
|
||||
import { Paths, File } from 'expo-file-system';
|
||||
import { Paths, File, Directory } from 'expo-file-system';
|
||||
// Progress-reporting downloads aren't in the new OO File API yet, so we use the
|
||||
// (still supported) legacy resumable downloader for the model fetch.
|
||||
import { createDownloadResumable } from 'expo-file-system/legacy';
|
||||
import type {
|
||||
ModelId,
|
||||
PcmAudio,
|
||||
@@ -36,33 +39,95 @@ import type {
|
||||
TranscribeOptions,
|
||||
} from '../types';
|
||||
import { MODELS, recommendModel } from '../models/catalog';
|
||||
import { errorMessage } from '../errorMessage';
|
||||
import type { EngineCapabilities, TranscriptionEngine } from './engine';
|
||||
|
||||
/** Loaded whisper.cpp contexts, keyed by model id. */
|
||||
const loaded = new Map<ModelId, WhisperContext>();
|
||||
|
||||
/**
|
||||
* Resolve the on-device path of a model's ggml file.
|
||||
* Where the official whisper.cpp ggml models are hosted. The catalog's `ggml`
|
||||
* field is the exact filename in this repo (e.g. 'ggml-tiny.en.bin').
|
||||
*/
|
||||
const GGML_BASE_URL = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main';
|
||||
|
||||
/**
|
||||
* The on-device file handle for a model's ggml binary.
|
||||
*
|
||||
* Models live under `<documentDirectory>/models/<ggml-filename>`. The document
|
||||
* directory survives app restarts and OS storage pressure, which is what we want
|
||||
* for multi-hundred-MB model files.
|
||||
*
|
||||
* TODO: the model DOWNLOAD UI / fetcher is NOT built yet. This function only
|
||||
* computes where the file *should* be; the .bin must already exist there
|
||||
* (e.g. side-loaded during development) or loadModel() will reject. Wire a
|
||||
* downloader (File.downloadFileAsync into Paths.document/'models') before
|
||||
* shipping.
|
||||
*
|
||||
* whisper.rn's initWhisper expects a plain filesystem path. Expo's File `.uri`
|
||||
* is a `file://` URI, so we strip the scheme.
|
||||
*/
|
||||
function modelPathFor(id: ModelId): string {
|
||||
const file = new File(Paths.document, 'models', MODELS[id].ggml);
|
||||
// e.g. 'file:///data/.../Documents/models/ggml-tiny.en.bin' -> '/data/.../ggml-tiny.en.bin'
|
||||
function modelFile(id: ModelId): File {
|
||||
return new File(Paths.document, 'models', MODELS[id].ggml);
|
||||
}
|
||||
|
||||
/** whisper.rn's initWhisper wants a plain path; Expo's `.uri` is `file://…`. */
|
||||
function plainPath(file: File): string {
|
||||
return file.uri.replace(/^file:\/\//, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a model's ggml file is present on disk, downloading it from Hugging
|
||||
* Face if needed. Reports 0..1 download progress via `onProgress`. Downloads to
|
||||
* a `.part` file and renames on success, so an interrupted download never leaves
|
||||
* a truncated file that a later run would mistake for a complete model.
|
||||
*/
|
||||
async function ensureModelDownloaded(
|
||||
id: ModelId,
|
||||
onProgress?: (p: number) => void,
|
||||
): Promise<File> {
|
||||
const dest = modelFile(id);
|
||||
if (dest.exists) return dest;
|
||||
|
||||
// Make sure <documentDirectory>/models/ exists.
|
||||
const dir = new Directory(Paths.document, 'models');
|
||||
if (!dir.exists) dir.create({ intermediates: true, idempotent: true });
|
||||
|
||||
const url = `${GGML_BASE_URL}/${MODELS[id].ggml}`;
|
||||
const partUri = `${dest.uri}.part`;
|
||||
|
||||
// Clean up any leftover .part from a previously-aborted attempt.
|
||||
const part = new File(partUri);
|
||||
if (part.exists) {
|
||||
try {
|
||||
part.delete();
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
}
|
||||
|
||||
const task = createDownloadResumable(url, partUri, {}, (p) => {
|
||||
if (p.totalBytesExpectedToWrite > 0) {
|
||||
onProgress?.(p.totalBytesWritten / p.totalBytesExpectedToWrite);
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await task.downloadAsync();
|
||||
if (!result || result.status !== 200) {
|
||||
throw new Error(
|
||||
`server returned ${result?.status ?? 'no response'}`,
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
try {
|
||||
if (part.exists) part.delete();
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
throw new Error(
|
||||
`Couldn't download the ${MODELS[id].label} model (~${MODELS[id].approxMB} MB): ` +
|
||||
`${errorMessage(e)}. Check your connection and try again.`,
|
||||
);
|
||||
}
|
||||
|
||||
// Atomically put the completed file in place.
|
||||
part.move(dest);
|
||||
onProgress?.(1);
|
||||
return dest;
|
||||
}
|
||||
|
||||
export const engine: TranscriptionEngine = {
|
||||
platform: 'native',
|
||||
|
||||
@@ -81,13 +146,31 @@ export const engine: TranscriptionEngine = {
|
||||
};
|
||||
},
|
||||
|
||||
async loadModel(modelId: ModelId): Promise<void> {
|
||||
async loadModel(
|
||||
modelId: ModelId,
|
||||
onProgress?: (p: number) => void,
|
||||
): Promise<void> {
|
||||
if (loaded.has(modelId)) return; // idempotent
|
||||
|
||||
const filePath = modelPathFor(modelId);
|
||||
// initWhisper rejects if the file is missing/corrupt. See the TODO in
|
||||
// modelPathFor: until the download UI exists, the .bin must already be there.
|
||||
const ctx = await initWhisper({ filePath });
|
||||
// Fetch the ggml file on first use (with download progress), then load it.
|
||||
const file = await ensureModelDownloaded(modelId, onProgress);
|
||||
|
||||
let ctx: WhisperContext;
|
||||
try {
|
||||
ctx = await initWhisper({ filePath: plainPath(file) });
|
||||
} catch (e) {
|
||||
// The file is present but unusable (corrupt / truncated). Delete it so the
|
||||
// next attempt re-downloads a clean copy rather than failing forever.
|
||||
try {
|
||||
if (file.exists) file.delete();
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
throw new Error(
|
||||
`The ${MODELS[modelId].label} model couldn't be loaded and was cleared — ` +
|
||||
`try again. (${errorMessage(e)})`,
|
||||
);
|
||||
}
|
||||
loaded.set(modelId, ctx);
|
||||
},
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import { create } from 'zustand';
|
||||
|
||||
import { getDecoder, type AudioFileInput } from '@/lib/audio';
|
||||
import { getRepo } from '@/lib/db';
|
||||
import { errorMessage } from '@/lib/errorMessage';
|
||||
import { DEFAULT_MODEL } from '@/lib/models/catalog';
|
||||
import { getEngine } from '@/lib/transcription';
|
||||
import { transcribe } from '@/lib/transcription/pipeline';
|
||||
@@ -119,7 +120,7 @@ export const useTranscribe = create<TranscribeState>((set, get) => ({
|
||||
set({ status: 'idle', progress: 0 });
|
||||
return undefined;
|
||||
}
|
||||
set({ status: 'error', error: err instanceof Error ? err.message : String(err) });
|
||||
set({ status: 'error', error: errorMessage(err) });
|
||||
return undefined;
|
||||
}
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user