Native: download whisper model on demand + surface real errors
Two native-only fixes behind the "transcription failed [object Object]" report on the APK: 1. Model download. engineImpl.native.loadModel assumed the ggml .bin was already on disk (the downloader was a TODO), so on a real device it rejected — and since in-app recording is web-only, no model was ever present. Now loadModel fetches the model from Hugging Face (ggerganov/whisper.cpp) into <documentDirectory>/models on first use, reporting 0..1 progress through the existing "Loading model… X%" UI. Downloads to a .part file and renames on success so an interrupted download can't leave a truncated model; if initWhisper still rejects, the file is deleted so the next try re-downloads cleanly. Model URLs verified (tiny.en 77.7MB / base.en 148MB / small.en 488MB). 2. Error surfacing. transcribeStore did `String(err)`, which renders a non-Error native rejection (whisper.rn / file system throw plain objects) as "[object Object]". New shared errorMessage() pulls message/reason/code (or JSON) out of whatever was thrown, so failures are actionable instead of opaque. Progress downloads aren't in the new expo-file-system OO API yet, so the fetch uses the still-supported expo-file-system/legacy resumable downloader; native-only, not bundled on web. Validated: tsc clean, 279 tests pass, web export unaffected (legacy not in the web bundle), arm64 APK builds and the native JS bundle resolves the legacy import. On-device download + transcribe still needs a real phone to confirm end to end. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,24 @@
|
|||||||
|
// Extract a human-readable message from an unknown thrown value.
|
||||||
|
//
|
||||||
|
// `String(err)` on a non-Error object yields the useless "[object Object]",
|
||||||
|
// which is exactly what surfaced when native modules (whisper.rn, the audio
|
||||||
|
// decoder, file system) reject with a plain `{ message, code }`-shaped object
|
||||||
|
// instead of an `Error`. This pulls the most useful text out of whatever was
|
||||||
|
// thrown so the UI shows something actionable.
|
||||||
|
export function errorMessage(err: unknown): string {
|
||||||
|
if (err instanceof Error) return err.message;
|
||||||
|
if (typeof err === 'string') return err;
|
||||||
|
if (err && typeof err === 'object') {
|
||||||
|
const o = err as { message?: unknown; reason?: unknown; code?: unknown };
|
||||||
|
const parts = [o.message, o.reason, o.code]
|
||||||
|
.filter((x) => x != null && x !== '')
|
||||||
|
.map(String);
|
||||||
|
if (parts.length > 0) return parts.join(' — ');
|
||||||
|
try {
|
||||||
|
return JSON.stringify(err);
|
||||||
|
} catch {
|
||||||
|
/* fall through to String() for circular/unserializable objects */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return String(err);
|
||||||
|
}
|
||||||
@@ -28,7 +28,10 @@
|
|||||||
|
|
||||||
import { initWhisper } from 'whisper.rn';
|
import { initWhisper } from 'whisper.rn';
|
||||||
import type { WhisperContext } from 'whisper.rn';
|
import type { WhisperContext } from 'whisper.rn';
|
||||||
import { Paths, File } from 'expo-file-system';
|
import { Paths, File, Directory } from 'expo-file-system';
|
||||||
|
// Progress-reporting downloads aren't in the new OO File API yet, so we use the
|
||||||
|
// (still supported) legacy resumable downloader for the model fetch.
|
||||||
|
import { createDownloadResumable } from 'expo-file-system/legacy';
|
||||||
import type {
|
import type {
|
||||||
ModelId,
|
ModelId,
|
||||||
PcmAudio,
|
PcmAudio,
|
||||||
@@ -36,33 +39,95 @@ import type {
|
|||||||
TranscribeOptions,
|
TranscribeOptions,
|
||||||
} from '../types';
|
} from '../types';
|
||||||
import { MODELS, recommendModel } from '../models/catalog';
|
import { MODELS, recommendModel } from '../models/catalog';
|
||||||
|
import { errorMessage } from '../errorMessage';
|
||||||
import type { EngineCapabilities, TranscriptionEngine } from './engine';
|
import type { EngineCapabilities, TranscriptionEngine } from './engine';
|
||||||
|
|
||||||
/** Loaded whisper.cpp contexts, keyed by model id. */
|
/** Loaded whisper.cpp contexts, keyed by model id. */
|
||||||
const loaded = new Map<ModelId, WhisperContext>();
|
const loaded = new Map<ModelId, WhisperContext>();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolve the on-device path of a model's ggml file.
|
* Where the official whisper.cpp ggml models are hosted. The catalog's `ggml`
|
||||||
|
* field is the exact filename in this repo (e.g. 'ggml-tiny.en.bin').
|
||||||
|
*/
|
||||||
|
const GGML_BASE_URL = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The on-device file handle for a model's ggml binary.
|
||||||
*
|
*
|
||||||
* Models live under `<documentDirectory>/models/<ggml-filename>`. The document
|
* Models live under `<documentDirectory>/models/<ggml-filename>`. The document
|
||||||
* directory survives app restarts and OS storage pressure, which is what we want
|
* directory survives app restarts and OS storage pressure, which is what we want
|
||||||
* for multi-hundred-MB model files.
|
* for multi-hundred-MB model files.
|
||||||
*
|
|
||||||
* TODO: the model DOWNLOAD UI / fetcher is NOT built yet. This function only
|
|
||||||
* computes where the file *should* be; the .bin must already exist there
|
|
||||||
* (e.g. side-loaded during development) or loadModel() will reject. Wire a
|
|
||||||
* downloader (File.downloadFileAsync into Paths.document/'models') before
|
|
||||||
* shipping.
|
|
||||||
*
|
|
||||||
* whisper.rn's initWhisper expects a plain filesystem path. Expo's File `.uri`
|
|
||||||
* is a `file://` URI, so we strip the scheme.
|
|
||||||
*/
|
*/
|
||||||
function modelPathFor(id: ModelId): string {
|
function modelFile(id: ModelId): File {
|
||||||
const file = new File(Paths.document, 'models', MODELS[id].ggml);
|
return new File(Paths.document, 'models', MODELS[id].ggml);
|
||||||
// e.g. 'file:///data/.../Documents/models/ggml-tiny.en.bin' -> '/data/.../ggml-tiny.en.bin'
|
}
|
||||||
|
|
||||||
|
/** whisper.rn's initWhisper wants a plain path; Expo's `.uri` is `file://…`. */
|
||||||
|
function plainPath(file: File): string {
|
||||||
return file.uri.replace(/^file:\/\//, '');
|
return file.uri.replace(/^file:\/\//, '');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure a model's ggml file is present on disk, downloading it from Hugging
|
||||||
|
* Face if needed. Reports 0..1 download progress via `onProgress`. Downloads to
|
||||||
|
* a `.part` file and renames on success, so an interrupted download never leaves
|
||||||
|
* a truncated file that a later run would mistake for a complete model.
|
||||||
|
*/
|
||||||
|
async function ensureModelDownloaded(
|
||||||
|
id: ModelId,
|
||||||
|
onProgress?: (p: number) => void,
|
||||||
|
): Promise<File> {
|
||||||
|
const dest = modelFile(id);
|
||||||
|
if (dest.exists) return dest;
|
||||||
|
|
||||||
|
// Make sure <documentDirectory>/models/ exists.
|
||||||
|
const dir = new Directory(Paths.document, 'models');
|
||||||
|
if (!dir.exists) dir.create({ intermediates: true, idempotent: true });
|
||||||
|
|
||||||
|
const url = `${GGML_BASE_URL}/${MODELS[id].ggml}`;
|
||||||
|
const partUri = `${dest.uri}.part`;
|
||||||
|
|
||||||
|
// Clean up any leftover .part from a previously-aborted attempt.
|
||||||
|
const part = new File(partUri);
|
||||||
|
if (part.exists) {
|
||||||
|
try {
|
||||||
|
part.delete();
|
||||||
|
} catch {
|
||||||
|
/* best effort */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const task = createDownloadResumable(url, partUri, {}, (p) => {
|
||||||
|
if (p.totalBytesExpectedToWrite > 0) {
|
||||||
|
onProgress?.(p.totalBytesWritten / p.totalBytesExpectedToWrite);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await task.downloadAsync();
|
||||||
|
if (!result || result.status !== 200) {
|
||||||
|
throw new Error(
|
||||||
|
`server returned ${result?.status ?? 'no response'}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
try {
|
||||||
|
if (part.exists) part.delete();
|
||||||
|
} catch {
|
||||||
|
/* best effort */
|
||||||
|
}
|
||||||
|
throw new Error(
|
||||||
|
`Couldn't download the ${MODELS[id].label} model (~${MODELS[id].approxMB} MB): ` +
|
||||||
|
`${errorMessage(e)}. Check your connection and try again.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Atomically put the completed file in place.
|
||||||
|
part.move(dest);
|
||||||
|
onProgress?.(1);
|
||||||
|
return dest;
|
||||||
|
}
|
||||||
|
|
||||||
export const engine: TranscriptionEngine = {
|
export const engine: TranscriptionEngine = {
|
||||||
platform: 'native',
|
platform: 'native',
|
||||||
|
|
||||||
@@ -81,13 +146,31 @@ export const engine: TranscriptionEngine = {
|
|||||||
};
|
};
|
||||||
},
|
},
|
||||||
|
|
||||||
async loadModel(modelId: ModelId): Promise<void> {
|
async loadModel(
|
||||||
|
modelId: ModelId,
|
||||||
|
onProgress?: (p: number) => void,
|
||||||
|
): Promise<void> {
|
||||||
if (loaded.has(modelId)) return; // idempotent
|
if (loaded.has(modelId)) return; // idempotent
|
||||||
|
|
||||||
const filePath = modelPathFor(modelId);
|
// Fetch the ggml file on first use (with download progress), then load it.
|
||||||
// initWhisper rejects if the file is missing/corrupt. See the TODO in
|
const file = await ensureModelDownloaded(modelId, onProgress);
|
||||||
// modelPathFor: until the download UI exists, the .bin must already be there.
|
|
||||||
const ctx = await initWhisper({ filePath });
|
let ctx: WhisperContext;
|
||||||
|
try {
|
||||||
|
ctx = await initWhisper({ filePath: plainPath(file) });
|
||||||
|
} catch (e) {
|
||||||
|
// The file is present but unusable (corrupt / truncated). Delete it so the
|
||||||
|
// next attempt re-downloads a clean copy rather than failing forever.
|
||||||
|
try {
|
||||||
|
if (file.exists) file.delete();
|
||||||
|
} catch {
|
||||||
|
/* best effort */
|
||||||
|
}
|
||||||
|
throw new Error(
|
||||||
|
`The ${MODELS[modelId].label} model couldn't be loaded and was cleared — ` +
|
||||||
|
`try again. (${errorMessage(e)})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
loaded.set(modelId, ctx);
|
loaded.set(modelId, ctx);
|
||||||
},
|
},
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import { create } from 'zustand';
|
|||||||
|
|
||||||
import { getDecoder, type AudioFileInput } from '@/lib/audio';
|
import { getDecoder, type AudioFileInput } from '@/lib/audio';
|
||||||
import { getRepo } from '@/lib/db';
|
import { getRepo } from '@/lib/db';
|
||||||
|
import { errorMessage } from '@/lib/errorMessage';
|
||||||
import { DEFAULT_MODEL } from '@/lib/models/catalog';
|
import { DEFAULT_MODEL } from '@/lib/models/catalog';
|
||||||
import { getEngine } from '@/lib/transcription';
|
import { getEngine } from '@/lib/transcription';
|
||||||
import { transcribe } from '@/lib/transcription/pipeline';
|
import { transcribe } from '@/lib/transcription/pipeline';
|
||||||
@@ -119,7 +120,7 @@ export const useTranscribe = create<TranscribeState>((set, get) => ({
|
|||||||
set({ status: 'idle', progress: 0 });
|
set({ status: 'idle', progress: 0 });
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
set({ status: 'error', error: err instanceof Error ? err.message : String(err) });
|
set({ status: 'error', error: errorMessage(err) });
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
Reference in New Issue
Block a user