From 7aa8b05971ec2cc9ac9da50ae630332d25118b8b Mon Sep 17 00:00:00 2001 From: Nils Briggen Date: Mon, 15 Jun 2026 09:03:31 +0200 Subject: [PATCH] Native audio: decode m4a/mp3/etc. (not just WAV) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On native, importing anything but WAV failed with "Only WAV is supported on native" — ffmpeg-kit-react-native was retired in 2025 and a real decoder was left as a follow-up. Since in-app recording is web-only, the native app's only way to add audio is importing a phone recording — which is m4a/aac — so transcription was effectively broken on the APK. Add react-native-audio-api (Software Mansion, maintained) and route non-WAV files through its standalone decodeAudioData(ArrayBuffer), which uses platform codecs + bundled FFmpeg. WAV keeps its pure-JS fast path. The decoded channels go through the existing toMono16k, mirroring the web AudioContext.decodeAudioData path exactly. Bytes are read locally and passed as an ArrayBuffer, so it doesn't matter whether the picker yields file:// or content://. WAV is now sniffed by RIFF/WAVE content, not just the extension. The lib's config plugin is intentionally NOT enabled: it only sets up playback (background audio mode, a media-playback foreground service, mic permission), none of which decoding needs — and adding those would contradict the app's on-device/no-extra-permissions stance. Validated locally (arm64 Docker build): react-native-audio-api compiles and autolinks against RN 0.85 / new arch; the arm64-v8a APK ships libreact-native-audio-api.so + libav{codec,format,util}/libswresample.so + liboboe.so (48MB -> 58MB). tsc clean, 279 tests pass, web export does NOT bundle the native dep. Runtime decode still needs an on-device check. Co-Authored-By: Claude Opus 4.8 --- bun.lock | 3 ++ package.json | 1 + src/lib/audio/decode.native.ts | 87 ++++++++++++++++++++++++---------- 3 files changed, 66 insertions(+), 25 deletions(-) diff --git a/bun.lock b/bun.lock index a41ce38..19006fd 100644 --- a/bun.lock +++ b/bun.lock @@ -27,6 +27,7 @@ "react": "19.2.3", "react-dom": "19.2.3", "react-native": "0.85.3", + "react-native-audio-api": "0.12.2", "react-native-gesture-handler": "~2.31.1", "react-native-reanimated": "4.3.1", "react-native-safe-area-context": "~5.7.0", @@ -1450,6 +1451,8 @@ "react-native": ["react-native@0.85.3", "", { "dependencies": { "@react-native/assets-registry": "0.85.3", "@react-native/codegen": "0.85.3", "@react-native/community-cli-plugin": "0.85.3", "@react-native/gradle-plugin": "0.85.3", "@react-native/js-polyfills": "0.85.3", "@react-native/normalize-colors": "0.85.3", "@react-native/virtualized-lists": "0.85.3", "abort-controller": "^3.0.0", "anser": "^1.4.9", "ansi-regex": "^5.0.0", "babel-plugin-syntax-hermes-parser": "0.33.3", "base64-js": "^1.5.1", "commander": "^12.0.0", "flow-enums-runtime": "^0.0.6", "hermes-compiler": "250829098.0.10", "invariant": "^2.2.4", "memoize-one": "^5.0.0", "metro-runtime": "^0.84.3", "metro-source-map": "^0.84.3", "nullthrows": "^1.1.1", "pretty-format": "^29.7.0", "promise": "^8.3.0", "react-devtools-core": "^6.1.5", "react-refresh": "^0.14.0", "regenerator-runtime": "^0.13.2", "scheduler": "0.27.0", "semver": "^7.1.3", "stacktrace-parser": "^0.1.10", "tinyglobby": "^0.2.15", "whatwg-fetch": "^3.0.0", "ws": "^7.5.10", "yargs": "^17.6.2" }, "peerDependencies": { "@react-native/jest-preset": "0.85.3", "@types/react": "^19.1.1", "react": "^19.2.3" }, "optionalPeers": ["@react-native/jest-preset", "@types/react"], "bin": { "react-native": "cli.js" } }, "sha512-HN/fGC+3nZVcDNcw7gfbM/DuqZAvI9Mz+/SxuhODaua4JY0BPzhfTzWXRyTR4mRgMHmShTPpH2PYMTxvZrsdZA=="], + "react-native-audio-api": ["react-native-audio-api@0.12.2", "", { "dependencies": { "semver": "^7.7.3" }, "peerDependencies": { "react": "*", "react-native": "*", "react-native-worklets": ">= 0.6.0" }, "optionalPeers": ["react-native-worklets"], "bin": { "setup-rn-audio-api-web": "scripts/setup-rn-audio-api-web.js" } }, "sha512-OizgzmolTzCwgMlNv8eyRTh888up6mmS158O1+h9M8dHE4dTSZjjT0622zDJ+Z58trEyYHRcOJP9evBMTzm4UQ=="], + "react-native-drawer-layout": ["react-native-drawer-layout@4.2.5", "", { "dependencies": { "color": "^4.2.3", "use-latest-callback": "^0.2.4" }, "peerDependencies": { "react": ">= 18.2.0", "react-native": "*", "react-native-gesture-handler": ">= 2.0.0", "react-native-reanimated": ">= 2.0.0" } }, "sha512-Yl82uLkXjXuq7222hWGIDsq5A6R/bsCeCEgdIxQUxAEHf00oRdDnRByLx3Fsij3qwtmYNPGrHV1NH8G8hbCbLQ=="], "react-native-gesture-handler": ["react-native-gesture-handler@2.31.2", "", { "dependencies": { "@egjs/hammerjs": "^2.0.17", "@types/react-test-renderer": "^19.1.0", "hoist-non-react-statics": "^3.3.0", "invariant": "^2.2.4" }, "peerDependencies": { "react": "*", "react-native": "*" } }, "sha512-rw5q74i2AfS7YGYdbxQDhOU7xqgY6WRM1132/CCm3erqjblhECZDZFHIm0tteHoC9ih24wogVBVVzcTBQtZ+5A=="], diff --git a/package.json b/package.json index 5473697..1b8097a 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "react": "19.2.3", "react-dom": "19.2.3", "react-native": "0.85.3", + "react-native-audio-api": "0.12.2", "react-native-gesture-handler": "~2.31.1", "react-native-reanimated": "4.3.1", "react-native-safe-area-context": "~5.7.0", diff --git a/src/lib/audio/decode.native.ts b/src/lib/audio/decode.native.ts index 590a664..e4f4bd5 100644 --- a/src/lib/audio/decode.native.ts +++ b/src/lib/audio/decode.native.ts @@ -1,24 +1,27 @@ // Native (iOS/Android) audio decoder. // -// Unlike the web, React Native has no built-in media decoder we can rely on. -// `ffmpeg-kit-react-native` — the package everyone used to reach for — was -// RETIRED by its maintainer in early 2025, so we deliberately do NOT depend on -// it. For now we support only WAV, which we can decode in pure JS via our own -// `decodeWav`. Everything else throws a clear, actionable error. +// WAV is decoded in pure JS (`decodeWav`) — fast and dependency-free. Every +// other container/codec (m4a/aac, mp3, ogg/opus, flac, …) is decoded by +// `react-native-audio-api`, which uses the platform codecs (Android MediaCodec / +// iOS AVFoundation, with a bundled FFmpeg fallback). This mirrors the web +// decoder's `AudioContext.decodeAudioData` path. Decoding is fully on-device — +// the audio bytes never leave the phone. // -// TODO(audio/native): add a maintained native decoder for compressed formats -// (mp3/m4a/aac/ogg/flac). The current front-runner is -// `react-native-audio-api` (a Web-Audio-style API for RN). When added, route -// non-WAV URIs through it and downmix/resample with `toMono16k`, mirroring the -// web path. +// Why this matters on native: in-app recording is web-only, so importing a file +// from the phone's own recorder is the ONLY way to get audio in — and those are +// almost always m4a/aac. (Previously native accepted WAV only.) +// +// We hand react-native-audio-api the raw file BYTES (ArrayBuffer), not a URI, so +// we don't depend on how the OS file picker spells the path (file:// vs +// content://) — `readFileBytes` already reads either via Expo's File API. // // File reading uses Expo SDK 56's object-oriented `File` API // (`new File(uri).bytes()`), which returns the raw bytes as a `Uint8Array` -// directly — no base64 round-trip needed. We still keep a base64 decode helper -// below as a documented fallback for environments/URIs where `.bytes()` isn't -// available (it reads via `.base64()` and decodes to a Uint8Array by hand). +// directly — no base64 round-trip needed. We keep a base64 fallback for +// environments/URIs where `.bytes()` isn't available. import { File } from 'expo-file-system'; +import { decodeAudioData } from 'react-native-audio-api'; import type { AudioDecoder, AudioFileInput } from './decode'; import type { PcmAudio } from '../types'; @@ -90,6 +93,25 @@ async function readFileBytes(uri: string): Promise { } } +/** + * True if these bytes are a RIFF/WAVE container. Sniffing the content (rather + * than trusting the file extension) lets us take the fast pure-JS WAV path even + * for mislabeled files, and avoids handing WAV to the heavier native decoder. + */ +function isWavBytes(b: Uint8Array): boolean { + return ( + b.length >= 12 && + b[0] === 0x52 && // 'R' + b[1] === 0x49 && // 'I' + b[2] === 0x46 && // 'F' + b[3] === 0x46 && // 'F' + b[8] === 0x57 && // 'W' + b[9] === 0x41 && // 'A' + b[10] === 0x56 && // 'V' + b[11] === 0x45 // 'E' + ); +} + export const decoder: AudioDecoder = { async decode(input: AudioFileInput): Promise { const { uri } = input; @@ -97,22 +119,37 @@ export const decoder: AudioDecoder = { throw new Error('Native decoder requires a file `uri`.'); } - // Sniff the container from the URI extension (case-insensitive), ignoring - // any query string / fragment that content URIs sometimes carry. - const path = uri.split(/[?#]/, 1)[0] ?? uri; - const isWav = path.toLowerCase().endsWith('.wav'); + const bytes = await readFileBytes(uri); - if (!isWav) { + // Fast path: WAV is decoded in pure JS (no native call). + if (isWavBytes(bytes)) { + const { sampleRate, channelData } = decodeWav(bytes); + return { sampleRate: 16000, samples: toMono16k(channelData, sampleRate) }; + } + + // Everything else (m4a/aac, mp3, ogg/opus, flac, …) -> platform codecs via + // react-native-audio-api. A fresh, contiguous ArrayBuffer of exactly the + // file bytes is what `decodeAudioData` expects. + const arrayBuffer = new Uint8Array(bytes).buffer as ArrayBuffer; + let audioBuffer; + try { + audioBuffer = await decodeAudioData(arrayBuffer); + } catch (e) { + const detail = e instanceof Error ? e.message : String(e); throw new Error( - 'Only WAV is supported on native for now — ffmpeg-kit-react-native ' + - 'was retired in 2025; a maintained native decoder (e.g. ' + - 'react-native-audio-api) is a follow-up.', + `Couldn't decode this audio on device (${detail}). Supported formats: ` + + 'WAV, m4a/aac, mp3, ogg, flac. If it keeps failing, convert it to ' + + 'WAV and import again.', ); } - const bytes = await readFileBytes(uri); - const { sampleRate, channelData } = decodeWav(bytes); - const samples = toMono16k(channelData, sampleRate); - return { sampleRate: 16000, samples }; + const channelData: Float32Array[] = Array.from( + { length: audioBuffer.numberOfChannels }, + (_unused, c) => audioBuffer.getChannelData(c), + ); + return { + sampleRate: 16000, + samples: toMono16k(channelData, audioBuffer.sampleRate), + }; }, };