From 251d88a37a7904227129f67a5e8099f1a647033b Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Wed, 8 Jul 2026 13:45:02 +0200 Subject: [PATCH] Stop Android audio interruptions crashing voice mode (#1941) * fix(android): handle denied voice audio focus Android can deny voice-mode audio focus while another system audio owner, such as an incoming call, is active. Treat that as an interruption so resume does not crash the app and JS stops voice mode coherently. * fix(voice): keep interruption state consistent Make native interruption handling payload-aware so iOS resume events do not stop the shared JS runtime, avoid ending voice mode for duck-only Android focus changes, and roll host voice mode back if capture fails after startup enabled it. * fix(voice): avoid duplicate interruption handling Keep exactly one Android audio-focus request active at a time and leave JS capture state untouched for non-terminal interruption events. * fix(android): abandon blocked audio focus requests When Android denies or blocks voice audio focus, abandon the pending focus request so delayed focus gain cannot arrive after voice mode has already stopped. * fix(voice): handle interrupted dictation capture Stop Android resume from replaying after recording restart fails, and propagate native blocked interruptions into dictation so users do not remain in a stale recording state. --- docs/mobile-testing.md | 13 ++++ packages/app/src/contexts/voice-context.tsx | 5 ++ .../use-dictation-audio-source.native.ts | 7 +- .../hooks/use-dictation-audio-source.types.ts | 1 + packages/app/src/hooks/use-dictation.ts | 37 ++++++---- packages/app/src/voice/audio-engine-types.ts | 1 + packages/app/src/voice/audio-engine.native.ts | 23 ++++++- packages/app/src/voice/voice-runtime.test.ts | 16 +++++ packages/app/src/voice/voice-runtime.ts | 5 ++ .../expo/modules/twowayaudio/AudioEngine.kt | 69 ++++++++++++++++--- 10 files changed, 152 insertions(+), 25 deletions(-) diff --git a/docs/mobile-testing.md b/docs/mobile-testing.md index 216c5d0cd..450ab3b4b 100644 --- a/docs/mobile-testing.md +++ b/docs/mobile-testing.md @@ -198,6 +198,19 @@ for i in $(seq 1 "$ITERATIONS"); do done ``` +### Android audio focus interruptions + +Voice mode uses the custom `expo-two-way-audio` Android module, so incoming calls and other system audio owners must be tested with emulator/system commands, not a JS-only test. To verify that voice resume handles denied audio focus without crashing: + +```bash +adb shell am start -n sh.paseo/.MainActivity +# Start voice mode in an existing composer, then background Paseo with Home. +adb emu gsm call 5551234 +# Foreground Paseo while the call is still ringing. +``` + +Expected result: Paseo does not throw `RuntimeException: Audio focus request failed`; native audio reports an interruption and voice mode stops or pauses coherently. + ## Unistyles + Reanimated ### The crash diff --git a/packages/app/src/contexts/voice-context.tsx b/packages/app/src/contexts/voice-context.tsx index a94fc736a..7bb0b9b7e 100644 --- a/packages/app/src/contexts/voice-context.tsx +++ b/packages/app/src/contexts/voice-context.tsx @@ -125,6 +125,11 @@ export function VoiceProvider({ children }: VoiceProviderProps) { onVolumeLevel: (level) => { runtime?.handleCaptureVolume(level); }, + onInterruption: () => { + void runtime?.stopVoice().catch((error) => { + console.error("[VoiceEngine] Failed to stop after audio interruption:", error); + }); + }, onError: (error) => { console.error("[VoiceEngine] Capture error:", error); }, diff --git a/packages/app/src/hooks/use-dictation-audio-source.native.ts b/packages/app/src/hooks/use-dictation-audio-source.native.ts index 50642fa15..aa50a8eea 100644 --- a/packages/app/src/hooks/use-dictation-audio-source.native.ts +++ b/packages/app/src/hooks/use-dictation-audio-source.native.ts @@ -12,6 +12,7 @@ import type { export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource { const onPcmSegmentRef = useRef(config.onPcmSegment); const onErrorRef = useRef(config.onError); + const onInterruptionRef = useRef(config.onInterruption); const [volume, setVolume] = useState(0); const engineRef = useRef | null>(null); @@ -30,6 +31,9 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic onError: (error) => { onErrorRef.current?.(error); }, + onInterruption: () => { + onInterruptionRef.current?.(); + }, }); return engineRef.current; }, []); @@ -37,7 +41,8 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic useEffect(() => { onPcmSegmentRef.current = config.onPcmSegment; onErrorRef.current = config.onError; - }, [config.onPcmSegment, config.onError]); + onInterruptionRef.current = config.onInterruption; + }, [config.onPcmSegment, config.onError, config.onInterruption]); const start = useCallback(async () => { const engine = getOrCreateEngine(); diff --git a/packages/app/src/hooks/use-dictation-audio-source.types.ts b/packages/app/src/hooks/use-dictation-audio-source.types.ts index 331c53945..711cf690c 100644 --- a/packages/app/src/hooks/use-dictation-audio-source.types.ts +++ b/packages/app/src/hooks/use-dictation-audio-source.types.ts @@ -1,6 +1,7 @@ export interface DictationAudioSourceConfig { onPcmSegment: (pcm16Base64: string) => void; onError?: (error: Error) => void; + onInterruption?: () => void; } export interface DictationAudioSource { diff --git a/packages/app/src/hooks/use-dictation.ts b/packages/app/src/hooks/use-dictation.ts index 2b01d9c52..6525ffa8a 100644 --- a/packages/app/src/hooks/use-dictation.ts +++ b/packages/app/src/hooks/use-dictation.ts @@ -184,19 +184,6 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { }); }, [client]); - const audio = useDictationAudioSource({ - onPcmSegment: (audioData) => { - senderRef.current?.enqueueSegment(audioData); - }, - onError: (err) => { - onErrorRef.current?.(err); - }, - }); - const audioStopRef = useRef(audio.stop); - useEffect(() => { - audioStopRef.current = audio.stop; - }, [audio.stop]); - const handleStreamingTranscriptionSuccess = useCallback( (text: string, requestId: string) => { setIsProcessing(false); @@ -220,6 +207,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { (failure: unknown) => { const normalized = toError(failure); const failureId = generateMessageId(); + stopDurationTracking(); setIsProcessing(false); isProcessingRef.current = false; isRecordingRef.current = false; @@ -234,9 +222,30 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { reportError(normalized, "Failed to complete dictation"); }, - [reportError], + [reportError, stopDurationTracking], ); + const audio = useDictationAudioSource({ + onPcmSegment: (audioData) => { + senderRef.current?.enqueueSegment(audioData); + }, + onError: (err) => { + onErrorRef.current?.(err); + }, + onInterruption: () => { + try { + senderRef.current?.cancel(); + } catch { + // no-op + } + handleDictationFailure(new Error("Dictation was interrupted by another audio source.")); + }, + }); + const audioStopRef = useRef(audio.stop); + useEffect(() => { + audioStopRef.current = audio.stop; + }, [audio.stop]); + const startDictation = useCallback(async () => { if ( actionGateRef.current.starting || diff --git a/packages/app/src/voice/audio-engine-types.ts b/packages/app/src/voice/audio-engine-types.ts index 44c6f4b84..03108b5cf 100644 --- a/packages/app/src/voice/audio-engine-types.ts +++ b/packages/app/src/voice/audio-engine-types.ts @@ -1,6 +1,7 @@ export interface AudioEngineCallbacks { onCaptureData(pcm: Uint8Array): void; onVolumeLevel(level: number): void; + onInterruption?(): void; onError?(error: Error): void; } diff --git a/packages/app/src/voice/audio-engine.native.ts b/packages/app/src/voice/audio-engine.native.ts index 0f543be86..81f0fda85 100644 --- a/packages/app/src/voice/audio-engine.native.ts +++ b/packages/app/src/voice/audio-engine.native.ts @@ -116,6 +116,21 @@ export function createAudioEngine( callbacks.onVolumeLevel(level); }, ); + const interruptionSubscription = native.addExpoTwoWayAudioEventListener( + "onAudioInterruption", + (event: { data: string }) => { + if (event.data !== "blocked") { + return; + } + const wasCaptureActive = refs.captureActive; + refs.captureActive = false; + refs.muted = false; + callbacks.onVolumeLevel(0); + if (wasCaptureActive) { + callbacks.onInterruption?.(); + } + }, + ); async function ensureInitialized(): Promise { if (refs.initialized) { @@ -234,6 +249,7 @@ export function createAudioEngine( } microphoneSubscription.remove(); volumeSubscription.remove(); + interruptionSubscription.remove(); }, async startCapture() { @@ -244,7 +260,12 @@ export function createAudioEngine( try { await ensureMicrophonePermission(); await ensureInitialized(); - native.toggleRecording(true); + const isRecording = native.toggleRecording(true); + if (!isRecording) { + throw new Error( + "Microphone capture could not start because Android audio focus is unavailable.", + ); + } refs.captureActive = true; } catch (error) { const wrapped = error instanceof Error ? error : new Error(String(error)); diff --git a/packages/app/src/voice/voice-runtime.test.ts b/packages/app/src/voice/voice-runtime.test.ts index 3d5ce92e8..aa79d6b24 100644 --- a/packages/app/src/voice/voice-runtime.test.ts +++ b/packages/app/src/voice/voice-runtime.test.ts @@ -423,6 +423,22 @@ describe("voice runtime", () => { expect(runtime.shouldPlayVoiceAudio("server-1")).toBe(false); }); + it("rolls daemon voice mode back when capture fails after enabling it", async () => { + const adapter = createSessionAdapter(); + const engine = createAudioEngineMock(); + vi.mocked(engine.startCapture).mockRejectedValue(new Error("audio focus unavailable")); + const { runtime } = createRuntime({ engine }); + runtime.registerSession(adapter); + + await expect(runtime.startVoice("server-1", "agent-1")).rejects.toThrow( + "audio focus unavailable", + ); + + expect(adapter.setVoiceMode).toHaveBeenNthCalledWith(1, true, "agent-1"); + expect(adapter.setVoiceMode).toHaveBeenNthCalledWith(2, false); + expect(runtime.getSnapshot().phase).toBe("disabled"); + }); + it("returns an explicit not-ready error when the adapter is missing", async () => { const { runtime } = createRuntime(); await expect(runtime.startVoice("server-1", "agent-1")).rejects.toThrow( diff --git a/packages/app/src/voice/voice-runtime.ts b/packages/app/src/voice/voice-runtime.ts index b43ca41eb..6bb42499e 100644 --- a/packages/app/src/voice/voice-runtime.ts +++ b/packages/app/src/voice/voice-runtime.ts @@ -730,6 +730,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { const previousServerId = state.snapshot.activeServerId; const previousAgentId = state.snapshot.activeAgentId; const generation = state.generation + 1; + let enabledCurrentVoiceMode = false; state.generation = generation; state.transportReady = false; patchSnapshot((prev) => ({ @@ -759,6 +760,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { await deps.engine.initialize(); await session.adapter.setVoiceMode(true, agentId); + enabledCurrentVoiceMode = true; await deps.engine.startCapture(); if (state.generation !== generation) { return; @@ -776,6 +778,9 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { isMuted: deps.engine.isMuted(), })); } catch (error) { + if (enabledCurrentVoiceMode) { + await session.adapter.setVoiceMode(false).catch(() => undefined); + } await performLocalStop(); throw error; } diff --git a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt index bca4262ac..cfa32ba59 100644 --- a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt +++ b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt @@ -83,7 +83,9 @@ class AudioEngine (context: Context) { private fun initializeAudio(context:Context) { audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager audioManager.mode = AudioManager.MODE_IN_COMMUNICATION - requestAudioFocus() + if (!requestAudioFocus()) { + handleAudioFocusBlocked() + } // Route audio to external device if connected, otherwise route to speaker updateAudioRouting() @@ -168,7 +170,12 @@ class AudioEngine (context: Context) { } @SuppressLint("NewApi") - private fun requestAudioFocus() { + private fun requestAudioFocus(): Boolean { + audioFocusRequest?.let { request -> + audioManager.abandonAudioFocusRequest(request) + audioFocusRequest = null + } + val focusRequest = AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE) .setAudioAttributes( @@ -180,9 +187,19 @@ class AudioEngine (context: Context) { .setAcceptsDelayedFocusGain(true) .setOnAudioFocusChangeListener { focusChange -> when (focusChange) { + AudioManager.AUDIOFOCUS_GAIN -> { + Log.d("AudioEngine", "Audio focus gained") + } AudioManager.AUDIOFOCUS_LOSS -> { Log.d("AudioEngine", "Audio focus lost") - onAudioInterruptionCallback?.let { it("blocked") } + handleAudioFocusBlocked() + } + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT -> { + Log.d("AudioEngine", "Audio focus lost transiently") + handleAudioFocusBlocked() + } + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK -> { + Log.d("AudioEngine", "Audio focus duck requested") } } } @@ -191,14 +208,40 @@ class AudioEngine (context: Context) { audioFocusRequest = focusRequest val result = audioManager.requestAudioFocus(focusRequest) - if (result != AudioManager.AUDIOFOCUS_REQUEST_GRANTED) { - throw RuntimeException("Audio focus request failed") + if (result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED) { + return true } + + Log.w("AudioEngine", "Audio focus request was not granted: $result") + return false + } + + private fun handleAudioFocusBlocked() { + if (isRecording) { + stopRecording() + } + audioFocusRequest?.let { request -> + audioManager.abandonAudioFocusRequest(request) + audioFocusRequest = null + } + if (::audioTrack.isInitialized) { + audioTrack.pause() + } + audioSampleQueue.clear() + isPlaying = false + isRecordingBeforePause = false + onOutputVolumeCallback?.invoke(0.0F) + onAudioInterruptionCallback?.let { it("blocked") } } @RequiresApi(Build.VERSION_CODES.Q) @SuppressLint("MissingPermission") - private fun startRecording(){ + private fun startRecording(): Boolean { + if (!requestAudioFocus()) { + handleAudioFocusBlocked() + return false + } + val bufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT) audioRecord = AudioRecord( MediaRecorder.AudioSource.VOICE_COMMUNICATION, @@ -231,6 +274,7 @@ class AudioEngine (context: Context) { audioRecord.startRecording() isRecording = true startMicSampleTap() + return true } private fun startMicSampleTap(){ @@ -274,7 +318,7 @@ class AudioEngine (context: Context) { if (value == isRecording) return isRecording if (value) { - startRecording() + return startRecording() } else { stopRecording() } @@ -353,8 +397,15 @@ class AudioEngine (context: Context) { @RequiresApi(Build.VERSION_CODES.Q) fun resumeRecordingAndPlayer() { - requestAudioFocus() - isRecording = toggleRecording(isRecordingBeforePause) + if (!requestAudioFocus()) { + handleAudioFocusBlocked() + return + } + val wasRecordingBeforePause = isRecordingBeforePause + isRecording = toggleRecording(wasRecordingBeforePause) + if (wasRecordingBeforePause && !isRecording) { + return + } audioTrack.play() }