diff --git a/docs/mobile-testing.md b/docs/mobile-testing.md index 216c5d0cd..450ab3b4b 100644 --- a/docs/mobile-testing.md +++ b/docs/mobile-testing.md @@ -198,6 +198,19 @@ for i in $(seq 1 "$ITERATIONS"); do done ``` +### Android audio focus interruptions + +Voice mode uses the custom `expo-two-way-audio` Android module, so incoming calls and other system audio owners must be tested with emulator/system commands, not a JS-only test. To verify that voice resume handles denied audio focus without crashing: + +```bash +adb shell am start -n sh.paseo/.MainActivity +# Start voice mode in an existing composer, then background Paseo with Home. +adb emu gsm call 5551234 +# Foreground Paseo while the call is still ringing. +``` + +Expected result: Paseo does not throw `RuntimeException: Audio focus request failed`; native audio reports an interruption and voice mode stops or pauses coherently. + ## Unistyles + Reanimated ### The crash diff --git a/packages/app/src/contexts/voice-context.tsx b/packages/app/src/contexts/voice-context.tsx index a94fc736a..7bb0b9b7e 100644 --- a/packages/app/src/contexts/voice-context.tsx +++ b/packages/app/src/contexts/voice-context.tsx @@ -125,6 +125,11 @@ export function VoiceProvider({ children }: VoiceProviderProps) { onVolumeLevel: (level) => { runtime?.handleCaptureVolume(level); }, + onInterruption: () => { + void runtime?.stopVoice().catch((error) => { + console.error("[VoiceEngine] Failed to stop after audio interruption:", error); + }); + }, onError: (error) => { console.error("[VoiceEngine] Capture error:", error); }, diff --git a/packages/app/src/hooks/use-dictation-audio-source.native.ts b/packages/app/src/hooks/use-dictation-audio-source.native.ts index 50642fa15..aa50a8eea 100644 --- a/packages/app/src/hooks/use-dictation-audio-source.native.ts +++ b/packages/app/src/hooks/use-dictation-audio-source.native.ts @@ -12,6 +12,7 @@ import type { export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource { const onPcmSegmentRef = useRef(config.onPcmSegment); const onErrorRef = useRef(config.onError); + const onInterruptionRef = useRef(config.onInterruption); const [volume, setVolume] = useState(0); const engineRef = useRef | null>(null); @@ -30,6 +31,9 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic onError: (error) => { onErrorRef.current?.(error); }, + onInterruption: () => { + onInterruptionRef.current?.(); + }, }); return engineRef.current; }, []); @@ -37,7 +41,8 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic useEffect(() => { onPcmSegmentRef.current = config.onPcmSegment; onErrorRef.current = config.onError; - }, [config.onPcmSegment, config.onError]); + onInterruptionRef.current = config.onInterruption; + }, [config.onPcmSegment, config.onError, config.onInterruption]); const start = useCallback(async () => { const engine = getOrCreateEngine(); diff --git a/packages/app/src/hooks/use-dictation-audio-source.types.ts b/packages/app/src/hooks/use-dictation-audio-source.types.ts index 331c53945..711cf690c 100644 --- a/packages/app/src/hooks/use-dictation-audio-source.types.ts +++ b/packages/app/src/hooks/use-dictation-audio-source.types.ts @@ -1,6 +1,7 @@ export interface DictationAudioSourceConfig { onPcmSegment: (pcm16Base64: string) => void; onError?: (error: Error) => void; + onInterruption?: () => void; } export interface DictationAudioSource { diff --git a/packages/app/src/hooks/use-dictation.ts b/packages/app/src/hooks/use-dictation.ts index 2b01d9c52..6525ffa8a 100644 --- a/packages/app/src/hooks/use-dictation.ts +++ b/packages/app/src/hooks/use-dictation.ts @@ -184,19 +184,6 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { }); }, [client]); - const audio = useDictationAudioSource({ - onPcmSegment: (audioData) => { - senderRef.current?.enqueueSegment(audioData); - }, - onError: (err) => { - onErrorRef.current?.(err); - }, - }); - const audioStopRef = useRef(audio.stop); - useEffect(() => { - audioStopRef.current = audio.stop; - }, [audio.stop]); - const handleStreamingTranscriptionSuccess = useCallback( (text: string, requestId: string) => { setIsProcessing(false); @@ -220,6 +207,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { (failure: unknown) => { const normalized = toError(failure); const failureId = generateMessageId(); + stopDurationTracking(); setIsProcessing(false); isProcessingRef.current = false; isRecordingRef.current = false; @@ -234,9 +222,30 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { reportError(normalized, "Failed to complete dictation"); }, - [reportError], + [reportError, stopDurationTracking], ); + const audio = useDictationAudioSource({ + onPcmSegment: (audioData) => { + senderRef.current?.enqueueSegment(audioData); + }, + onError: (err) => { + onErrorRef.current?.(err); + }, + onInterruption: () => { + try { + senderRef.current?.cancel(); + } catch { + // no-op + } + handleDictationFailure(new Error("Dictation was interrupted by another audio source.")); + }, + }); + const audioStopRef = useRef(audio.stop); + useEffect(() => { + audioStopRef.current = audio.stop; + }, [audio.stop]); + const startDictation = useCallback(async () => { if ( actionGateRef.current.starting || diff --git a/packages/app/src/voice/audio-engine-types.ts b/packages/app/src/voice/audio-engine-types.ts index 44c6f4b84..03108b5cf 100644 --- a/packages/app/src/voice/audio-engine-types.ts +++ b/packages/app/src/voice/audio-engine-types.ts @@ -1,6 +1,7 @@ export interface AudioEngineCallbacks { onCaptureData(pcm: Uint8Array): void; onVolumeLevel(level: number): void; + onInterruption?(): void; onError?(error: Error): void; } diff --git a/packages/app/src/voice/audio-engine.native.ts b/packages/app/src/voice/audio-engine.native.ts index 0f543be86..81f0fda85 100644 --- a/packages/app/src/voice/audio-engine.native.ts +++ b/packages/app/src/voice/audio-engine.native.ts @@ -116,6 +116,21 @@ export function createAudioEngine( callbacks.onVolumeLevel(level); }, ); + const interruptionSubscription = native.addExpoTwoWayAudioEventListener( + "onAudioInterruption", + (event: { data: string }) => { + if (event.data !== "blocked") { + return; + } + const wasCaptureActive = refs.captureActive; + refs.captureActive = false; + refs.muted = false; + callbacks.onVolumeLevel(0); + if (wasCaptureActive) { + callbacks.onInterruption?.(); + } + }, + ); async function ensureInitialized(): Promise { if (refs.initialized) { @@ -234,6 +249,7 @@ export function createAudioEngine( } microphoneSubscription.remove(); volumeSubscription.remove(); + interruptionSubscription.remove(); }, async startCapture() { @@ -244,7 +260,12 @@ export function createAudioEngine( try { await ensureMicrophonePermission(); await ensureInitialized(); - native.toggleRecording(true); + const isRecording = native.toggleRecording(true); + if (!isRecording) { + throw new Error( + "Microphone capture could not start because Android audio focus is unavailable.", + ); + } refs.captureActive = true; } catch (error) { const wrapped = error instanceof Error ? error : new Error(String(error)); diff --git a/packages/app/src/voice/voice-runtime.test.ts b/packages/app/src/voice/voice-runtime.test.ts index 3d5ce92e8..aa79d6b24 100644 --- a/packages/app/src/voice/voice-runtime.test.ts +++ b/packages/app/src/voice/voice-runtime.test.ts @@ -423,6 +423,22 @@ describe("voice runtime", () => { expect(runtime.shouldPlayVoiceAudio("server-1")).toBe(false); }); + it("rolls daemon voice mode back when capture fails after enabling it", async () => { + const adapter = createSessionAdapter(); + const engine = createAudioEngineMock(); + vi.mocked(engine.startCapture).mockRejectedValue(new Error("audio focus unavailable")); + const { runtime } = createRuntime({ engine }); + runtime.registerSession(adapter); + + await expect(runtime.startVoice("server-1", "agent-1")).rejects.toThrow( + "audio focus unavailable", + ); + + expect(adapter.setVoiceMode).toHaveBeenNthCalledWith(1, true, "agent-1"); + expect(adapter.setVoiceMode).toHaveBeenNthCalledWith(2, false); + expect(runtime.getSnapshot().phase).toBe("disabled"); + }); + it("returns an explicit not-ready error when the adapter is missing", async () => { const { runtime } = createRuntime(); await expect(runtime.startVoice("server-1", "agent-1")).rejects.toThrow( diff --git a/packages/app/src/voice/voice-runtime.ts b/packages/app/src/voice/voice-runtime.ts index b43ca41eb..6bb42499e 100644 --- a/packages/app/src/voice/voice-runtime.ts +++ b/packages/app/src/voice/voice-runtime.ts @@ -730,6 +730,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { const previousServerId = state.snapshot.activeServerId; const previousAgentId = state.snapshot.activeAgentId; const generation = state.generation + 1; + let enabledCurrentVoiceMode = false; state.generation = generation; state.transportReady = false; patchSnapshot((prev) => ({ @@ -759,6 +760,7 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { await deps.engine.initialize(); await session.adapter.setVoiceMode(true, agentId); + enabledCurrentVoiceMode = true; await deps.engine.startCapture(); if (state.generation !== generation) { return; @@ -776,6 +778,9 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { isMuted: deps.engine.isMuted(), })); } catch (error) { + if (enabledCurrentVoiceMode) { + await session.adapter.setVoiceMode(false).catch(() => undefined); + } await performLocalStop(); throw error; } diff --git a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt index bca4262ac..cfa32ba59 100644 --- a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt +++ b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt @@ -83,7 +83,9 @@ class AudioEngine (context: Context) { private fun initializeAudio(context:Context) { audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager audioManager.mode = AudioManager.MODE_IN_COMMUNICATION - requestAudioFocus() + if (!requestAudioFocus()) { + handleAudioFocusBlocked() + } // Route audio to external device if connected, otherwise route to speaker updateAudioRouting() @@ -168,7 +170,12 @@ class AudioEngine (context: Context) { } @SuppressLint("NewApi") - private fun requestAudioFocus() { + private fun requestAudioFocus(): Boolean { + audioFocusRequest?.let { request -> + audioManager.abandonAudioFocusRequest(request) + audioFocusRequest = null + } + val focusRequest = AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE) .setAudioAttributes( @@ -180,9 +187,19 @@ class AudioEngine (context: Context) { .setAcceptsDelayedFocusGain(true) .setOnAudioFocusChangeListener { focusChange -> when (focusChange) { + AudioManager.AUDIOFOCUS_GAIN -> { + Log.d("AudioEngine", "Audio focus gained") + } AudioManager.AUDIOFOCUS_LOSS -> { Log.d("AudioEngine", "Audio focus lost") - onAudioInterruptionCallback?.let { it("blocked") } + handleAudioFocusBlocked() + } + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT -> { + Log.d("AudioEngine", "Audio focus lost transiently") + handleAudioFocusBlocked() + } + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK -> { + Log.d("AudioEngine", "Audio focus duck requested") } } } @@ -191,14 +208,40 @@ class AudioEngine (context: Context) { audioFocusRequest = focusRequest val result = audioManager.requestAudioFocus(focusRequest) - if (result != AudioManager.AUDIOFOCUS_REQUEST_GRANTED) { - throw RuntimeException("Audio focus request failed") + if (result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED) { + return true } + + Log.w("AudioEngine", "Audio focus request was not granted: $result") + return false + } + + private fun handleAudioFocusBlocked() { + if (isRecording) { + stopRecording() + } + audioFocusRequest?.let { request -> + audioManager.abandonAudioFocusRequest(request) + audioFocusRequest = null + } + if (::audioTrack.isInitialized) { + audioTrack.pause() + } + audioSampleQueue.clear() + isPlaying = false + isRecordingBeforePause = false + onOutputVolumeCallback?.invoke(0.0F) + onAudioInterruptionCallback?.let { it("blocked") } } @RequiresApi(Build.VERSION_CODES.Q) @SuppressLint("MissingPermission") - private fun startRecording(){ + private fun startRecording(): Boolean { + if (!requestAudioFocus()) { + handleAudioFocusBlocked() + return false + } + val bufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT) audioRecord = AudioRecord( MediaRecorder.AudioSource.VOICE_COMMUNICATION, @@ -231,6 +274,7 @@ class AudioEngine (context: Context) { audioRecord.startRecording() isRecording = true startMicSampleTap() + return true } private fun startMicSampleTap(){ @@ -274,7 +318,7 @@ class AudioEngine (context: Context) { if (value == isRecording) return isRecording if (value) { - startRecording() + return startRecording() } else { stopRecording() } @@ -353,8 +397,15 @@ class AudioEngine (context: Context) { @RequiresApi(Build.VERSION_CODES.Q) fun resumeRecordingAndPlayer() { - requestAudioFocus() - isRecording = toggleRecording(isRecordingBeforePause) + if (!requestAudioFocus()) { + handleAudioFocusBlocked() + return + } + val wasRecordingBeforePause = isRecordingBeforePause + isRecording = toggleRecording(wasRecordingBeforePause) + if (wasRecordingBeforePause && !isRecording) { + return + } audioTrack.play() }