From 6e1ec21c6a4e784131a1247670c4705ddda9f524 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Sat, 17 Oct 2026 12:53:06 +0700 Subject: [PATCH] Keep Android dictation running through audio interruptions (#7200) * fix(android): preserve dictation across audio interruptions Playback focus loss does not mean microphone capture is unavailable. Keep dictation alive across media takeovers and wait for Android input silencing to clear, while preserving conversation interruption protection. * test(android): verify dictation clears its selected audio route --- docs/mobile-testing.md | 27 ++- packages/app/src/audio/audio-engine-types.ts | 6 + packages/app/src/audio/audio-engine.native.ts | 21 +- packages/app/src/audio/audio-engine.web.ts | 9 +- .../app/src/components/dictation-controls.tsx | 9 +- .../use-dictation-audio-source.native.ts | 32 ++- .../hooks/use-dictation-audio-source.types.ts | 1 + .../hooks/use-dictation-audio-source.web.ts | 1 + .../app/src/hooks/use-dictation.shared.ts | 2 +- packages/app/src/hooks/use-dictation.ts | 2 +- packages/app/src/i18n/resources/ar.ts | 1 + packages/app/src/i18n/resources/en.ts | 1 + packages/app/src/i18n/resources/es.ts | 1 + packages/app/src/i18n/resources/fr.ts | 1 + packages/app/src/i18n/resources/ja.ts | 1 + packages/app/src/i18n/resources/ko.ts | 1 + packages/app/src/i18n/resources/pt-BR.ts | 1 + packages/app/src/i18n/resources/ru.ts | 1 + packages/app/src/i18n/resources/zh-CN.ts | 1 + .../expo-two-way-audio/android/build.gradle | 6 + .../src/androidTest/AndroidManifest.xml | 8 + .../modules/twowayaudio/AudioEngineTest.kt | 223 ++++++++++++++++++ .../expo/modules/twowayaudio/AudioEngine.kt | 212 ++++++++++------- .../twowayaudio/ExpoTwoWayAudioModule.kt | 6 + .../ios/ExpoTwoWayAudioModule.swift | 6 + packages/expo-two-way-audio/src/core.ts | 10 + 26 files changed, 479 insertions(+), 111 deletions(-) create mode 100644 packages/expo-two-way-audio/android/src/androidTest/AndroidManifest.xml create mode 100644 packages/expo-two-way-audio/android/src/androidTest/java/expo/modules/twowayaudio/AudioEngineTest.kt diff --git a/docs/mobile-testing.md b/docs/mobile-testing.md index 93fa35aedd..375da30708 100644 --- a/docs/mobile-testing.md +++ b/docs/mobile-testing.md @@ -300,16 +300,33 @@ done ### Android audio focus interruptions -Voice mode uses the custom `expo-two-way-audio` Android module, so incoming calls and other system audio owners must be tested with emulator/system commands, not a JS-only test. To verify that voice resume handles denied audio focus without crashing: +Use native instrumentation for audio interruptions. JS-only tests cannot prove Android focus, +input silencing, or route recovery. The module's `android/src/androidTest` suite runs through +Gradle's `connectedDebugAndroidTest` task and uses real `AudioManager` and `AudioRecord` instances. +Run the microphone-privacy cases on Android 12 or newer. + +Verify these distinct outcomes: + +- Dictation continues when another media owner takes permanent, temporary, or ducking focus. +- Disabling microphone access pauses audio delivery without discarding the dictation. Enabling + access resumes it. Starting dictation while access is disabled waits for the same recovery. +- Returning from music controls resumes dictation without pausing music again. Cancelling a + backgrounded recording must prevent the lifecycle callback from restarting it. +- Two-way voice still stops coherently on focus loss. Keep the incoming-call check below: it + exercises denied focus during lifecycle resume, beyond an ordinary media takeover. ```bash -adb shell am start -n sh.paseo/.MainActivity -# Start voice mode in an existing composer, then background Paseo with Home. +# Start voice mode in an existing composer, then background Paseo. adb emu gsm call 5551234 -# Foreground Paseo while the call is still ringing. +# Foreground Paseo while the call is still ringing, then end the simulated call. +adb emu gsm cancel 5551234 ``` -Expected result: Paseo does not throw `RuntimeException: Audio focus request failed`; native audio reports an interruption and voice mode stops or pauses coherently. +Expected result: no `RuntimeException: Audio focus request failed`, and voice mode stops or pauses +coherently. Repeat with dictation: it must retain captured audio and recover microphone access. +Use browser/device automation to operate the app; the commands above only control emulator telephony. + +This fix requires a rebuilt native app. An OTA update on an older binary retains legacy capture. ### Releasing the audio session when idle diff --git a/packages/app/src/audio/audio-engine-types.ts b/packages/app/src/audio/audio-engine-types.ts index 90bec732fb..d96f9c5ef9 100644 --- a/packages/app/src/audio/audio-engine-types.ts +++ b/packages/app/src/audio/audio-engine-types.ts @@ -2,6 +2,7 @@ export interface AudioEngineCallbacks { onCaptureData(pcm: Uint8Array): void; onVolumeLevel(level: number): void; onInterruption?(): void; + onCaptureAvailabilityChange?(available: boolean): void; onError?(error: Error): void; } @@ -25,3 +26,8 @@ export interface AudioEngine { clearQueue(): void; isPlaying(): boolean; } + +export interface AudioEngineOptions { + traceLabel?: string; + captureMode?: "conversation" | "dictation"; +} diff --git a/packages/app/src/audio/audio-engine.native.ts b/packages/app/src/audio/audio-engine.native.ts index 86a6fc3d26..ed9f15a765 100644 --- a/packages/app/src/audio/audio-engine.native.ts +++ b/packages/app/src/audio/audio-engine.native.ts @@ -1,4 +1,9 @@ -import type { AudioEngine, AudioEngineCallbacks, AudioPlaybackSource } from "./audio-engine-types"; +import type { + AudioEngine, + AudioEngineCallbacks, + AudioEngineOptions, + AudioPlaybackSource, +} from "./audio-engine-types"; import { createAudioPlayer, setAudioModeAsync } from "expo-audio"; import { File, Paths } from "expo-file-system"; @@ -8,7 +13,7 @@ import { playPcm16 } from "./pcm"; export function createAudioEngine( callbacks: AudioEngineCallbacks, - _options?: { traceLabel?: string }, + options?: AudioEngineOptions, ): AudioEngine { const native = require("@getpaseo/expo-two-way-audio"); @@ -47,6 +52,12 @@ export function createAudioEngine( const interruptionSubscription = native.addExpoTwoWayAudioEventListener( "onAudioInterruption", (event: { data: string }) => { + if (event.data === "capture_paused" || event.data === "capture_resumed") { + if (refs.captureActive) { + callbacks.onCaptureAvailabilityChange?.(event.data === "capture_resumed"); + } + return; + } if (event.data !== "blocked") { return; } @@ -180,7 +191,10 @@ export function createAudioEngine( try { await ensureMicrophonePermission(); await ensureInitialized(); - const isRecording = native.toggleRecording(true); + const isRecording = + options?.captureMode === "dictation" + ? native.startDictation() + : native.toggleRecording(true); if (!isRecording) { throw new Error( "Microphone capture could not start because Android audio focus is unavailable.", @@ -201,6 +215,7 @@ export function createAudioEngine( refs.captureActive = false; refs.muted = false; callbacks.onVolumeLevel(0); + callbacks.onCaptureAvailabilityChange?.(true); releaseSessionIfIdle(); }, diff --git a/packages/app/src/audio/audio-engine.web.ts b/packages/app/src/audio/audio-engine.web.ts index dc0f049ca7..5516575e69 100644 --- a/packages/app/src/audio/audio-engine.web.ts +++ b/packages/app/src/audio/audio-engine.web.ts @@ -1,6 +1,11 @@ import { parsePcmSampleRate, resampleToPcm16 } from "./pcm"; import { isElectronRuntime } from "@/desktop/host"; -import type { AudioEngine, AudioEngineCallbacks, AudioPlaybackSource } from "./audio-engine-types"; +import type { + AudioEngine, + AudioEngineCallbacks, + AudioEngineOptions, + AudioPlaybackSource, +} from "./audio-engine-types"; import { createPlaybackQueue } from "./playback"; @@ -46,7 +51,7 @@ async function decodeAudioData(context: AudioContext, buffer: ArrayBuffer): Prom export function createAudioEngine( callbacks: AudioEngineCallbacks, - _options?: { traceLabel?: string }, + _options?: AudioEngineOptions, ): AudioEngine { const refs: { playbackContext: AudioContext | null; diff --git a/packages/app/src/components/dictation-controls.tsx b/packages/app/src/components/dictation-controls.tsx index 5c8a990987..2191f0d480 100644 --- a/packages/app/src/components/dictation-controls.tsx +++ b/packages/app/src/components/dictation-controls.tsx @@ -87,7 +87,9 @@ export function DictationControls({ - {formatDuration(duration)} + + {status === "paused" ? t("message.dictation.paused") : formatDuration(duration)} + {formatDuration(duration)} + {status === "paused" ? ( + + {t("message.dictation.paused")} + + ) : null} {isFailed ? ( {errorText diff --git a/packages/app/src/hooks/use-dictation-audio-source.native.ts b/packages/app/src/hooks/use-dictation-audio-source.native.ts index 293412fe1a..c34f9bbd02 100644 --- a/packages/app/src/hooks/use-dictation-audio-source.native.ts +++ b/packages/app/src/hooks/use-dictation-audio-source.native.ts @@ -14,6 +14,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic const onErrorRef = useRef(config.onError); const onInterruptionRef = useRef(config.onInterruption); const [volume, setVolume] = useState(0); + const [isPaused, setIsPaused] = useState(false); const engineRef = useRef | null>(null); const getOrCreateEngine = useCallback(() => { @@ -21,20 +22,24 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic return engineRef.current; } - engineRef.current = createAudioEngine({ - onCaptureData: (pcm) => { - onPcmSegmentRef.current(Buffer.from(pcm).toString("base64")); + engineRef.current = createAudioEngine( + { + onCaptureData: (pcm) => { + onPcmSegmentRef.current(Buffer.from(pcm).toString("base64")); + }, + onVolumeLevel: (level) => { + setVolume(level); + }, + onError: (error) => { + onErrorRef.current?.(error); + }, + onInterruption: () => { + onInterruptionRef.current?.(); + }, + onCaptureAvailabilityChange: (available) => setIsPaused(!available), }, - onVolumeLevel: (level) => { - setVolume(level); - }, - onError: (error) => { - onErrorRef.current?.(error); - }, - onInterruption: () => { - onInterruptionRef.current?.(); - }, - }); + { captureMode: "dictation" }, + ); return engineRef.current; }, []); @@ -67,5 +72,6 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic start, stop, volume, + isPaused, }; } diff --git a/packages/app/src/hooks/use-dictation-audio-source.types.ts b/packages/app/src/hooks/use-dictation-audio-source.types.ts index 711cf690c8..8fe82ef63e 100644 --- a/packages/app/src/hooks/use-dictation-audio-source.types.ts +++ b/packages/app/src/hooks/use-dictation-audio-source.types.ts @@ -8,4 +8,5 @@ export interface DictationAudioSource { start: () => Promise; stop: () => Promise; volume: number; + isPaused: boolean; } diff --git a/packages/app/src/hooks/use-dictation-audio-source.web.ts b/packages/app/src/hooks/use-dictation-audio-source.web.ts index c482bece18..6b71ca7181 100644 --- a/packages/app/src/hooks/use-dictation-audio-source.web.ts +++ b/packages/app/src/hooks/use-dictation-audio-source.web.ts @@ -442,6 +442,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic }, stop, volume, + isPaused: false, }), [start, stop, volume], ); diff --git a/packages/app/src/hooks/use-dictation.shared.ts b/packages/app/src/hooks/use-dictation.shared.ts index 275c9d93d7..fee3da46c1 100644 --- a/packages/app/src/hooks/use-dictation.shared.ts +++ b/packages/app/src/hooks/use-dictation.shared.ts @@ -1,6 +1,6 @@ import { i18n } from "@/i18n/i18next"; -export type DictationStatus = "idle" | "recording" | "uploading" | "failed"; +export type DictationStatus = "idle" | "recording" | "paused" | "uploading" | "failed"; export interface UseDictationOptions { client: import("@getpaseo/client/internal/daemon-client").DaemonClient | null; diff --git a/packages/app/src/hooks/use-dictation.ts b/packages/app/src/hooks/use-dictation.ts index dde72964bc..a2a3f46bcc 100644 --- a/packages/app/src/hooks/use-dictation.ts +++ b/packages/app/src/hooks/use-dictation.ts @@ -461,7 +461,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { volume: audio.volume, duration, error, - status, + status: status === "recording" && audio.isPaused ? "paused" : status, startDictation, cancelDictation, confirmDictation, diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index c5b9cf42d6..2f1fd7397f 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -360,6 +360,7 @@ export const ar: TranslationResources = { details: "تفاصيل", }, dictation: { + paused: "تمت مقاطعة الميكروفون. في انتظار الاستئناف…", start: "بدء الإملاء الصوتي", cancel: "إلغاء الإملاء", retry: "أعد محاولة الإملاء", diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index 76135fb6c9..14cde732e7 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -357,6 +357,7 @@ export const en = { details: "Details", }, dictation: { + paused: "Microphone interrupted. Waiting to resume…", start: "Start voice dictation", cancel: "Cancel dictation", retry: "Retry dictation", diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index d2572a4bab..18eb17f291 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -365,6 +365,7 @@ export const es: TranslationResources = { details: "Detalles", }, dictation: { + paused: "Micrófono interrumpido. Esperando para reanudar…", start: "Iniciar dictado de voz", cancel: "Cancelar dictado", retry: "Reintentar el dictado", diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index 48fd08c9ed..2bae7b8647 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -365,6 +365,7 @@ export const fr: TranslationResources = { details: "Détails", }, dictation: { + paused: "Microphone interrompu. En attente de reprise…", start: "Démarrer la dictée vocale", cancel: "Annuler la dictée", retry: "Relancer la dictée", diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index b3482f5c15..58e2706b95 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -365,6 +365,7 @@ export const ja: TranslationResources = { details: "詳細", }, dictation: { + paused: "マイクが中断されました。再開を待っています…", start: "音声入力を開始", cancel: "音声入力をキャンセル", retry: "音声入力を再試行", diff --git a/packages/app/src/i18n/resources/ko.ts b/packages/app/src/i18n/resources/ko.ts index e39d625253..fa962a5562 100644 --- a/packages/app/src/i18n/resources/ko.ts +++ b/packages/app/src/i18n/resources/ko.ts @@ -361,6 +361,7 @@ export const ko: TranslationResources = { details: "세부 정보", }, dictation: { + paused: "마이크가 중단되었습니다. 재개를 기다리는 중…", start: "음성 받아쓰기 시작", cancel: "받아쓰기 취소", retry: "받아쓰기 다시 시도", diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index 8c3d93499a..c5c6607ca6 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -364,6 +364,7 @@ export const ptBR: TranslationResources = { details: "Detalhes", }, dictation: { + paused: "Microfone interrompido. Aguardando para retomar…", start: "Iniciar ditado por voz", cancel: "Cancelar ditado", retry: "Tentar ditado novamente", diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index df57855cb6..4df3f721bb 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -364,6 +364,7 @@ export const ru: TranslationResources = { details: "Подробности", }, dictation: { + paused: "Микрофон прерван. Ожидание возобновления…", start: "Начать диктовку голосом", cancel: "Отменить диктовку", retry: "Повторить диктовку", diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index 984cc0d71a..bad9cc4690 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -360,6 +360,7 @@ export const zhCN: TranslationResources = { details: "详情", }, dictation: { + paused: "麦克风已中断，正在等待恢复…", start: "开始语音听写", cancel: "取消听写", retry: "重试听写", diff --git a/packages/expo-two-way-audio/android/build.gradle b/packages/expo-two-way-audio/android/build.gradle index 84a7837cf9..5bb7508992 100644 --- a/packages/expo-two-way-audio/android/build.gradle +++ b/packages/expo-two-way-audio/android/build.gradle @@ -36,8 +36,14 @@ android { defaultConfig { versionCode 1 versionName "0.1.0" + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" } lintOptions { abortOnError false } } + +dependencies { + androidTestImplementation 'androidx.test:runner:1.6.2' + androidTestImplementation 'androidx.test.ext:junit:1.2.1' +} diff --git a/packages/expo-two-way-audio/android/src/androidTest/AndroidManifest.xml b/packages/expo-two-way-audio/android/src/androidTest/AndroidManifest.xml new file mode 100644 index 0000000000..6501d613de --- /dev/null +++ b/packages/expo-two-way-audio/android/src/androidTest/AndroidManifest.xml @@ -0,0 +1,8 @@ + + + + + + + diff --git a/packages/expo-two-way-audio/android/src/androidTest/java/expo/modules/twowayaudio/AudioEngineTest.kt b/packages/expo-two-way-audio/android/src/androidTest/java/expo/modules/twowayaudio/AudioEngineTest.kt new file mode 100644 index 0000000000..ad840deb20 --- /dev/null +++ b/packages/expo-two-way-audio/android/src/androidTest/java/expo/modules/twowayaudio/AudioEngineTest.kt @@ -0,0 +1,223 @@ +package expo.modules.twowayaudio + +import AudioEngine +import android.app.Activity +import android.content.Context +import android.content.Intent +import android.media.AudioAttributes +import android.media.AudioFocusRequest +import android.media.AudioManager +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.filters.SdkSuppress +import org.junit.After +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.atomic.AtomicInteger + +class AudioEngineTestActivity : Activity() + +/** Real AudioManager and AudioRecord: focus loss is delivered by Android, never injected. */ +@RunWith(AndroidJUnit4::class) +class AudioEngineTest { + private val instrumentation = InstrumentationRegistry.getInstrumentation() + private lateinit var activity: Activity + private lateinit var engine: AudioEngine + private lateinit var audio: AudioManager + private val packets = AtomicInteger() + private val events = CopyOnWriteArrayList() + private val requests = mutableListOf() + + @Before fun setUp() { + val context = instrumentation.context + shell("pm grant ${context.packageName} android.permission.RECORD_AUDIO") + activity = instrumentation.startActivitySync( + Intent(context, AudioEngineTestActivity::class.java).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + ) + instrumentation.waitForIdleSync() + audio = activity.getSystemService(Context.AUDIO_SERVICE) as AudioManager + main { + engine = AudioEngine(activity) + engine.onMicDataCallback = { packets.incrementAndGet() } + engine.onAudioInterruptionCallback = { events.add(it) } + } + } + + @After fun tearDown() { + shell("cmd sensor_privacy disable 0 microphone") + main { + requests.forEach { audio.abandonAudioFocusRequest(it) } + engine.tearDown() + activity.finish() + } + } + + @Test fun dictationContinuesWhenMusicTakesFocus() { + assertDictationSurvives(AudioManager.AUDIOFOCUS_GAIN) + } + + @Test fun dictationContinuesWhenFocusIsTemporarilyLost() { + assertDictationSurvives(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT) + } + + @Test fun dictationContinuesWhenAnotherAppRequestsDucking() { + assertDictationSurvives(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK) + } + + @Test fun conversationStillStopsOnFocusLoss() { + main { assertTrue(engine.toggleRecording(true)) } + await("conversation microphone data") { packets.get() > 2 } + takeFocus(AudioManager.AUDIOFOCUS_GAIN) + await("terminal conversation interruption") { events.contains("blocked") } + assertFalse(engine.isRecording) + } + + @Test fun dictationStartsWhileMusicOwnsFocusAndCanBeStartedAgain() { + takeFocus(AudioManager.AUDIOFOCUS_GAIN) + startDictation() + main { engine.toggleRecording(false); engine.releaseAudioSession() } + val stoppedAt = packets.get() + startDictation() + await("new recording data") { packets.get() > stoppedAt + 2 } + assertFalse(events.contains("blocked")) + } + + @Test fun cancellingWhileBackgroundedDoesNotRestartOnResume() { + startDictation() + main { + engine.pauseRecordingAndPlayer() + engine.toggleRecording(false) + engine.releaseAudioSession() + engine.resumeRecordingAndPlayer() + } + assertFalse(engine.isRecording) + } + + @Test fun dictationKeepsItsCaptureModeAfterBackgroundResume() { + startDictation() + main { engine.pauseRecordingAndPlayer(); engine.resumeRecordingAndPlayer() } + val resumedAt = packets.get() + takeFocus(AudioManager.AUDIOFOCUS_GAIN) + await("resumed dictation data after focus loss") { packets.get() > resumedAt + 3 } + assertTrue(engine.isRecording) + assertFalse(events.contains("blocked")) + } + + @Test @SdkSuppress(minSdkVersion = 31) + fun microphonePrivacyInterruptionPreservesCaptureAndRecovers() { + startDictation() + shell("cmd sensor_privacy enable 0 microphone") + await("input paused") { events.contains("capture_paused") } + assertTrue(engine.isRecording) + assertFalse(events.contains("blocked")) + // Once Android reports silencing, do not enqueue artificial silence for transcription. + SystemClock.sleep(100) + val pausedAt = packets.get() + SystemClock.sleep(150) + assertEquals(pausedAt, packets.get()) + shell("cmd sensor_privacy disable 0 microphone") + await("input resumed") { events.contains("capture_resumed") } + await("microphone data after recovery") { packets.get() > pausedAt + 3 } + assertTrue(engine.isRecording) + assertEquals(listOf("capture_paused", "capture_resumed"), events.toList()) + } + + @Test @SdkSuppress(minSdkVersion = 31) + fun startingWithMicrophonePrivacyEnabledWaitsAndRecovers() { + shell("cmd sensor_privacy enable 0 microphone") + main { assertTrue(engine.startDictation()) } + await("initial input paused") { events.contains("capture_paused") } + assertTrue(engine.isRecording) + assertFalse(events.contains("blocked")) + shell("cmd sensor_privacy disable 0 microphone") + await("initial input resumed") { events.contains("capture_resumed") } + await("microphone data") { packets.get() > 3 } + } + + @Test fun resumingDictationDoesNotTakeFocusBackFromMusic() { + startDictation() + main { engine.pauseRecordingAndPlayer() } + val changes = CopyOnWriteArrayList() + val request = AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN) + .setOnAudioFocusChangeListener({ changes.add(it) }, Handler(Looper.getMainLooper())) + .build() + requests.add(request) + main { assertEquals(AudioManager.AUDIOFOCUS_REQUEST_GRANTED, audio.requestAudioFocus(request)) } + val before = packets.get() + main { engine.resumeRecordingAndPlayer() } + await("resumed microphone data") { packets.get() > before + 3 } + assertTrue(engine.isRecording) + assertTrue("Music lost focus on dictation resume: $changes", changes.isEmpty()) + } + + @Test @SdkSuppress(minSdkVersion = 31) + fun stoppingDictationReleasesTheCommunicationRoute() { + startDictation() + await("selected communication route") { hasCommunicationRoute() } + main { engine.toggleRecording(false); engine.releaseAudioSession() } + await("normal audio mode") { audio.mode == AudioManager.MODE_NORMAL } + await("cleared communication route") { !hasCommunicationRoute() } + assertFalse(engine.isRecording) + } + + private fun assertDictationSurvives(gain: Int) { + startDictation() + val before = packets.get() + val request = takeFocus(gain) + await("microphone data while focus is lost") { packets.get() > before + 3 } + assertTrue(engine.isRecording) + assertFalse(events.contains("blocked")) + val during = packets.get() + main { audio.abandonAudioFocusRequest(request) } + await("microphone data after focus release") { packets.get() > during + 3 } + assertTrue(engine.isRecording) + assertFalse(events.contains("blocked")) + } + + private fun startDictation() { + val before = packets.get() + main { assertTrue(engine.startDictation()) } + await("initial microphone data") { packets.get() > before + 3 } + } + + private fun takeFocus(gain: Int): AudioFocusRequest { + val request = AudioFocusRequest.Builder(gain) + .setAudioAttributes(AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA) + .setContentType(AudioAttributes.CONTENT_TYPE_MUSIC).build()) + .setOnAudioFocusChangeListener({}, Handler(Looper.getMainLooper())) + .build() + requests.add(request) + main { assertEquals(AudioManager.AUDIOFOCUS_REQUEST_GRANTED, audio.requestAudioFocus(request)) } + return request + } + + private fun main(action: () -> Unit) = instrumentation.runOnMainSync(action) + + private fun await(description: String, condition: () -> Boolean) { + val deadline = SystemClock.elapsedRealtime() + 5000 + while (!condition() && SystemClock.elapsedRealtime() < deadline) SystemClock.sleep(20) + assertTrue("Timed out waiting for $description; events=$events packets=${packets.get()}", condition()) + } + + // communicationDevice reports the effective system device even after our selection is cleared. + // Inspect only current route clients, never the historical routing event log. + private fun hasCommunicationRoute(): Boolean { + val dump = shell("dumpsys audio") + assertTrue("Missing Android communication route diagnostics", dump.contains("Communication route clients:")) + return dump.lineSequence().any { + it.trimStart().startsWith("[CommunicationRouteClient: mUid: ${android.os.Process.myUid()} ") + } + } + + private fun shell(command: String): String = + instrumentation.uiAutomation.executeShellCommand(command).use { + android.os.ParcelFileDescriptor.AutoCloseInputStream(it).readBytes().toString(Charsets.UTF_8) + } +} diff --git a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt index a6fe25303e..48aff1f1df 100644 --- a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt +++ b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt @@ -6,12 +6,14 @@ import android.media.AudioFocusRequest import android.media.AudioFormat import android.media.AudioManager import android.media.AudioRecord +import android.media.AudioRecordingConfiguration import android.media.AudioTrack import android.media.MediaRecorder import android.media.audiofx.AcousticEchoCanceler import android.media.audiofx.NoiseSuppressor import android.os.Build -import android.os.PowerManager +import android.os.Handler +import android.os.Looper import android.util.Log import androidx.annotation.RequiresApi import java.util.LinkedList @@ -34,7 +36,26 @@ class AudioEngine (context: Context) { private var noiseSuppressor: NoiseSuppressor? = null private val executorServiceMicrophone = Executors.newFixedThreadPool(1) private val executorServicePlayback = Executors.newFixedThreadPool(1) - private var speakerDevice: AudioDeviceInfo? = null + private enum class CaptureMode { CONVERSATION, DICTATION } + private var captureMode = CaptureMode.CONVERSATION + private val mainHandler = Handler(Looper.getMainLooper()) + @Volatile private var captureSilenced = false + private val recordingCallback = object : AudioManager.AudioRecordingCallback() { + override fun onRecordingConfigChanged(configs: MutableList) { + if (!isRecording || !::audioRecord.isInitialized) return + val config = configs.firstOrNull { it.clientAudioSessionId == audioRecord.audioSessionId } + ?: return + setCaptureSilenced(config.isClientSilenced) + } + } + private val deviceCallback = object : android.media.AudioDeviceCallback() { + override fun onAudioDevicesAdded(devices: Array) { + if (communicationRouteActive) updateAudioRouting() + } + override fun onAudioDevicesRemoved(devices: Array) { + if (communicationRouteActive) updateAudioRouting() + } + } private var communicationRouteActive = false private var bridgeWindowStartedAtMs = System.currentTimeMillis() private var micEvents = 0 @@ -44,7 +65,7 @@ class AudioEngine (context: Context) { private var playbackWrites = 0 private var playbackWriteBytes = 0L - var isRecording = false + @Volatile var isRecording = false private var isRecordingBeforePause = false var isPlaying = false @@ -83,28 +104,9 @@ class AudioEngine (context: Context) { @SuppressLint("NewApi") private fun initializeAudio(context:Context) { audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager - activateCommunicationRoute() - if (!requestAudioFocus()) { - handleAudioFocusBlocked() - } - - // Listen for changes in audio routing - audioManager.registerAudioDeviceCallback(object:android.media.AudioDeviceCallback(){ - override fun onAudioDevicesAdded(addedDevices: Array?) { - Log.d("AudioEngine", "onAudioDevicesAdded") - super.onAudioDevicesAdded(addedDevices) - if (communicationRouteActive) { - updateAudioRouting() - } - } - override fun onAudioDevicesRemoved(removedDevices: Array?) { - Log.d("AudioEngine", "onAudioDevicesRemoved") - super.onAudioDevicesRemoved(removedDevices) - if (communicationRouteActive) { - updateAudioRouting() - } - } - }, null) + // Initialization must not interrupt music before a capture/playback operation owns a session. + audioManager.registerAudioDeviceCallback(deviceCallback, mainHandler) + audioManager.registerAudioRecordingCallback(recordingCallback, mainHandler) val bufferSize = AudioTrack.getMinBufferSize( SAMPLE_RATE, @@ -125,49 +127,42 @@ class AudioEngine (context: Context) { bufferSize, AudioTrack.MODE_STREAM, audioManager.generateAudioSessionId() - ).apply { - play() - } + ) } + @SuppressLint("NewApi") private fun updateAudioRouting() { - val devices = audioManager.getDevices(AudioManager.GET_DEVICES_OUTPUTS) - var isExternalDeviceConnected = false - var selectedDevice: AudioDeviceInfo? = null - - for (device in devices) { - if (device.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER) { - speakerDevice = device - } - if (device.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES || - device.type == AudioDeviceInfo.TYPE_WIRED_HEADSET || - device.type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP || - device.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO) { - isExternalDeviceConnected = true - selectedDevice = device - break - } else if (device.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER) { - selectedDevice = device - } - } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - // Use the modern API for Android S and above - try { - selectedDevice?.let { - audioManager.setCommunicationDevice(it) - } - }catch (e:Exception){ - Log.e("AudioEngine", "Error setting communication device. Using speaker") - speakerDevice?.let { - audioManager.setCommunicationDevice(it) - } + val devices = audioManager.availableCommunicationDevices + // A2DP is a music output, not a microphone route. Android chooses the matching + // input for a valid communication sink (including SCO and BLE headsets). + val selected = devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO || + it.type == AudioDeviceInfo.TYPE_BLE_HEADSET } + ?: devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_WIRED_HEADSET || + it.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES || + it.type == AudioDeviceInfo.TYPE_USB_HEADSET } + ?: devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER } + val routed = try { + selected != null && audioManager.setCommunicationDevice(selected) + } catch (_: IllegalArgumentException) { + // The device disconnected between enumeration and selection. + false } - + if (!routed) audioManager.clearCommunicationDevice() } else { - // Fall back to deprecated method for older Android versions @Suppress("DEPRECATION") - audioManager.isSpeakerphoneOn = !isExternalDeviceConnected + val devices = audioManager.getDevices(AudioManager.GET_DEVICES_OUTPUTS) + // Before Android 12, A2DP signals a connected headset; start SCO to open its mic. + val bluetooth = devices.any { it.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO || + it.type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP } + @Suppress("DEPRECATION") + if (bluetooth) audioManager.startBluetoothSco() else audioManager.stopBluetoothSco() + @Suppress("DEPRECATION") + audioManager.isSpeakerphoneOn = !bluetooth && !devices.any { + it.type == AudioDeviceInfo.TYPE_WIRED_HEADSET || + it.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES || + it.type == AudioDeviceInfo.TYPE_USB_HEADSET + } } } @@ -186,6 +181,8 @@ class AudioEngine (context: Context) { } else { @Suppress("DEPRECATION") audioManager.isSpeakerphoneOn = false + @Suppress("DEPRECATION") + audioManager.stopBluetoothSco() } audioManager.mode = AudioManager.MODE_NORMAL } @@ -245,8 +242,8 @@ class AudioEngine (context: Context) { audioFocusRequest = null } - val focusRequest = - AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE) + lateinit var focusRequest: AudioFocusRequest + focusRequest = AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE) .setAudioAttributes( AudioAttributes.Builder() .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) @@ -255,6 +252,7 @@ class AudioEngine (context: Context) { ) .setAcceptsDelayedFocusGain(true) .setOnAudioFocusChangeListener { focusChange -> + if (audioFocusRequest !== focusRequest) return@setOnAudioFocusChangeListener when (focusChange) { AudioManager.AUDIOFOCUS_GAIN -> { Log.d("AudioEngine", "Audio focus gained") @@ -286,6 +284,14 @@ class AudioEngine (context: Context) { } private fun handleAudioFocusBlocked() { + // Focus governs playback. Dictation keeps its AudioRecord alive; Android's + // recording configuration reports real input silencing and restores it after calls. + if (captureMode == CaptureMode.DICTATION) { + stopPlayback() + audioFocusRequest?.let { audioManager.abandonAudioFocusRequest(it) } + audioFocusRequest = null + return + } if (isRecording) { stopRecording() } @@ -306,11 +312,11 @@ class AudioEngine (context: Context) { @RequiresApi(Build.VERSION_CODES.Q) @SuppressLint("MissingPermission") - private fun startRecording(): Boolean { + private fun startRecording(requestFocus: Boolean = true): Boolean { activateCommunicationRoute() - if (!requestAudioFocus()) { + if (requestFocus && !requestAudioFocus()) { handleAudioFocusBlocked() - return false + if (captureMode == CaptureMode.CONVERSATION) return false } val bufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT) @@ -344,17 +350,27 @@ class AudioEngine (context: Context) { audioRecord.startRecording() isRecording = true - startMicSampleTap() + setCaptureSilenced(audioRecord.activeRecordingConfiguration?.isClientSilenced ?: false) + startMicSampleTap(audioRecord) return true } - private fun startMicSampleTap(){ + private fun setCaptureSilenced(silenced: Boolean) { + if (captureSilenced == silenced) return + captureSilenced = silenced + if (silenced) onInputVolumeCallback?.invoke(0.0F) + onAudioInterruptionCallback?.invoke(if (silenced) "capture_paused" else "capture_resumed") + } + + private fun startMicSampleTap(recorder: AudioRecord){ executorServiceMicrophone.execute { val buffer = ByteArray(1024) try { - while (isRecording) { - val read = audioRecord.read(buffer, 0, buffer.size) - if (read > 0) { + while (isRecording && audioRecord === recorder) { + val read = recorder.read(buffer, 0, buffer.size) + if (!isRecording || audioRecord !== recorder) break + if (read < 0) throw IllegalStateException("AudioRecord read failed: $read") + if (read > 0 && !captureSilenced) { val data = buffer.copyOf(read) micEvents += 1 micBytes += data.size.toLong() @@ -367,9 +383,13 @@ class AudioEngine (context: Context) { Log.d("AudioEngine", "Mic sample tap stopped.") }catch (e: Exception){ Log.e("AudioEngine", "Error reading mic sample data", e) - isRecording = false - tearDown() - throw e + mainHandler.post { + if (isRecording && audioRecord === recorder) { + stopRecording() + releaseAudioSession() + onAudioInterruptionCallback?.invoke("blocked") + } + } } } } @@ -379,16 +399,31 @@ class AudioEngine (context: Context) { isRecording = false if (audioRecord.recordingState == AudioRecord.RECORDSTATE_RECORDING) { audioRecord.stop() - audioRecord.release() } + audioRecord.release() + echoCanceler?.release() + echoCanceler = null + noiseSuppressor?.release() + noiseSuppressor = null + setCaptureSilenced(false) onInputVolumeCallback?.invoke(0.0F) } + @RequiresApi(Build.VERSION_CODES.Q) + fun startDictation(): Boolean { + isRecordingBeforePause = false + if (isRecording) return true + captureMode = CaptureMode.DICTATION + return startRecording() + } + @RequiresApi(Build.VERSION_CODES.Q) fun toggleRecording(value: Boolean): Boolean { + isRecordingBeforePause = false if (value == isRecording) return isRecording if (value) { + captureMode = CaptureMode.CONVERSATION return startRecording() } else { stopRecording() @@ -401,6 +436,7 @@ class AudioEngine (context: Context) { @SuppressLint("NewApi") fun playPCMData(data: ByteArray) { acquireAudioSessionIfNeeded() + audioTrack.play() audioSampleQueue.add(data) playbackEvents += 1 playbackQueuedBytes += data.size.toLong() @@ -464,7 +500,7 @@ class AudioEngine (context: Context) { @RequiresApi(Build.VERSION_CODES.Q) fun pauseRecordingAndPlayer() { isRecordingBeforePause = isRecording - isRecording = toggleRecording(false) + stopRecording() audioTrack.pause() } @@ -478,16 +514,18 @@ class AudioEngine (context: Context) { if (!wasRecordingBeforePause && !isPlaying) { return } - activateCommunicationRoute() - if (!requestAudioFocus()) { - handleAudioFocusBlocked() - return + isRecordingBeforePause = false + if (wasRecordingBeforePause) { + // Returning from music controls must not pause music the user explicitly resumed. + if (!startRecording(requestFocus = captureMode == CaptureMode.CONVERSATION)) return + } else { + activateCommunicationRoute() + if (!requestAudioFocus()) { + handleAudioFocusBlocked() + return + } } - isRecording = toggleRecording(wasRecordingBeforePause) - if (wasRecordingBeforePause && !isRecording) { - return - } - audioTrack.play() + if (captureMode == CaptureMode.CONVERSATION) audioTrack.play() } fun stopPlayback() { @@ -520,7 +558,11 @@ class AudioEngine (context: Context) { audioManager.abandonAudioFocusRequest(request) } audioFocusRequest = null + audioManager.unregisterAudioDeviceCallback(deviceCallback) + audioManager.unregisterAudioRecordingCallback(recordingCallback) + audioTrack.release() executorServiceMicrophone.shutdownNow() + executorServicePlayback.shutdownNow() } diff --git a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/ExpoTwoWayAudioModule.kt b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/ExpoTwoWayAudioModule.kt index f9b107c99f..68f1489de7 100644 --- a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/ExpoTwoWayAudioModule.kt +++ b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/ExpoTwoWayAudioModule.kt @@ -42,6 +42,12 @@ class ExpoTwoWayAudioModule : Module() { audioEngine?.isRecording ?: false } + Function("startDictation") { + val isRecording = audioEngine?.startDictation() ?: false + sendEvent(ON_RECORDING_CHANGE_EVENT, mapOf("data" to isRecording)) + isRecording + } + Function("toggleRecording") { value: Boolean -> audioEngine?.let { engine -> val isRecording = engine.toggleRecording(value) diff --git a/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift b/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift index baec747995..046f856747 100644 --- a/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift +++ b/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift @@ -59,6 +59,12 @@ public class ExpoTwoWayAudioModule: Module { return audioEngine.isRecording } + Function("startDictation") { () -> Bool in + let isRecording = self.audioEngine?.toggleRecording(true) ?? false + self.sendEvent(ON_RECORDING_CHANGE_EVENT_NAME, ["data": isRecording]) + return isRecording + } + Function("toggleRecording") { (val: Bool) -> Bool in guard let audioEngine = self.audioEngine else { print("AudioEngine not initialized") diff --git a/packages/expo-two-way-audio/src/core.ts b/packages/expo-two-way-audio/src/core.ts index 1a54e0e9ce..48215f2bae 100644 --- a/packages/expo-two-way-audio/src/core.ts +++ b/packages/expo-two-way-audio/src/core.ts @@ -17,6 +17,16 @@ export function toggleRecording(val: boolean): boolean { return ExpoTwoWayAudioModule.toggleRecording(val); } +/** Capture speech without treating another app's playback focus as a microphone failure. */ +export function startDictation(): boolean { + // COMPAT(dictationCapture): added after v0.11.2, remove after 2027-01-16 once + // the native binary floor includes startDictation. Older OTA clients retain legacy capture. + if (typeof ExpoTwoWayAudioModule.startDictation !== "function") { + return ExpoTwoWayAudioModule.toggleRecording(true); + } + return ExpoTwoWayAudioModule.startDictation(); +} + export function isRecording(): boolean { return ExpoTwoWayAudioModule.isRecording(); }