diff --git a/docs/mobile-testing.md b/docs/mobile-testing.md index 93fa35aedd..375da30708 100644 --- a/docs/mobile-testing.md +++ b/docs/mobile-testing.md @@ -300,16 +300,33 @@ done ### Android audio focus interruptions -Voice mode uses the custom `expo-two-way-audio` Android module, so incoming calls and other system audio owners must be tested with emulator/system commands, not a JS-only test. To verify that voice resume handles denied audio focus without crashing: +Use native instrumentation for audio interruptions. JS-only tests cannot prove Android focus, +input silencing, or route recovery. The module's `android/src/androidTest` suite runs through +Gradle's `connectedDebugAndroidTest` task and uses real `AudioManager` and `AudioRecord` instances. +Run the microphone-privacy cases on Android 12 or newer. + +Verify these distinct outcomes: + +- Dictation continues when another media owner takes permanent, temporary, or ducking focus. +- Disabling microphone access pauses audio delivery without discarding the dictation. Enabling + access resumes it. Starting dictation while access is disabled waits for the same recovery. +- Returning from music controls resumes dictation without pausing music again. Cancelling a + backgrounded recording must prevent the lifecycle callback from restarting it. +- Two-way voice still stops coherently on focus loss. Keep the incoming-call check below: it + exercises denied focus during lifecycle resume, beyond an ordinary media takeover. ```bash -adb shell am start -n sh.paseo/.MainActivity -# Start voice mode in an existing composer, then background Paseo with Home. +# Start voice mode in an existing composer, then background Paseo. adb emu gsm call 5551234 -# Foreground Paseo while the call is still ringing. +# Foreground Paseo while the call is still ringing, then end the simulated call. +adb emu gsm cancel 5551234 ``` -Expected result: Paseo does not throw `RuntimeException: Audio focus request failed`; native audio reports an interruption and voice mode stops or pauses coherently. +Expected result: no `RuntimeException: Audio focus request failed`, and voice mode stops or pauses +coherently. Repeat with dictation: it must retain captured audio and recover microphone access. +Use browser/device automation to operate the app; the commands above only control emulator telephony. + +This fix requires a rebuilt native app. An OTA update on an older binary retains legacy capture. ### Releasing the audio session when idle diff --git a/packages/app/src/audio/audio-engine-types.ts b/packages/app/src/audio/audio-engine-types.ts index 90bec732fb..d96f9c5ef9 100644 --- a/packages/app/src/audio/audio-engine-types.ts +++ b/packages/app/src/audio/audio-engine-types.ts @@ -2,6 +2,7 @@ export interface AudioEngineCallbacks { onCaptureData(pcm: Uint8Array): void; onVolumeLevel(level: number): void; onInterruption?(): void; + onCaptureAvailabilityChange?(available: boolean): void; onError?(error: Error): void; } @@ -25,3 +26,8 @@ export interface AudioEngine { clearQueue(): void; isPlaying(): boolean; } + +export interface AudioEngineOptions { + traceLabel?: string; + captureMode?: "conversation" | "dictation"; +} diff --git a/packages/app/src/audio/audio-engine.native.ts b/packages/app/src/audio/audio-engine.native.ts index 86a6fc3d26..ed9f15a765 100644 --- a/packages/app/src/audio/audio-engine.native.ts +++ b/packages/app/src/audio/audio-engine.native.ts @@ -1,4 +1,9 @@ -import type { AudioEngine, AudioEngineCallbacks, AudioPlaybackSource } from "./audio-engine-types"; +import type { + AudioEngine, + AudioEngineCallbacks, + AudioEngineOptions, + AudioPlaybackSource, +} from "./audio-engine-types"; import { createAudioPlayer, setAudioModeAsync } from "expo-audio"; import { File, Paths } from "expo-file-system"; @@ -8,7 +13,7 @@ import { playPcm16 } from "./pcm"; export function createAudioEngine( callbacks: AudioEngineCallbacks, - _options?: { traceLabel?: string }, + options?: AudioEngineOptions, ): AudioEngine { const native = require("@getpaseo/expo-two-way-audio"); @@ -47,6 +52,12 @@ export function createAudioEngine( const interruptionSubscription = native.addExpoTwoWayAudioEventListener( "onAudioInterruption", (event: { data: string }) => { + if (event.data === "capture_paused" || event.data === "capture_resumed") { + if (refs.captureActive) { + callbacks.onCaptureAvailabilityChange?.(event.data === "capture_resumed"); + } + return; + } if (event.data !== "blocked") { return; } @@ -180,7 +191,10 @@ export function createAudioEngine( try { await ensureMicrophonePermission(); await ensureInitialized(); - const isRecording = native.toggleRecording(true); + const isRecording = + options?.captureMode === "dictation" + ? native.startDictation() + : native.toggleRecording(true); if (!isRecording) { throw new Error( "Microphone capture could not start because Android audio focus is unavailable.", @@ -201,6 +215,7 @@ export function createAudioEngine( refs.captureActive = false; refs.muted = false; callbacks.onVolumeLevel(0); + callbacks.onCaptureAvailabilityChange?.(true); releaseSessionIfIdle(); }, diff --git a/packages/app/src/audio/audio-engine.web.ts b/packages/app/src/audio/audio-engine.web.ts index dc0f049ca7..5516575e69 100644 --- a/packages/app/src/audio/audio-engine.web.ts +++ b/packages/app/src/audio/audio-engine.web.ts @@ -1,6 +1,11 @@ import { parsePcmSampleRate, resampleToPcm16 } from "./pcm"; import { isElectronRuntime } from "@/desktop/host"; -import type { AudioEngine, AudioEngineCallbacks, AudioPlaybackSource } from "./audio-engine-types"; +import type { + AudioEngine, + AudioEngineCallbacks, + AudioEngineOptions, + AudioPlaybackSource, +} from "./audio-engine-types"; import { createPlaybackQueue } from "./playback"; @@ -46,7 +51,7 @@ async function decodeAudioData(context: AudioContext, buffer: ArrayBuffer): Prom export function createAudioEngine( callbacks: AudioEngineCallbacks, - _options?: { traceLabel?: string }, + _options?: AudioEngineOptions, ): AudioEngine { const refs: { playbackContext: AudioContext | null; diff --git a/packages/app/src/components/dictation-controls.tsx b/packages/app/src/components/dictation-controls.tsx index 5c8a990987..2191f0d480 100644 --- a/packages/app/src/components/dictation-controls.tsx +++ b/packages/app/src/components/dictation-controls.tsx @@ -87,7 +87,9 @@ export function DictationControls({ - {formatDuration(duration)} + + {status === "paused" ? t("message.dictation.paused") : formatDuration(duration)} + {formatDuration(duration)} + {status === "paused" ? ( + + {t("message.dictation.paused")} + + ) : null} {isFailed ? ( {errorText diff --git a/packages/app/src/hooks/use-dictation-audio-source.native.ts b/packages/app/src/hooks/use-dictation-audio-source.native.ts index 293412fe1a..c34f9bbd02 100644 --- a/packages/app/src/hooks/use-dictation-audio-source.native.ts +++ b/packages/app/src/hooks/use-dictation-audio-source.native.ts @@ -14,6 +14,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic const onErrorRef = useRef(config.onError); const onInterruptionRef = useRef(config.onInterruption); const [volume, setVolume] = useState(0); + const [isPaused, setIsPaused] = useState(false); const engineRef = useRef | null>(null); const getOrCreateEngine = useCallback(() => { @@ -21,20 +22,24 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic return engineRef.current; } - engineRef.current = createAudioEngine({ - onCaptureData: (pcm) => { - onPcmSegmentRef.current(Buffer.from(pcm).toString("base64")); + engineRef.current = createAudioEngine( + { + onCaptureData: (pcm) => { + onPcmSegmentRef.current(Buffer.from(pcm).toString("base64")); + }, + onVolumeLevel: (level) => { + setVolume(level); + }, + onError: (error) => { + onErrorRef.current?.(error); + }, + onInterruption: () => { + onInterruptionRef.current?.(); + }, + onCaptureAvailabilityChange: (available) => setIsPaused(!available), }, - onVolumeLevel: (level) => { - setVolume(level); - }, - onError: (error) => { - onErrorRef.current?.(error); - }, - onInterruption: () => { - onInterruptionRef.current?.(); - }, - }); + { captureMode: "dictation" }, + ); return engineRef.current; }, []); @@ -67,5 +72,6 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic start, stop, volume, + isPaused, }; } diff --git a/packages/app/src/hooks/use-dictation-audio-source.types.ts b/packages/app/src/hooks/use-dictation-audio-source.types.ts index 711cf690c8..8fe82ef63e 100644 --- a/packages/app/src/hooks/use-dictation-audio-source.types.ts +++ b/packages/app/src/hooks/use-dictation-audio-source.types.ts @@ -8,4 +8,5 @@ export interface DictationAudioSource { start: () => Promise; stop: () => Promise; volume: number; + isPaused: boolean; } diff --git a/packages/app/src/hooks/use-dictation-audio-source.web.ts b/packages/app/src/hooks/use-dictation-audio-source.web.ts index c482bece18..6b71ca7181 100644 --- a/packages/app/src/hooks/use-dictation-audio-source.web.ts +++ b/packages/app/src/hooks/use-dictation-audio-source.web.ts @@ -442,6 +442,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic }, stop, volume, + isPaused: false, }), [start, stop, volume], ); diff --git a/packages/app/src/hooks/use-dictation.shared.ts b/packages/app/src/hooks/use-dictation.shared.ts index 275c9d93d7..fee3da46c1 100644 --- a/packages/app/src/hooks/use-dictation.shared.ts +++ b/packages/app/src/hooks/use-dictation.shared.ts @@ -1,6 +1,6 @@ import { i18n } from "@/i18n/i18next"; -export type DictationStatus = "idle" | "recording" | "uploading" | "failed"; +export type DictationStatus = "idle" | "recording" | "paused" | "uploading" | "failed"; export interface UseDictationOptions { client: import("@getpaseo/client/internal/daemon-client").DaemonClient | null; diff --git a/packages/app/src/hooks/use-dictation.ts b/packages/app/src/hooks/use-dictation.ts index dde72964bc..a2a3f46bcc 100644 --- a/packages/app/src/hooks/use-dictation.ts +++ b/packages/app/src/hooks/use-dictation.ts @@ -461,7 +461,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { volume: audio.volume, duration, error, - status, + status: status === "recording" && audio.isPaused ? "paused" : status, startDictation, cancelDictation, confirmDictation, diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index c5b9cf42d6..2f1fd7397f 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -360,6 +360,7 @@ export const ar: TranslationResources = { details: "تفاصيل", }, dictation: { + paused: "تمت مقاطعة الميكروفون. في انتظار الاستئناف…", start: "بدء الإملاء الصوتي", cancel: "إلغاء الإملاء", retry: "أعد محاولة الإملاء", diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index 76135fb6c9..14cde732e7 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -357,6 +357,7 @@ export const en = { details: "Details", }, dictation: { + paused: "Microphone interrupted. Waiting to resume…", start: "Start voice dictation", cancel: "Cancel dictation", retry: "Retry dictation", diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index d2572a4bab..18eb17f291 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -365,6 +365,7 @@ export const es: TranslationResources = { details: "Detalles", }, dictation: { + paused: "Micrófono interrumpido. Esperando para reanudar…", start: "Iniciar dictado de voz", cancel: "Cancelar dictado", retry: "Reintentar el dictado", diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index 48fd08c9ed..2bae7b8647 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -365,6 +365,7 @@ export const fr: TranslationResources = { details: "Détails", }, dictation: { + paused: "Microphone interrompu. En attente de reprise…", start: "Démarrer la dictée vocale", cancel: "Annuler la dictée", retry: "Relancer la dictée", diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index b3482f5c15..58e2706b95 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -365,6 +365,7 @@ export const ja: TranslationResources = { details: "詳細", }, dictation: { + paused: "マイクが中断されました。再開を待っています…", start: "音声入力を開始", cancel: "音声入力をキャンセル", retry: "音声入力を再試行", diff --git a/packages/app/src/i18n/resources/ko.ts b/packages/app/src/i18n/resources/ko.ts index e39d625253..fa962a5562 100644 --- a/packages/app/src/i18n/resources/ko.ts +++ b/packages/app/src/i18n/resources/ko.ts @@ -361,6 +361,7 @@ export const ko: TranslationResources = { details: "세부 정보", }, dictation: { + paused: "마이크가 중단되었습니다. 재개를 기다리는 중…", start: "음성 받아쓰기 시작", cancel: "받아쓰기 취소", retry: "받아쓰기 다시 시도", diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index 8c3d93499a..c5c6607ca6 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -364,6 +364,7 @@ export const ptBR: TranslationResources = { details: "Detalhes", }, dictation: { + paused: "Microfone interrompido. Aguardando para retomar…", start: "Iniciar ditado por voz", cancel: "Cancelar ditado", retry: "Tentar ditado novamente", diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index df57855cb6..4df3f721bb 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -364,6 +364,7 @@ export const ru: TranslationResources = { details: "Подробности", }, dictation: { + paused: "Микрофон прерван. Ожидание возобновления…", start: "Начать диктовку голосом", cancel: "Отменить диктовку", retry: "Повторить диктовку", diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index 984cc0d71a..bad9cc4690 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -360,6 +360,7 @@ export const zhCN: TranslationResources = { details: "详情", }, dictation: { + paused: "麦克风已中断，正在等待恢复…", start: "开始语音听写", cancel: "取消听写", retry: "重试听写", diff --git a/packages/expo-two-way-audio/android/build.gradle b/packages/expo-two-way-audio/android/build.gradle index 84a7837cf9..5bb7508992 100644 --- a/packages/expo-two-way-audio/android/build.gradle +++ b/packages/expo-two-way-audio/android/build.gradle @@ -36,8 +36,14 @@ android { defaultConfig { versionCode 1 versionName "0.1.0" + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" } lintOptions { abortOnError false } } + +dependencies { + androidTestImplementation 'androidx.test:runner:1.6.2' + androidTestImplementation 'androidx.test.ext:junit:1.2.1' +} diff --git a/packages/expo-two-way-audio/android/src/androidTest/AndroidManifest.xml b/packages/expo-two-way-audio/android/src/androidTest/AndroidManifest.xml new file mode 100644 index 0000000000..6501d613de --- /dev/null +++ b/packages/expo-two-way-audio/android/src/androidTest/AndroidManifest.xml @@ -0,0 +1,8 @@ + + + + + + + diff --git a/packages/expo-two-way-audio/android/src/androidTest/java/expo/modules/twowayaudio/AudioEngineTest.kt b/packages/expo-two-way-audio/android/src/androidTest/java/expo/modules/twowayaudio/AudioEngineTest.kt new file mode 100644 index 0000000000..ad840deb20 --- /dev/null +++ b/packages/expo-two-way-audio/android/src/androidTest/java/expo/modules/twowayaudio/AudioEngineTest.kt @@ -0,0 +1,223 @@ +package expo.modules.twowayaudio + +import AudioEngine +import android.app.Activity +import android.content.Context +import android.content.Intent +import android.media.AudioAttributes +import android.media.AudioFocusRequest +import android.media.AudioManager +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.filters.SdkSuppress +import org.junit.After +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.atomic.AtomicInteger + +class AudioEngineTestActivity : Activity() + +/** Real AudioManager and AudioRecord: focus loss is delivered by Android, never injected. */ +@RunWith(AndroidJUnit4::class) +class AudioEngineTest { + private val instrumentation = InstrumentationRegistry.getInstrumentation() + private lateinit var activity: Activity + private lateinit var engine: AudioEngine + private lateinit var audio: AudioManager + private val packets = AtomicInteger() + private val events = CopyOnWriteArrayList() + private val requests = mutableListOf() + + @Before fun setUp() { + val context = instrumentation.context + shell("pm grant ${context.packageName} android.permission.RECORD_AUDIO") + activity = instrumentation.startActivitySync( + Intent(context, AudioEngineTestActivity::class.java).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + ) + instrumentation.waitForIdleSync() + audio = activity.getSystemService(Context.AUDIO_SERVICE) as AudioManager + main { + engine = AudioEngine(activity) + engine.onMicDataCallback = { packets.incrementAndGet() } + engine.onAudioInterruptionCallback = { events.add(it) } + } + } + + @After fun tearDown() { + shell("cmd sensor_privacy disable 0 microphone") + main { + requests.forEach { audio.abandonAudioFocusRequest(it) } + engine.tearDown() + activity.finish() + } + } + + @Test fun dictationContinuesWhenMusicTakesFocus() { + assertDictationSurvives(AudioManager.AUDIOFOCUS_GAIN) + } + + @Test fun dictationContinuesWhenFocusIsTemporarilyLost() { + assertDictationSurvives(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT) + } + + @Test fun dictationContinuesWhenAnotherAppRequestsDucking() { + assertDictationSurvives(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK) + } + + @Test fun conversationStillStopsOnFocusLoss() { + main { assertTrue(engine.toggleRecording(true)) } + await("conversation microphone data") { packets.get() > 2 } + takeFocus(AudioManager.AUDIOFOCUS_GAIN) + await("terminal conversation interruption") { events.contains("blocked") } + assertFalse(engine.isRecording) + } + + @Test fun dictationStartsWhileMusicOwnsFocusAndCanBeStartedAgain() { + takeFocus(AudioManager.AUDIOFOCUS_GAIN) + startDictation() + main { engine.toggleRecording(false); engine.releaseAudioSession() } + val stoppedAt = packets.get() + startDictation() + await("new recording data") { packets.get() > stoppedAt + 2 } + assertFalse(events.contains("blocked")) + } + + @Test fun cancellingWhileBackgroundedDoesNotRestartOnResume() { + startDictation() + main { + engine.pauseRecordingAndPlayer() + engine.toggleRecording(false) + engine.releaseAudioSession() + engine.resumeRecordingAndPlayer() + } + assertFalse(engine.isRecording) + } + + @Test fun dictationKeepsItsCaptureModeAfterBackgroundResume() { + startDictation() + main { engine.pauseRecordingAndPlayer(); engine.resumeRecordingAndPlayer() } + val resumedAt = packets.get() + takeFocus(AudioManager.AUDIOFOCUS_GAIN) + await("resumed dictation data after focus loss") { packets.get() > resumedAt + 3 } + assertTrue(engine.isRecording) + assertFalse(events.contains("blocked")) + } + + @Test @SdkSuppress(minSdkVersion = 31) + fun microphonePrivacyInterruptionPreservesCaptureAndRecovers() { + startDictation() + shell("cmd sensor_privacy enable 0 microphone") + await("input paused") { events.contains("capture_paused") } + assertTrue(engine.isRecording) + assertFalse(events.contains("blocked")) + // Once Android reports silencing, do not enqueue artificial silence for transcription. + SystemClock.sleep(100) + val pausedAt = packets.get() + SystemClock.sleep(150) + assertEquals(pausedAt, packets.get()) + shell("cmd sensor_privacy disable 0 microphone") + await("input resumed") { events.contains("capture_resumed") } + await("microphone data after recovery") { packets.get() > pausedAt + 3 } + assertTrue(engine.isRecording) + assertEquals(listOf("capture_paused", "capture_resumed"), events.toList()) + } + + @Test @SdkSuppress(minSdkVersion = 31) + fun startingWithMicrophonePrivacyEnabledWaitsAndRecovers() { + shell("cmd sensor_privacy enable 0 microphone") + main { assertTrue(engine.startDictation()) } + await("initial input paused") { events.contains("capture_paused") } + assertTrue(engine.isRecording) + assertFalse(events.contains("blocked")) + shell("cmd sensor_privacy disable 0 microphone") + await("initial input resumed") { events.contains("capture_resumed") } + await("microphone data") { packets.get() > 3 } + } + + @Test fun resumingDictationDoesNotTakeFocusBackFromMusic() { + startDictation() + main { engine.pauseRecordingAndPlayer() } + val changes = CopyOnWriteArrayList() + val request = AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN) + .setOnAudioFocusChangeListener({ changes.add(it) }, Handler(Looper.getMainLooper())) + .build() + requests.add(request) + main { assertEquals(AudioManager.AUDIOFOCUS_REQUEST_GRANTED, audio.requestAudioFocus(request)) } + val before = packets.get() + main { engine.resumeRecordingAndPlayer() } + await("resumed microphone data") { packets.get() > before + 3 } + assertTrue(engine.isRecording) + assertTrue("Music lost focus on dictation resume: $changes", changes.isEmpty()) + } + + @Test @SdkSuppress(minSdkVersion = 31) + fun stoppingDictationReleasesTheCommunicationRoute() { + startDictation() + await("selected communication route") { hasCommunicationRoute() } + main { engine.toggleRecording(false); engine.releaseAudioSession() } + await("normal audio mode") { audio.mode == AudioManager.MODE_NORMAL } + await("cleared communication route") { !hasCommunicationRoute() } + assertFalse(engine.isRecording) + } + + private fun assertDictationSurvives(gain: Int) { + startDictation() + val before = packets.get() + val request = takeFocus(gain) + await("microphone data while focus is lost") { packets.get() > before + 3 } + assertTrue(engine.isRecording) + assertFalse(events.contains("blocked")) + val during = packets.get() + main { audio.abandonAudioFocusRequest(request) } + await("microphone data after focus release") { packets.get() > during + 3 } + assertTrue(engine.isRecording) + assertFalse(events.contains("blocked")) + } + + private fun startDictation() { + val before = packets.get() + main { assertTrue(engine.startDictation()) } + await("initial microphone data") { packets.get() > before + 3 } + } + + private fun takeFocus(gain: Int): AudioFocusRequest { + val request = AudioFocusRequest.Builder(gain) + .setAudioAttributes(AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA) + .setContentType(AudioAttributes.CONTENT_TYPE_MUSIC).build()) + .setOnAudioFocusChangeListener({}, Handler(Looper.getMainLooper())) + .build() + requests.add(request) + main { assertEquals(AudioManager.AUDIOFOCUS_REQUEST_GRANTED, audio.requestAudioFocus(request)) } + return request + } + + private fun main(action: () -> Unit) = instrumentation.runOnMainSync(action) + + private fun await(description: String, condition: () -> Boolean) { + val deadline = SystemClock.elapsedRealtime() + 5000 + while (!condition() && SystemClock.elapsedRealtime() < deadline) SystemClock.sleep(20) + assertTrue("Timed out waiting for $description; events=$events packets=${packets.get()}", condition()) + } + + // communicationDevice reports the effective system device even after our selection is cleared. + // Inspect only current route clients, never the historical routing event log. + private fun hasCommunicationRoute(): Boolean { + val dump = shell("dumpsys audio") + assertTrue("Missing Android communication route diagnostics", dump.contains("Communication route clients:")) + return dump.lineSequence().any { + it.trimStart().startsWith("[CommunicationRouteClient: mUid: ${android.os.Process.myUid()} ") + } + } + + private fun shell(command: String): String = + instrumentation.uiAutomation.executeShellCommand(command).use { + android.os.ParcelFileDescriptor.AutoCloseInputStream(it).readBytes().toString(Charsets.UTF_8) + } +} diff --git a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt index a6fe25303e..48aff1f1df 100644 --- a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt +++ b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt @@ -6,12 +6,14 @@ import android.media.AudioFocusRequest import android.media.AudioFormat import android.media.AudioManager import android.media.AudioRecord +import android.media.AudioRecordingConfiguration import android.media.AudioTrack import android.media.MediaRecorder import android.media.audiofx.AcousticEchoCanceler import android.media.audiofx.NoiseSuppressor import android.os.Build -import android.os.PowerManager +import android.os.Handler +import android.os.Looper import android.util.Log import androidx.annotation.RequiresApi import java.util.LinkedList @@ -34,7 +36,26 @@ class AudioEngine (context: Context) { private var noiseSuppressor: NoiseSuppressor? = null private val executorServiceMicrophone = Executors.newFixedThreadPool(1) private val executorServicePlayback = Executors.newFixedThreadPool(1) - private var speakerDevice: AudioDeviceInfo? = null + private enum class CaptureMode { CONVERSATION, DICTATION } + private var captureMode = CaptureMode.CONVERSATION + private val mainHandler = Handler(Looper.getMainLooper()) + @Volatile private var captureSilenced = false + private val recordingCallback = object : AudioManager.AudioRecordingCallback() { + override fun onRecordingConfigChanged(configs: MutableList) { + if (!isRecording || !::audioRecord.isInitialized) return + val config = configs.firstOrNull { it.clientAudioSessionId == audioRecord.audioSessionId } + ?: return + setCaptureSilenced(config.isClientSilenced) + } + } + private val deviceCallback = object : android.media.AudioDeviceCallback() { + override fun onAudioDevicesAdded(devices: Array) { + if (communicationRouteActive) updateAudioRouting() + } + override fun onAudioDevicesRemoved(devices: Array) { + if (communicationRouteActive) updateAudioRouting() + } + } private var communicationRouteActive = false private var bridgeWindowStartedAtMs = System.currentTimeMillis() private var micEvents = 0 @@ -44,7 +65,7 @@ class AudioEngine (context: Context) { private var playbackWrites = 0 private var playbackWriteBytes = 0L - var isRecording = false + @Volatile var isRecording = false private var isRecordingBeforePause = false var isPlaying = false @@ -83,28 +104,9 @@ class AudioEngine (context: Context) { @SuppressLint("NewApi") private fun initializeAudio(context:Context) { audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager - activateCommunicationRoute() - if (!requestAudioFocus()) { - handleAudioFocusBlocked() - } - - // Listen for changes in audio routing - audioManager.registerAudioDeviceCallback(object:android.media.AudioDeviceCallback(){ - override fun onAudioDevicesAdded(addedDevices: Array?) { - Log.d("AudioEngine", "onAudioDevicesAdded") - super.onAudioDevicesAdded(addedDevices) - if (communicationRouteActive) { - updateAudioRouting() - } - } - override fun onAudioDevicesRemoved(removedDevices: Array?) { - Log.d("AudioEngine", "onAudioDevicesRemoved") - super.onAudioDevicesRemoved(removedDevices) - if (communicationRouteActive) { - updateAudioRouting() - } - } - }, null) + // Initialization must not interrupt music before a capture/playback operation owns a session. + audioManager.registerAudioDeviceCallback(deviceCallback, mainHandler) + audioManager.registerAudioRecordingCallback(recordingCallback, mainHandler) val bufferSize = AudioTrack.getMinBufferSize( SAMPLE_RATE, @@ -125,49 +127,42 @@ class AudioEngine (context: Context) { bufferSize, AudioTrack.MODE_STREAM, audioManager.generateAudioSessionId() - ).apply { - play() - } + ) } + @SuppressLint("NewApi") private fun updateAudioRouting() { - val devices = audioManager.getDevices(AudioManager.GET_DEVICES_OUTPUTS) - var isExternalDeviceConnected = false - var selectedDevice: AudioDeviceInfo? = null - - for (device in devices) { - if (device.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER) { - speakerDevice = device - } - if (device.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES || - device.type == AudioDeviceInfo.TYPE_WIRED_HEADSET || - device.type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP || - device.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO) { - isExternalDeviceConnected = true - selectedDevice = device - break - } else if (device.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER) { - selectedDevice = device - } - } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - // Use the modern API for Android S and above - try { - selectedDevice?.let { - audioManager.setCommunicationDevice(it) - } - }catch (e:Exception){ - Log.e("AudioEngine", "Error setting communication device. Using speaker") - speakerDevice?.let { - audioManager.setCommunicationDevice(it) - } + val devices = audioManager.availableCommunicationDevices + // A2DP is a music output, not a microphone route. Android chooses the matching + // input for a valid communication sink (including SCO and BLE headsets). + val selected = devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO || + it.type == AudioDeviceInfo.TYPE_BLE_HEADSET } + ?: devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_WIRED_HEADSET || + it.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES || + it.type == AudioDeviceInfo.TYPE_USB_HEADSET } + ?: devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER } + val routed = try { + selected != null && audioManager.setCommunicationDevice(selected) + } catch (_: IllegalArgumentException) { + // The device disconnected between enumeration and selection. + false } - + if (!routed) audioManager.clearCommunicationDevice() } else { - // Fall back to deprecated method for older Android versions @Suppress("DEPRECATION") - audioManager.isSpeakerphoneOn = !isExternalDeviceConnected + val devices = audioManager.getDevices(AudioManager.GET_DEVICES_OUTPUTS) + // Before Android 12, A2DP signals a connected headset; start SCO to open its mic. + val bluetooth = devices.any { it.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO || + it.type == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP } + @Suppress("DEPRECATION") + if (bluetooth) audioManager.startBluetoothSco() else audioManager.stopBluetoothSco() + @Suppress("DEPRECATION") + audioManager.isSpeakerphoneOn = !bluetooth && !devices.any { + it.type == AudioDeviceInfo.TYPE_WIRED_HEADSET || + it.type == AudioDeviceInfo.TYPE_WIRED_HEADPHONES || + it.type == AudioDeviceInfo.TYPE_USB_HEADSET + } } } @@ -186,6 +181,8 @@ class AudioEngine (context: Context) { } else { @Suppress("DEPRECATION") audioManager.isSpeakerphoneOn = false + @Suppress("DEPRECATION") + audioManager.stopBluetoothSco() } audioManager.mode = AudioManager.MODE_NORMAL } @@ -245,8 +242,8 @@ class AudioEngine (context: Context) { audioFocusRequest = null } - val focusRequest = - AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE) + lateinit var focusRequest: AudioFocusRequest + focusRequest = AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE) .setAudioAttributes( AudioAttributes.Builder() .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) @@ -255,6 +252,7 @@ class AudioEngine (context: Context) { ) .setAcceptsDelayedFocusGain(true) .setOnAudioFocusChangeListener { focusChange -> + if (audioFocusRequest !== focusRequest) return@setOnAudioFocusChangeListener when (focusChange) { AudioManager.AUDIOFOCUS_GAIN -> { Log.d("AudioEngine", "Audio focus gained") @@ -286,6 +284,14 @@ class AudioEngine (context: Context) { } private fun handleAudioFocusBlocked() { + // Focus governs playback. Dictation keeps its AudioRecord alive; Android's + // recording configuration reports real input silencing and restores it after calls. + if (captureMode == CaptureMode.DICTATION) { + stopPlayback() + audioFocusRequest?.let { audioManager.abandonAudioFocusRequest(it) } + audioFocusRequest = null + return + } if (isRecording) { stopRecording() } @@ -306,11 +312,11 @@ class AudioEngine (context: Context) { @RequiresApi(Build.VERSION_CODES.Q) @SuppressLint("MissingPermission") - private fun startRecording(): Boolean { + private fun startRecording(requestFocus: Boolean = true): Boolean { activateCommunicationRoute() - if (!requestAudioFocus()) { + if (requestFocus && !requestAudioFocus()) { handleAudioFocusBlocked() - return false + if (captureMode == CaptureMode.CONVERSATION) return false } val bufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, AUDIO_FORMAT) @@ -344,17 +350,27 @@ class AudioEngine (context: Context) { audioRecord.startRecording() isRecording = true - startMicSampleTap() + setCaptureSilenced(audioRecord.activeRecordingConfiguration?.isClientSilenced ?: false) + startMicSampleTap(audioRecord) return true } - private fun startMicSampleTap(){ + private fun setCaptureSilenced(silenced: Boolean) { + if (captureSilenced == silenced) return + captureSilenced = silenced + if (silenced) onInputVolumeCallback?.invoke(0.0F) + onAudioInterruptionCallback?.invoke(if (silenced) "capture_paused" else "capture_resumed") + } + + private fun startMicSampleTap(recorder: AudioRecord){ executorServiceMicrophone.execute { val buffer = ByteArray(1024) try { - while (isRecording) { - val read = audioRecord.read(buffer, 0, buffer.size) - if (read > 0) { + while (isRecording && audioRecord === recorder) { + val read = recorder.read(buffer, 0, buffer.size) + if (!isRecording || audioRecord !== recorder) break + if (read < 0) throw IllegalStateException("AudioRecord read failed: $read") + if (read > 0 && !captureSilenced) { val data = buffer.copyOf(read) micEvents += 1 micBytes += data.size.toLong() @@ -367,9 +383,13 @@ class AudioEngine (context: Context) { Log.d("AudioEngine", "Mic sample tap stopped.") }catch (e: Exception){ Log.e("AudioEngine", "Error reading mic sample data", e) - isRecording = false - tearDown() - throw e + mainHandler.post { + if (isRecording && audioRecord === recorder) { + stopRecording() + releaseAudioSession() + onAudioInterruptionCallback?.invoke("blocked") + } + } } } } @@ -379,16 +399,31 @@ class AudioEngine (context: Context) { isRecording = false if (audioRecord.recordingState == AudioRecord.RECORDSTATE_RECORDING) { audioRecord.stop() - audioRecord.release() } + audioRecord.release() + echoCanceler?.release() + echoCanceler = null + noiseSuppressor?.release() + noiseSuppressor = null + setCaptureSilenced(false) onInputVolumeCallback?.invoke(0.0F) } + @RequiresApi(Build.VERSION_CODES.Q) + fun startDictation(): Boolean { + isRecordingBeforePause = false + if (isRecording) return true + captureMode = CaptureMode.DICTATION + return startRecording() + } + @RequiresApi(Build.VERSION_CODES.Q) fun toggleRecording(value: Boolean): Boolean { + isRecordingBeforePause = false if (value == isRecording) return isRecording if (value) { + captureMode = CaptureMode.CONVERSATION return startRecording() } else { stopRecording() @@ -401,6 +436,7 @@ class AudioEngine (context: Context) { @SuppressLint("NewApi") fun playPCMData(data: ByteArray) { acquireAudioSessionIfNeeded() + audioTrack.play() audioSampleQueue.add(data) playbackEvents += 1 playbackQueuedBytes += data.size.toLong() @@ -464,7 +500,7 @@ class AudioEngine (context: Context) { @RequiresApi(Build.VERSION_CODES.Q) fun pauseRecordingAndPlayer() { isRecordingBeforePause = isRecording - isRecording = toggleRecording(false) + stopRecording() audioTrack.pause() } @@ -478,16 +514,18 @@ class AudioEngine (context: Context) { if (!wasRecordingBeforePause && !isPlaying) { return } - activateCommunicationRoute() - if (!requestAudioFocus()) { - handleAudioFocusBlocked() - return + isRecordingBeforePause = false + if (wasRecordingBeforePause) { + // Returning from music controls must not pause music the user explicitly resumed. + if (!startRecording(requestFocus = captureMode == CaptureMode.CONVERSATION)) return + } else { + activateCommunicationRoute() + if (!requestAudioFocus()) { + handleAudioFocusBlocked() + return + } } - isRecording = toggleRecording(wasRecordingBeforePause) - if (wasRecordingBeforePause && !isRecording) { - return - } - audioTrack.play() + if (captureMode == CaptureMode.CONVERSATION) audioTrack.play() } fun stopPlayback() { @@ -520,7 +558,11 @@ class AudioEngine (context: Context) { audioManager.abandonAudioFocusRequest(request) } audioFocusRequest = null + audioManager.unregisterAudioDeviceCallback(deviceCallback) + audioManager.unregisterAudioRecordingCallback(recordingCallback) + audioTrack.release() executorServiceMicrophone.shutdownNow() + executorServicePlayback.shutdownNow() } diff --git a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/ExpoTwoWayAudioModule.kt b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/ExpoTwoWayAudioModule.kt index f9b107c99f..68f1489de7 100644 --- a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/ExpoTwoWayAudioModule.kt +++ b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/ExpoTwoWayAudioModule.kt @@ -42,6 +42,12 @@ class ExpoTwoWayAudioModule : Module() { audioEngine?.isRecording ?: false } + Function("startDictation") { + val isRecording = audioEngine?.startDictation() ?: false + sendEvent(ON_RECORDING_CHANGE_EVENT, mapOf("data" to isRecording)) + isRecording + } + Function("toggleRecording") { value: Boolean -> audioEngine?.let { engine -> val isRecording = engine.toggleRecording(value) diff --git a/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift b/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift index baec747995..046f856747 100644 --- a/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift +++ b/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift @@ -59,6 +59,12 @@ public class ExpoTwoWayAudioModule: Module { return audioEngine.isRecording } + Function("startDictation") { () -> Bool in + let isRecording = self.audioEngine?.toggleRecording(true) ?? false + self.sendEvent(ON_RECORDING_CHANGE_EVENT_NAME, ["data": isRecording]) + return isRecording + } + Function("toggleRecording") { (val: Bool) -> Bool in guard let audioEngine = self.audioEngine else { print("AudioEngine not initialized") diff --git a/packages/expo-two-way-audio/src/core.ts b/packages/expo-two-way-audio/src/core.ts index 1a54e0e9ce..48215f2bae 100644 --- a/packages/expo-two-way-audio/src/core.ts +++ b/packages/expo-two-way-audio/src/core.ts @@ -17,6 +17,16 @@ export function toggleRecording(val: boolean): boolean { return ExpoTwoWayAudioModule.toggleRecording(val); } +/** Capture speech without treating another app's playback focus as a microphone failure. */ +export function startDictation(): boolean { + // COMPAT(dictationCapture): added after v0.11.2, remove after 2027-01-16 once + // the native binary floor includes startDictation. Older OTA clients retain legacy capture. + if (typeof ExpoTwoWayAudioModule.startDictation !== "function") { + return ExpoTwoWayAudioModule.toggleRecording(true); + } + return ExpoTwoWayAudioModule.startDictation(); +} + export function isRecording(): boolean { return ExpoTwoWayAudioModule.isRecording(); }