From ec087dc9cdccc36e89b4f252398d5f9098ffdda1 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 28 Aug 2026 07:01:09 +0000 Subject: [PATCH] fix(dictation): preserve audio queued at stop Stopping capture disabled native recording before the final microphone buffer had crossed the bridge, so accepting immediately could omit the end of a dictation. Wait for the native producer to publish its final buffer before completing stop, and serialize stop with restart and destruction. --- .../app/src/voice/audio-engine.native.test.ts | 126 ++++++++++++++++++ packages/app/src/voice/audio-engine.native.ts | 49 +++++-- .../expo/modules/twowayaudio/AudioEngine.kt | 46 ++++++- .../twowayaudio/ExpoTwoWayAudioModule.kt | 14 +- .../expo-two-way-audio/ios/AudioEngine.swift | 43 ++++++ .../ios/ExpoTwoWayAudioModule.swift | 14 ++ packages/expo-two-way-audio/src/core.ts | 9 ++ 7 files changed, 280 insertions(+), 21 deletions(-) create mode 100644 packages/app/src/voice/audio-engine.native.test.ts diff --git a/packages/app/src/voice/audio-engine.native.test.ts b/packages/app/src/voice/audio-engine.native.test.ts new file mode 100644 index 0000000000..9b19396fa6 --- /dev/null +++ b/packages/app/src/voice/audio-engine.native.test.ts @@ -0,0 +1,126 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +interface NativeListener { + eventName: string; + handler: (event: { data: unknown }) => void; +} + +const nativeAudio = vi.hoisted(() => { + const listeners: NativeListener[] = []; + let resolveStop: (() => void) | null = null; + return { + listeners, + initialize: vi.fn(async () => true), + getMicrophonePermissionsAsync: vi.fn(async () => ({ granted: true })), + requestMicrophonePermissionsAsync: vi.fn(async () => ({ granted: true })), + toggleRecording: vi.fn((active: boolean) => active), + stopRecording: vi.fn( + () => + new Promise((resolve) => { + resolveStop = resolve; + }), + ), + resolveStop: () => { + resolveStop?.(); + resolveStop = null; + }, + releaseAudioSession: vi.fn(), + addExpoTwoWayAudioEventListener: vi.fn( + (eventName: string, handler: (event: { data: unknown }) => void) => { + listeners.push({ eventName, handler }); + return { remove: vi.fn() }; + }, + ), + resumePlayback: vi.fn(), + playPCMData: vi.fn(), + tearDown: vi.fn(), + stopPlayback: vi.fn(), + }; +}); + +vi.mock("@getpaseo/expo-two-way-audio", () => nativeAudio); + +import { createAudioEngine } from "./audio-engine.native"; + +function emitMicrophoneData(data: Uint8Array): void { + const listener = nativeAudio.listeners.find(({ eventName }) => eventName === "onMicrophoneData"); + if (!listener) { + throw new Error("Microphone listener was not registered"); + } + listener.handler({ data }); +} + +describe("native audio capture", () => { + beforeEach(() => { + nativeAudio.listeners.length = 0; + nativeAudio.resolveStop(); + vi.clearAllMocks(); + }); + + it("drains microphone events queued while capture stops", async () => { + const captured: Uint8Array[] = []; + const engine = createAudioEngine({ + onCaptureData: (pcm) => captured.push(pcm), + onVolumeLevel: vi.fn(), + }); + + await engine.startCapture(); + let didStop = false; + const stopped = engine.stopCapture(); + void (async () => { + await stopped; + didStop = true; + })(); + const finalPcm = new Uint8Array([1, 2, 3, 4]); + emitMicrophoneData(finalPcm); + + await Promise.resolve(); + expect(nativeAudio.stopRecording).toHaveBeenCalledOnce(); + expect(didStop).toBe(false); + + nativeAudio.resolveStop(); + await stopped; + + expect(captured).toEqual([finalPcm]); + + emitMicrophoneData(new Uint8Array([5, 6])); + expect(captured).toEqual([finalPcm]); + }); + + it("waits for capture to stop before destroying the native engine", async () => { + const engine = createAudioEngine({ + onCaptureData: vi.fn(), + onVolumeLevel: vi.fn(), + }); + + await engine.startCapture(); + const destroyed = engine.destroy(); + await Promise.resolve(); + + expect(nativeAudio.tearDown).not.toHaveBeenCalled(); + + nativeAudio.resolveStop(); + await destroyed; + + expect(nativeAudio.tearDown).toHaveBeenCalledOnce(); + }); + + it("does not restart capture when destruction overtakes a pending stop", async () => { + const engine = createAudioEngine({ + onCaptureData: vi.fn(), + onVolumeLevel: vi.fn(), + }); + + await engine.startCapture(); + const stopped = engine.stopCapture(); + const restarted = engine.startCapture(); + const destroyed = engine.destroy(); + + nativeAudio.resolveStop(); + await stopped; + await expect(restarted).rejects.toThrow("Audio engine has been destroyed"); + await destroyed; + + expect(nativeAudio.toggleRecording).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/app/src/voice/audio-engine.native.ts b/packages/app/src/voice/audio-engine.native.ts index 465bab3c73..40440d06ae 100644 --- a/packages/app/src/voice/audio-engine.native.ts +++ b/packages/app/src/voice/audio-engine.native.ts @@ -1,3 +1,4 @@ +import * as native from "@getpaseo/expo-two-way-audio"; import type { AudioEngine, AudioEngineCallbacks, @@ -70,11 +71,10 @@ export function createAudioEngine( callbacks: AudioEngineCallbacks, _options?: AudioEngineTraceOptions, ): AudioEngine { - const native = require("@getpaseo/expo-two-way-audio"); - const refs: { initialized: boolean; captureActive: boolean; + captureStopPromise: Promise | null; muted: boolean; queue: QueuedAudio[]; processingQueue: boolean; @@ -88,6 +88,7 @@ export function createAudioEngine( } = { initialized: false, captureActive: false, + captureStopPromise: null, muted: false, queue: [], processingQueue: false, @@ -179,6 +180,31 @@ export function createAudioEngine( } } + async function stopCapture(): Promise { + if (refs.captureStopPromise) { + await refs.captureStopPromise; + return; + } + if (!refs.captureActive) { + return; + } + + const stopping = (async () => { + await native.stopRecording(); + refs.captureActive = false; + refs.muted = false; + callbacks.onVolumeLevel(0); + releaseSessionIfIdle(); + })(); + + refs.captureStopPromise = stopping; + try { + await stopping; + } finally { + refs.captureStopPromise = null; + } + } + async function playAudio(audio: AudioPlaybackSource): Promise { await ensureInitialized(); @@ -252,12 +278,9 @@ export function createAudioEngine( return; } refs.destroyed = true; + await stopCapture(); this.stop(); this.clearQueue(); - if (refs.captureActive) { - native.toggleRecording(false); - refs.captureActive = false; - } clearPlaybackTimeout(); refs.muted = false; callbacks.onVolumeLevel(0); @@ -271,6 +294,12 @@ export function createAudioEngine( }, async startCapture() { + if (refs.captureStopPromise) { + await refs.captureStopPromise; + } + if (refs.destroyed) { + throw new Error("Audio engine has been destroyed"); + } if (refs.captureActive) { return; } @@ -293,13 +322,7 @@ export function createAudioEngine( }, async stopCapture() { - if (refs.captureActive) { - native.toggleRecording(false); - } - refs.captureActive = false; - refs.muted = false; - callbacks.onVolumeLevel(0); - releaseSessionIfIdle(); + await stopCapture(); }, toggleMute() { diff --git a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt index 9906e5a1d1..d46d878629 100644 --- a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt +++ b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/AudioEngine.kt @@ -43,7 +43,10 @@ class AudioEngine (context: Context) { private var playbackWrites = 0 private var playbackWriteBytes = 0L - var isRecording = false + @Volatile var isRecording = false + @Volatile private var recordingStopRequested = false + private val recordingStopLock = Any() + private val recordingStopCompletions = mutableListOf<() -> Unit>() private var isRecordingBeforePause = false var isPlaying = false @@ -318,6 +321,7 @@ class AudioEngine (context: Context) { } audioRecord.startRecording() + recordingStopRequested = false isRecording = true startMicSampleTap() return true @@ -338,25 +342,53 @@ class AudioEngine (context: Context) { onInputVolumeCallback?.invoke(micVolume) onMicDataCallback?.invoke(data) } + if (recordingStopRequested) { + stopRecording() + } } Log.d("AudioEngine", "Mic sample tap stopped.") }catch (e: Exception){ - Log.e("AudioEngine", "Error reading mic sample data", e) - isRecording = false - tearDown() - throw e + if (isRecording) { + Log.e("AudioEngine", "Error reading mic sample data", e) + isRecording = false + tearDown() + throw e + } + Log.d("AudioEngine", "Mic sample tap released while stopping.") } } } private fun stopRecording() { - if (!isRecording) return - isRecording = false + val completions = synchronized(recordingStopLock) { + if (!isRecording && recordingStopCompletions.isEmpty()) return + isRecording = false + recordingStopRequested = false + val pending = recordingStopCompletions.toList() + recordingStopCompletions.clear() + pending + } if (audioRecord.recordingState == AudioRecord.RECORDSTATE_RECORDING) { audioRecord.stop() audioRecord.release() } onInputVolumeCallback?.invoke(0.0F) + completions.forEach { it() } + } + + fun stopRecordingWhenDrained(completion: () -> Unit) { + val completeNow = synchronized(recordingStopLock) { + if (!isRecording) { + true + } else { + recordingStopCompletions.add(completion) + recordingStopRequested = true + false + } + } + if (completeNow) { + completion() + } } @RequiresApi(Build.VERSION_CODES.Q) diff --git a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/ExpoTwoWayAudioModule.kt b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/ExpoTwoWayAudioModule.kt index f9b107c99f..750aed0f82 100644 --- a/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/ExpoTwoWayAudioModule.kt +++ b/packages/expo-two-way-audio/android/src/main/java/expo/modules/twowayaudio/ExpoTwoWayAudioModule.kt @@ -50,6 +50,18 @@ class ExpoTwoWayAudioModule : Module() { } ?: false } + AsyncFunction("stopRecording") { promise: Promise -> + val engine = audioEngine + if (engine == null) { + promise.resolve(null) + return@AsyncFunction + } + engine.stopRecordingWhenDrained { + sendEvent(ON_RECORDING_CHANGE_EVENT, mapOf("data" to false)) + promise.resolve(null) + } + } + Function("tearDown") { audioEngine?.tearDown() audioEngine = null @@ -140,4 +152,4 @@ class ExpoTwoWayAudioModule : Module() { } } } -} \ No newline at end of file +} diff --git a/packages/expo-two-way-audio/ios/AudioEngine.swift b/packages/expo-two-way-audio/ios/AudioEngine.swift index d385dab7aa..b441ec8146 100644 --- a/packages/expo-two-way-audio/ios/AudioEngine.swift +++ b/packages/expo-two-way-audio/ios/AudioEngine.swift @@ -28,6 +28,8 @@ class AudioEngine { private var hasFirstInputBeenDiscarded = false private var discardRecording = false private var discardFirstInputMillis = 2000 + private let recordingStopLock = NSLock() + private var recordingStopCompletions: [() -> Void] = [] /// Buffers scheduled on `speechPlayer` that have not finished rendering yet. /// @@ -177,6 +179,7 @@ class AudioEngine { if self?.isRecording == true && self?.discardRecording == false { self?.processMicrophoneBuffer(buffer) self?.updateInputVolume() + self?.finishPendingRecordingStop() } } @@ -313,6 +316,7 @@ class AudioEngine { // Reset input buffer, so that volume levels report 0 inputBuffer = [Float](repeating: 0, count: 2048) updateInputVolume() + resolvePendingRecordingStops() } else { activateAudioSessionIfNeeded() avAudioEngine.inputNode.isVoiceProcessingInputMuted = false @@ -321,6 +325,45 @@ class AudioEngine { return isRecording } + + func stopRecordingWhenDrained(_ completion: @escaping () -> Void) { + recordingStopLock.lock() + if !isRecording { + recordingStopLock.unlock() + completion() + return + } + recordingStopCompletions.append(completion) + let isFirstRequest = recordingStopCompletions.count == 1 + recordingStopLock.unlock() + + if isFirstRequest { + // The input tap can hold up to 2,048 frames (128 ms at 16 kHz). Let its next + // callback publish those frames before stopping; the timeout handles a stalled tap. + DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(250)) { [weak self] in + self?.finishPendingRecordingStop() + } + } + } + + private func finishPendingRecordingStop() { + recordingStopLock.lock() + let hasPendingStop = !recordingStopCompletions.isEmpty + recordingStopLock.unlock() + guard hasPendingStop else { return } + + DispatchQueue.main.async { [weak self] in + self?.toggleRecording(false) + } + } + + private func resolvePendingRecordingStops() { + recordingStopLock.lock() + let completions = recordingStopCompletions + recordingStopCompletions.removeAll() + recordingStopLock.unlock() + completions.forEach { $0() } + } func stopRecordingAndPlayer(){ toggleRecording(false) diff --git a/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift b/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift index baec747995..061e331dbf 100644 --- a/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift +++ b/packages/expo-two-way-audio/ios/ExpoTwoWayAudioModule.swift @@ -73,6 +73,20 @@ public class ExpoTwoWayAudioModule: Module { return isRecording } + AsyncFunction("stopRecording") { (promise: Promise) in + guard let audioEngine = self.audioEngine else { + promise.resolve(true) + return + } + audioEngine.stopRecordingWhenDrained { + self.sendEvent( + ON_RECORDING_CHANGE_EVENT_NAME, + ["data": false] + ) + promise.resolve(true) + } + } + Function("getMicrophoneModeIOS") { () -> String in if #available(iOS 15.0, *) { let mode = AVCaptureDevice.preferredMicrophoneMode.rawValue diff --git a/packages/expo-two-way-audio/src/core.ts b/packages/expo-two-way-audio/src/core.ts index 1a54e0e9ce..fd05fcd7e2 100644 --- a/packages/expo-two-way-audio/src/core.ts +++ b/packages/expo-two-way-audio/src/core.ts @@ -17,6 +17,15 @@ export function toggleRecording(val: boolean): boolean { return ExpoTwoWayAudioModule.toggleRecording(val); } +export async function stopRecording(): Promise { + // COMPAT(stopRecording): added in v0.7.0, remove after 2027-02-28. An OTA JS update can land on an older binary. + if (typeof ExpoTwoWayAudioModule.stopRecording !== "function") { + ExpoTwoWayAudioModule.toggleRecording(false); + return; + } + await ExpoTwoWayAudioModule.stopRecording(); +} + export function isRecording(): boolean { return ExpoTwoWayAudioModule.isRecording(); }