mirror of
https://github.com/getpaseo/paseo.git
synced 2026-11-19 23:05:21 +00:00
Keep dictation working after closing another chat (#10185)
* fix(app): keep dictation working after closing another chat Keep the shared native recorder alive while a client still uses it. Report microphone startup errors at the composer so the user can retry from the same control. Co-authored-by: Brandon Elliott <923787+Brandon168@users.noreply.github.com> * fix(app): explain how to recover from dictation startup failures Classify known microphone failures at the capture boundary. Keep daemon stream errors available to the composer and clear stale capture feedback on the next attempt. * fix(app): keep permission guidance when the native request fails * fix(app): show dictation errors after a terminal prompt shortcut --------- Co-authored-by: Brandon Elliott <923787+Brandon168@users.noreply.github.com>
This commit is contained in:
co-authored by
Brandon Elliott
parent
b09b17e6ee
commit
2c22e64644
@@ -16,6 +16,13 @@ import {
|
||||
expectComposerFullscreenControls,
|
||||
} from "../support/helpers/composer";
|
||||
import { waitForSidebarHydration } from "../support/helpers/workspace-ui";
|
||||
import {
|
||||
seedTerminalProfiles,
|
||||
selectLaunchOption,
|
||||
terminalPromptInput,
|
||||
} from "../support/helpers/new-workspace-launch";
|
||||
|
||||
test.use({ channel: "chromium", launchOptions: { args: ["--use-fake-device-for-media-stream"] } });
|
||||
|
||||
type WebSocketMessage = string | Buffer;
|
||||
|
||||
@@ -254,3 +261,82 @@ test("dictation shortcut leaves retained fullscreen and exposes recording contro
|
||||
await agent.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
async function controlMicrophonePermission(page: Page) {
|
||||
const cdp = await page.context().newCDPSession(page);
|
||||
const { targetInfo } = await cdp.send("Target.getTargetInfo");
|
||||
const origin = new URL(page.url()).origin;
|
||||
return async (setting: "denied" | "granted") => {
|
||||
await cdp.send("Browser.setPermission", {
|
||||
permission: { name: "microphone" },
|
||||
setting,
|
||||
origin,
|
||||
browserContextId: targetInfo.browserContextId,
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function startDictationByMic(page: Page) {
|
||||
return page.getByRole("button", { name: "Start dictation", exact: true }).click();
|
||||
}
|
||||
|
||||
function expectDictationPermissionError(page: Page) {
|
||||
return expect(page.getByTestId("message-input-root").getByRole("alert")).toHaveText(
|
||||
"Allow microphone access and try again.",
|
||||
);
|
||||
}
|
||||
|
||||
test("keeps the draft and retries from the mic after access is restored", async ({ page }) => {
|
||||
const agent = await seedMockAgentWorkspace({
|
||||
repoPrefix: "dictation-permission-",
|
||||
title: "Dictation permission",
|
||||
});
|
||||
const draft = "Keep my draft";
|
||||
await installDictationFailureHarness(page);
|
||||
try {
|
||||
await openAgentRoute(page, agent);
|
||||
await expectComposerVisible(page);
|
||||
await fillComposerDraft(page, draft);
|
||||
|
||||
const setMicrophonePermission = await controlMicrophonePermission(page);
|
||||
await setMicrophonePermission("denied");
|
||||
await startDictationByMic(page);
|
||||
await expectDictationPermissionError(page);
|
||||
await expect(composerLocator(page)).toHaveValue(draft);
|
||||
|
||||
await setMicrophonePermission("granted");
|
||||
await startDictationByMic(page);
|
||||
await expect(page.getByRole("button", { name: "Cancel dictation", exact: true })).toBeVisible();
|
||||
await expect(page.getByText("Allow microphone access and try again.")).toHaveCount(0);
|
||||
await page.getByRole("button", { name: "Cancel dictation", exact: true }).click();
|
||||
await expect(composerLocator(page)).toHaveValue(draft);
|
||||
} finally {
|
||||
await agent.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("shows dictation failure when the terminal prompt starts it by shortcut", async ({ page }) => {
|
||||
const workspace = await seedWorkspace({ repoPrefix: "terminal-dictation-permission-" });
|
||||
const profile = {
|
||||
id: "echo-prompt",
|
||||
name: "Echo prompt",
|
||||
command: "echo",
|
||||
args: ["{{{prompt}}}"],
|
||||
};
|
||||
const profiles = await seedTerminalProfiles([profile]);
|
||||
await installDictationFailureHarness(page);
|
||||
try {
|
||||
await gotoAppShell(page);
|
||||
await waitForSidebarHydration(page);
|
||||
await openNewWorkspaceComposer(page, workspace);
|
||||
await selectLaunchOption(page, profile.id);
|
||||
|
||||
const setMicrophonePermission = await controlMicrophonePermission(page);
|
||||
await setMicrophonePermission("denied");
|
||||
await terminalPromptInput(page).press("Control+d");
|
||||
await expectDictationPermissionError(page);
|
||||
} finally {
|
||||
await profiles.restore();
|
||||
await workspace.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -3,7 +3,6 @@ export interface AudioEngineCallbacks {
|
||||
onVolumeLevel(level: number): void;
|
||||
onInterruption?(): void;
|
||||
onCaptureAvailabilityChange?(available: boolean): void;
|
||||
onError?(error: Error): void;
|
||||
}
|
||||
|
||||
export interface AudioPlaybackSource {
|
||||
@@ -31,3 +30,21 @@ export interface AudioEngineOptions {
|
||||
traceLabel?: string;
|
||||
captureMode?: "conversation" | "dictation";
|
||||
}
|
||||
|
||||
export type AudioCaptureStartFailure =
|
||||
| "permissionDenied"
|
||||
| "insecureContext"
|
||||
| "unsupported"
|
||||
| "noMicrophone"
|
||||
| "startFailed";
|
||||
|
||||
export class AudioCaptureStartError extends Error {
|
||||
constructor(
|
||||
readonly reason: AudioCaptureStartFailure,
|
||||
message: string,
|
||||
options?: ErrorOptions,
|
||||
) {
|
||||
super(message, options);
|
||||
this.name = "AudioCaptureStartError";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,239 +1,60 @@
|
||||
import type {
|
||||
AudioEngine,
|
||||
AudioEngineCallbacks,
|
||||
AudioEngineOptions,
|
||||
AudioPlaybackSource,
|
||||
} from "./audio-engine-types";
|
||||
import type { AudioEngine, AudioEngineCallbacks, AudioEngineOptions } from "./audio-engine-types";
|
||||
|
||||
import { createAudioPlayer, setAudioModeAsync } from "expo-audio";
|
||||
import { File, Paths } from "expo-file-system";
|
||||
import { createPlaybackQueue } from "./playback";
|
||||
import { playFile } from "./file-playback";
|
||||
import { playPcm16 } from "./pcm";
|
||||
import { createNativeAudioEngine, type NativeAudioEngineDeps } from "./native-audio-engine";
|
||||
|
||||
// AVPlayer needs a file extension to recognize local encoded audio on iOS.
|
||||
const FILE_EXTENSIONS: Record<string, string> = {
|
||||
"audio/wav": "wav",
|
||||
"audio/x-wav": "wav",
|
||||
"audio/wave": "wav",
|
||||
"audio/mpeg": "mp3",
|
||||
"audio/mp3": "mp3",
|
||||
"audio/mp4": "m4a",
|
||||
"audio/aac": "aac",
|
||||
"audio/ogg": "ogg",
|
||||
"audio/flac": "flac",
|
||||
};
|
||||
|
||||
let nextFileId = 0;
|
||||
|
||||
async function playEncodedAudio(
|
||||
bytes: Uint8Array,
|
||||
mimeType: string,
|
||||
signal: AbortSignal,
|
||||
context: { isCaptureActive(): boolean },
|
||||
): Promise<number> {
|
||||
// Capture owns its audio session while active. File playback alone must not
|
||||
// initialize the microphone or the native two-way engine.
|
||||
if (!context.isCaptureActive()) {
|
||||
await setAudioModeAsync({
|
||||
playsInSilentMode: true,
|
||||
allowsRecording: false,
|
||||
interruptionMode: "duckOthers",
|
||||
interruptionModeAndroid: "duckOthers",
|
||||
});
|
||||
}
|
||||
if (signal.aborted) throw new Error("Playback stopped");
|
||||
const extension = FILE_EXTENSIONS[mimeType.split(";")[0].trim()] ?? "audio";
|
||||
const file = new File(Paths.cache, `paseo-audio-${Date.now()}-${nextFileId++}.${extension}`);
|
||||
try {
|
||||
file.write(bytes);
|
||||
const player = createAudioPlayer(file.uri, {
|
||||
updateInterval: 100,
|
||||
keepAudioSessionActive: context.isCaptureActive(),
|
||||
});
|
||||
return await playFile(player, signal);
|
||||
} finally {
|
||||
if (file.exists) file.delete();
|
||||
}
|
||||
}
|
||||
|
||||
export function createAudioEngine(
|
||||
callbacks: AudioEngineCallbacks,
|
||||
options?: AudioEngineOptions,
|
||||
): AudioEngine {
|
||||
const native = require("@getpaseo/expo-two-way-audio");
|
||||
|
||||
const refs: {
|
||||
initialized: boolean;
|
||||
captureActive: boolean;
|
||||
muted: boolean;
|
||||
destroyed: boolean;
|
||||
} = {
|
||||
initialized: false,
|
||||
captureActive: false,
|
||||
muted: false,
|
||||
destroyed: false,
|
||||
};
|
||||
|
||||
const microphoneSubscription = native.addExpoTwoWayAudioEventListener(
|
||||
"onMicrophoneData",
|
||||
(event: { data: Uint8Array }) => {
|
||||
if (!refs.captureActive || refs.muted) {
|
||||
return;
|
||||
}
|
||||
const pcm = event.data;
|
||||
callbacks.onCaptureData(pcm);
|
||||
},
|
||||
);
|
||||
const volumeSubscription = native.addExpoTwoWayAudioEventListener(
|
||||
"onInputVolumeLevelData",
|
||||
(event: { data: number }) => {
|
||||
if (!refs.captureActive) {
|
||||
return;
|
||||
}
|
||||
const level = refs.muted ? 0 : event.data;
|
||||
callbacks.onVolumeLevel(level);
|
||||
},
|
||||
);
|
||||
const interruptionSubscription = native.addExpoTwoWayAudioEventListener(
|
||||
"onAudioInterruption",
|
||||
(event: { data: string }) => {
|
||||
if (event.data === "capture_paused" || event.data === "capture_resumed") {
|
||||
if (refs.captureActive) {
|
||||
callbacks.onCaptureAvailabilityChange?.(event.data === "capture_resumed");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (event.data !== "blocked") {
|
||||
return;
|
||||
}
|
||||
const wasCaptureActive = refs.captureActive;
|
||||
refs.captureActive = false;
|
||||
refs.muted = false;
|
||||
callbacks.onVolumeLevel(0);
|
||||
if (wasCaptureActive) {
|
||||
callbacks.onInterruption?.();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
async function ensureInitialized(): Promise<void> {
|
||||
if (refs.initialized) {
|
||||
return;
|
||||
}
|
||||
const success = await native.initialize();
|
||||
if (!success) {
|
||||
throw new Error("expo-two-way-audio: native initialize() returned false");
|
||||
}
|
||||
refs.initialized = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the OS audio session as soon as we are neither capturing nor playing.
|
||||
* Holding it keeps the user's background music paused — on iOS the non-mixing
|
||||
* `.playAndRecord` category survives backgrounding and is re-asserted on every
|
||||
* foreground, so an unreleased session means their music never comes back.
|
||||
*/
|
||||
function releaseSessionIfIdle(): void {
|
||||
if (!refs.initialized || refs.destroyed) {
|
||||
return;
|
||||
}
|
||||
if (refs.captureActive || playback.isPlaying()) {
|
||||
return;
|
||||
}
|
||||
// The wrapper no-ops on binaries whose native module predates this function.
|
||||
native.releaseAudioSession();
|
||||
}
|
||||
|
||||
async function ensureMicrophonePermission(): Promise<void> {
|
||||
let permission = await native.getMicrophonePermissionsAsync().catch(() => null);
|
||||
if (!permission?.granted) {
|
||||
permission = await native.requestMicrophonePermissionsAsync().catch(() => null);
|
||||
}
|
||||
if (!permission?.granted) {
|
||||
throw new Error(
|
||||
"Microphone permission is required to capture audio. Please enable microphone access in system settings.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let nextFileId = 0;
|
||||
async function playAudio(audio: AudioPlaybackSource, signal: AbortSignal): Promise<number> {
|
||||
const bytes = new Uint8Array(await audio.arrayBuffer());
|
||||
if (signal.aborted) throw new Error("Playback stopped");
|
||||
if (audio.type.startsWith("audio/pcm")) {
|
||||
await ensureInitialized();
|
||||
return playPcm16(bytes, audio.type, signal, native);
|
||||
}
|
||||
// Capture owns its audio session while active. File playback alone must not
|
||||
// initialize the microphone or the native two-way engine.
|
||||
if (!refs.captureActive) {
|
||||
await setAudioModeAsync({
|
||||
playsInSilentMode: true,
|
||||
allowsRecording: false,
|
||||
interruptionMode: "duckOthers",
|
||||
interruptionModeAndroid: "duckOthers",
|
||||
});
|
||||
}
|
||||
if (signal.aborted) throw new Error("Playback stopped");
|
||||
// AVPlayer needs a file extension to recognize local encoded audio on iOS.
|
||||
const extension =
|
||||
{
|
||||
"audio/wav": "wav",
|
||||
"audio/x-wav": "wav",
|
||||
"audio/wave": "wav",
|
||||
"audio/mpeg": "mp3",
|
||||
"audio/mp3": "mp3",
|
||||
"audio/mp4": "m4a",
|
||||
"audio/aac": "aac",
|
||||
"audio/ogg": "ogg",
|
||||
"audio/flac": "flac",
|
||||
}[audio.type.split(";")[0].trim()] ?? "audio";
|
||||
const file = new File(Paths.cache, `paseo-audio-${Date.now()}-${nextFileId++}.${extension}`);
|
||||
try {
|
||||
file.write(bytes);
|
||||
const player = createAudioPlayer(file.uri, {
|
||||
updateInterval: 100,
|
||||
keepAudioSessionActive: refs.captureActive,
|
||||
});
|
||||
return await playFile(player, signal);
|
||||
} finally {
|
||||
if (file.exists) file.delete();
|
||||
}
|
||||
}
|
||||
const playback = createPlaybackQueue(playAudio, releaseSessionIfIdle);
|
||||
|
||||
return {
|
||||
async initialize() {
|
||||
await ensureInitialized();
|
||||
},
|
||||
|
||||
async destroy() {
|
||||
if (refs.destroyed) {
|
||||
return;
|
||||
}
|
||||
refs.destroyed = true;
|
||||
playback.destroy();
|
||||
if (refs.captureActive) {
|
||||
native.toggleRecording(false);
|
||||
refs.captureActive = false;
|
||||
}
|
||||
refs.muted = false;
|
||||
callbacks.onVolumeLevel(0);
|
||||
if (refs.initialized) {
|
||||
native.tearDown();
|
||||
refs.initialized = false;
|
||||
}
|
||||
microphoneSubscription.remove();
|
||||
volumeSubscription.remove();
|
||||
interruptionSubscription.remove();
|
||||
},
|
||||
|
||||
async startCapture() {
|
||||
if (refs.captureActive) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await ensureMicrophonePermission();
|
||||
await ensureInitialized();
|
||||
const isRecording =
|
||||
options?.captureMode === "dictation"
|
||||
? native.startDictation()
|
||||
: native.toggleRecording(true);
|
||||
if (!isRecording) {
|
||||
throw new Error(
|
||||
"Microphone capture could not start because Android audio focus is unavailable.",
|
||||
);
|
||||
}
|
||||
refs.captureActive = true;
|
||||
} catch (error) {
|
||||
const wrapped = error instanceof Error ? error : new Error(String(error));
|
||||
callbacks.onError?.(wrapped);
|
||||
throw wrapped;
|
||||
}
|
||||
},
|
||||
|
||||
async stopCapture() {
|
||||
if (refs.captureActive) {
|
||||
native.toggleRecording(false);
|
||||
}
|
||||
refs.captureActive = false;
|
||||
refs.muted = false;
|
||||
callbacks.onVolumeLevel(0);
|
||||
callbacks.onCaptureAvailabilityChange?.(true);
|
||||
releaseSessionIfIdle();
|
||||
},
|
||||
|
||||
toggleMute() {
|
||||
refs.muted = !refs.muted;
|
||||
if (refs.muted) {
|
||||
callbacks.onVolumeLevel(0);
|
||||
}
|
||||
return refs.muted;
|
||||
},
|
||||
|
||||
isMuted() {
|
||||
return refs.muted;
|
||||
},
|
||||
|
||||
play: playback.play,
|
||||
stop: playback.stop,
|
||||
clearQueue: playback.clearQueue,
|
||||
isPlaying: playback.isPlaying,
|
||||
};
|
||||
const native: NativeAudioEngineDeps["native"] = require("@getpaseo/expo-two-way-audio");
|
||||
return createNativeAudioEngine({ native, playEncodedAudio }, callbacks, options);
|
||||
}
|
||||
|
||||
@@ -292,9 +292,7 @@ export function createAudioEngine(
|
||||
refs.gain = gain;
|
||||
} catch (error) {
|
||||
await stopCapture();
|
||||
const wrapped = error instanceof Error ? error : new Error(String(error));
|
||||
callbacks.onError?.(wrapped);
|
||||
throw wrapped;
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
export { createAudioEngine } from "./audio-engine";
|
||||
export type { AudioEngine, AudioEngineCallbacks, AudioPlaybackSource } from "./audio-engine-types";
|
||||
export { createPlayAudio } from "./play-audio";
|
||||
export { AudioCaptureStartError, type AudioCaptureStartFailure } from "./audio-engine-types";
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
import { expect, test } from "vitest";
|
||||
import type { AudioEngineCallbacks } from "./audio-engine-types";
|
||||
import { createNativeAudioEngine, type NativeAudioEngineDeps } from "./native-audio-engine";
|
||||
|
||||
// In-memory stand-in for `@getpaseo/expo-two-way-audio`, matching its iOS and Android modules:
|
||||
// one native engine per app, `initialize()` reuses it, `tearDown()` frees it, and
|
||||
// `toggleRecording(true)` / `startDictation()` return false while no engine exists.
|
||||
// Session handling follows iOS: recording activates the OS audio session, stopping does not
|
||||
// deactivate it, and `releaseAudioSession()` deactivates it only while nothing is recording.
|
||||
function fakeTwoWayAudio({
|
||||
captureAvailable = true,
|
||||
permissionRequestError,
|
||||
}: { captureAvailable?: boolean; permissionRequestError?: Error } = {}) {
|
||||
let engine: { recording: boolean; sessionActive: boolean } | null = null;
|
||||
const calls = { initialize: 0, tearDown: 0 };
|
||||
const native: NativeAudioEngineDeps["native"] = {
|
||||
async initialize() {
|
||||
calls.initialize += 1;
|
||||
engine ??= { recording: false, sessionActive: false };
|
||||
return true;
|
||||
},
|
||||
tearDown() {
|
||||
calls.tearDown += 1;
|
||||
engine = null;
|
||||
},
|
||||
toggleRecording(value: boolean) {
|
||||
if (!engine) return false;
|
||||
engine.recording = value;
|
||||
if (value) engine.sessionActive = true;
|
||||
return value;
|
||||
},
|
||||
startDictation() {
|
||||
if (!engine || !captureAvailable) return false;
|
||||
engine.recording = true;
|
||||
engine.sessionActive = true;
|
||||
return true;
|
||||
},
|
||||
releaseAudioSession() {
|
||||
if (engine && !engine.recording) engine.sessionActive = false;
|
||||
},
|
||||
playPCMData() {},
|
||||
resumePlayback() {},
|
||||
stopPlayback() {},
|
||||
async getMicrophonePermissionsAsync() {
|
||||
return { granted: !permissionRequestError };
|
||||
},
|
||||
async requestMicrophonePermissionsAsync() {
|
||||
if (permissionRequestError) throw permissionRequestError;
|
||||
return { granted: true };
|
||||
},
|
||||
addExpoTwoWayAudioEventListener() {
|
||||
return { remove() {} };
|
||||
},
|
||||
};
|
||||
return {
|
||||
native,
|
||||
calls,
|
||||
hasEngine: () => engine !== null,
|
||||
sessionActive: () => engine?.sessionActive ?? false,
|
||||
};
|
||||
}
|
||||
|
||||
const recordingCallbacks: AudioEngineCallbacks = {
|
||||
onCaptureData() {},
|
||||
onVolumeLevel() {},
|
||||
};
|
||||
|
||||
function deps(native: NativeAudioEngineDeps["native"]): NativeAudioEngineDeps {
|
||||
return {
|
||||
native,
|
||||
playEncodedAudio: async () => 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Issue #4147: on iPhone, voice mode failed with "Android audio focus is unavailable"
|
||||
// after the dictation engine was destroyed.
|
||||
test("voice capture starts after the dictation engine is destroyed", async () => {
|
||||
const fake = fakeTwoWayAudio();
|
||||
const voiceEngine = createNativeAudioEngine(deps(fake.native), recordingCallbacks);
|
||||
await voiceEngine.initialize();
|
||||
|
||||
const dictationEngine = createNativeAudioEngine(deps(fake.native), recordingCallbacks, {
|
||||
captureMode: "dictation",
|
||||
});
|
||||
await dictationEngine.startCapture();
|
||||
await dictationEngine.stopCapture();
|
||||
await dictationEngine.destroy();
|
||||
|
||||
await expect(voiceEngine.startCapture()).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
test("destroying dictation while voice is capturing keeps the native engine", async () => {
|
||||
const fake = fakeTwoWayAudio();
|
||||
const voiceEngine = createNativeAudioEngine(deps(fake.native), recordingCallbacks);
|
||||
await voiceEngine.startCapture();
|
||||
|
||||
const dictationEngine = createNativeAudioEngine(deps(fake.native), recordingCallbacks, {
|
||||
captureMode: "dictation",
|
||||
});
|
||||
await dictationEngine.initialize();
|
||||
await dictationEngine.destroy();
|
||||
|
||||
expect(fake.hasEngine()).toBe(true);
|
||||
expect(fake.calls.tearDown).toBe(0);
|
||||
});
|
||||
|
||||
// Review on #9051: with voice holding the engine, destroying an active dictation engine must
|
||||
// still hand back the idle OS audio session so background music resumes (#2866).
|
||||
test("destroying active dictation releases the idle audio session while voice keeps the engine", async () => {
|
||||
const fake = fakeTwoWayAudio();
|
||||
const voiceEngine = createNativeAudioEngine(deps(fake.native), recordingCallbacks);
|
||||
await voiceEngine.initialize();
|
||||
|
||||
const dictationEngine = createNativeAudioEngine(deps(fake.native), recordingCallbacks, {
|
||||
captureMode: "dictation",
|
||||
});
|
||||
await dictationEngine.startCapture();
|
||||
await dictationEngine.destroy();
|
||||
|
||||
expect(fake.hasEngine()).toBe(true);
|
||||
expect(fake.sessionActive()).toBe(false);
|
||||
});
|
||||
|
||||
test("destroying dictation does not release the session while voice is recording", async () => {
|
||||
const fake = fakeTwoWayAudio();
|
||||
const voiceEngine = createNativeAudioEngine(deps(fake.native), recordingCallbacks);
|
||||
await voiceEngine.startCapture();
|
||||
|
||||
const dictationEngine = createNativeAudioEngine(deps(fake.native), recordingCallbacks, {
|
||||
captureMode: "dictation",
|
||||
});
|
||||
await dictationEngine.initialize();
|
||||
await dictationEngine.destroy();
|
||||
|
||||
expect(fake.sessionActive()).toBe(true);
|
||||
});
|
||||
|
||||
test("destroying the last engine tears the native engine down", async () => {
|
||||
const fake = fakeTwoWayAudio();
|
||||
const voiceEngine = createNativeAudioEngine(deps(fake.native), recordingCallbacks);
|
||||
const dictationEngine = createNativeAudioEngine(deps(fake.native), recordingCallbacks, {
|
||||
captureMode: "dictation",
|
||||
});
|
||||
await voiceEngine.initialize();
|
||||
await dictationEngine.initialize();
|
||||
|
||||
await dictationEngine.destroy();
|
||||
await voiceEngine.destroy();
|
||||
|
||||
expect(fake.calls.tearDown).toBe(1);
|
||||
expect(fake.hasEngine()).toBe(false);
|
||||
});
|
||||
|
||||
test("an engine destroyed while initializing does not keep the native engine", async () => {
|
||||
const fake = fakeTwoWayAudio();
|
||||
const engine = createNativeAudioEngine(deps(fake.native), recordingCallbacks);
|
||||
|
||||
const initializing = expect(engine.initialize()).rejects.toThrow("Audio engine destroyed");
|
||||
await engine.destroy();
|
||||
await initializing;
|
||||
|
||||
expect(fake.hasEngine()).toBe(false);
|
||||
});
|
||||
|
||||
test("a failed native recording start rejects with a platform-neutral error", async () => {
|
||||
const fake = fakeTwoWayAudio({ captureAvailable: false });
|
||||
const engine = createNativeAudioEngine(deps(fake.native), recordingCallbacks, {
|
||||
captureMode: "dictation",
|
||||
});
|
||||
|
||||
await expect(engine.startCapture()).rejects.toThrow(/^Microphone capture could not start\.$/);
|
||||
await engine.destroy();
|
||||
});
|
||||
|
||||
test("a failed native permission request retains the permission reason and original error", async () => {
|
||||
const permissionRequestError = new Error("The current activity is not available.");
|
||||
const fake = fakeTwoWayAudio({ permissionRequestError });
|
||||
const engine = createNativeAudioEngine(deps(fake.native), recordingCallbacks, {
|
||||
captureMode: "dictation",
|
||||
});
|
||||
|
||||
await expect(engine.startCapture()).rejects.toMatchObject({
|
||||
reason: "permissionDenied",
|
||||
cause: permissionRequestError,
|
||||
});
|
||||
expect(fake.hasEngine()).toBe(false);
|
||||
await engine.destroy();
|
||||
});
|
||||
@@ -0,0 +1,292 @@
|
||||
import type {
|
||||
AudioEngine,
|
||||
AudioEngineCallbacks,
|
||||
AudioEngineOptions,
|
||||
AudioPlaybackSource,
|
||||
} from "./audio-engine-types";
|
||||
|
||||
import { AudioCaptureStartError } from "./audio-engine-types";
|
||||
import { createNativeEngineLease, type NativeEngineLease } from "./native-engine-lease";
|
||||
import { createPlaybackQueue } from "./playback";
|
||||
import { playPcm16 } from "./pcm";
|
||||
|
||||
interface NativeSubscription {
|
||||
remove(): void;
|
||||
}
|
||||
|
||||
interface MicrophonePermission {
|
||||
granted: boolean;
|
||||
}
|
||||
|
||||
/** The surface of `@getpaseo/expo-two-way-audio` the engine drives. */
|
||||
export interface TwoWayAudioModule {
|
||||
initialize(): Promise<boolean>;
|
||||
tearDown(): void;
|
||||
toggleRecording(value: boolean): boolean;
|
||||
startDictation(): boolean;
|
||||
releaseAudioSession(): void;
|
||||
playPCMData(data: Uint8Array): void;
|
||||
getMicrophonePermissionsAsync(): Promise<MicrophonePermission>;
|
||||
requestMicrophonePermissionsAsync(): Promise<MicrophonePermission>;
|
||||
addExpoTwoWayAudioEventListener(
|
||||
event: "onMicrophoneData",
|
||||
listener: (event: { data: Uint8Array }) => void,
|
||||
): NativeSubscription;
|
||||
addExpoTwoWayAudioEventListener(
|
||||
event: "onInputVolumeLevelData",
|
||||
listener: (event: { data: number }) => void,
|
||||
): NativeSubscription;
|
||||
addExpoTwoWayAudioEventListener(
|
||||
event: "onAudioInterruption",
|
||||
listener: (event: { data: string }) => void,
|
||||
): NativeSubscription;
|
||||
}
|
||||
|
||||
export interface NativeAudioEngineDeps {
|
||||
native: TwoWayAudioModule & Parameters<typeof playPcm16>[3];
|
||||
/** Plays encoded audio (wav, mp3, ...) through the platform player. */
|
||||
playEncodedAudio(
|
||||
bytes: Uint8Array,
|
||||
mimeType: string,
|
||||
signal: AbortSignal,
|
||||
context: { isCaptureActive(): boolean },
|
||||
): Promise<number>;
|
||||
}
|
||||
|
||||
// Voice mode and dictation each create an engine, but the native module holds a single engine
|
||||
// per app. Every JS engine on the same module shares one lease, so one engine's teardown
|
||||
// cannot free the native engine another still uses (#4147).
|
||||
const leases = new WeakMap<TwoWayAudioModule, NativeEngineLease>();
|
||||
|
||||
function leaseFor(native: TwoWayAudioModule): NativeEngineLease {
|
||||
let lease = leases.get(native);
|
||||
if (!lease) {
|
||||
lease = createNativeEngineLease(native);
|
||||
leases.set(native, lease);
|
||||
}
|
||||
return lease;
|
||||
}
|
||||
|
||||
export function createNativeAudioEngine(
|
||||
deps: NativeAudioEngineDeps,
|
||||
callbacks: AudioEngineCallbacks,
|
||||
options?: AudioEngineOptions,
|
||||
): AudioEngine {
|
||||
const { native } = deps;
|
||||
const lease = leaseFor(native);
|
||||
|
||||
const refs: {
|
||||
initialized: boolean;
|
||||
captureActive: boolean;
|
||||
muted: boolean;
|
||||
destroyed: boolean;
|
||||
} = {
|
||||
initialized: false,
|
||||
captureActive: false,
|
||||
muted: false,
|
||||
destroyed: false,
|
||||
};
|
||||
|
||||
const microphoneSubscription = native.addExpoTwoWayAudioEventListener(
|
||||
"onMicrophoneData",
|
||||
(event: { data: Uint8Array }) => {
|
||||
if (!refs.captureActive || refs.muted) {
|
||||
return;
|
||||
}
|
||||
const pcm = event.data;
|
||||
callbacks.onCaptureData(pcm);
|
||||
},
|
||||
);
|
||||
const volumeSubscription = native.addExpoTwoWayAudioEventListener(
|
||||
"onInputVolumeLevelData",
|
||||
(event: { data: number }) => {
|
||||
if (!refs.captureActive) {
|
||||
return;
|
||||
}
|
||||
const level = refs.muted ? 0 : event.data;
|
||||
callbacks.onVolumeLevel(level);
|
||||
},
|
||||
);
|
||||
const interruptionSubscription = native.addExpoTwoWayAudioEventListener(
|
||||
"onAudioInterruption",
|
||||
(event: { data: string }) => {
|
||||
if (event.data === "capture_paused" || event.data === "capture_resumed") {
|
||||
if (refs.captureActive) {
|
||||
callbacks.onCaptureAvailabilityChange?.(event.data === "capture_resumed");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (event.data !== "blocked") {
|
||||
return;
|
||||
}
|
||||
const wasCaptureActive = refs.captureActive;
|
||||
refs.captureActive = false;
|
||||
refs.muted = false;
|
||||
callbacks.onVolumeLevel(0);
|
||||
if (wasCaptureActive) {
|
||||
callbacks.onInterruption?.();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
let initializing: Promise<void> | null = null;
|
||||
async function ensureInitialized(): Promise<void> {
|
||||
if (refs.destroyed) {
|
||||
throw new Error("Audio engine destroyed");
|
||||
}
|
||||
if (refs.initialized) {
|
||||
return;
|
||||
}
|
||||
initializing ??= (async () => {
|
||||
try {
|
||||
await lease.acquire();
|
||||
if (refs.destroyed) {
|
||||
releaseNativeEngine();
|
||||
throw new Error("Audio engine destroyed");
|
||||
}
|
||||
refs.initialized = true;
|
||||
} finally {
|
||||
initializing = null;
|
||||
}
|
||||
})();
|
||||
await initializing;
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the OS audio session as soon as we are neither capturing nor playing.
|
||||
* Holding it keeps the user's background music paused — on iOS the non-mixing
|
||||
* `.playAndRecord` category survives backgrounding and is re-asserted on every
|
||||
* foreground, so an unreleased session means their music never comes back.
|
||||
*/
|
||||
function releaseSessionIfIdle(): void {
|
||||
if (!refs.initialized || refs.destroyed) {
|
||||
return;
|
||||
}
|
||||
if (refs.captureActive || playback.isPlaying()) {
|
||||
return;
|
||||
}
|
||||
// The wrapper no-ops on binaries whose native module predates this function.
|
||||
native.releaseAudioSession();
|
||||
}
|
||||
|
||||
async function ensureMicrophonePermission(): Promise<void> {
|
||||
const permission = await native.getMicrophonePermissionsAsync().catch(() => null);
|
||||
if (permission?.granted) return;
|
||||
|
||||
let cause: unknown;
|
||||
try {
|
||||
if ((await native.requestMicrophonePermissionsAsync()).granted) return;
|
||||
} catch (error) {
|
||||
cause = error;
|
||||
}
|
||||
throw new AudioCaptureStartError(
|
||||
"permissionDenied",
|
||||
"Allow microphone access in system settings and try again.",
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
|
||||
function releaseNativeEngine(): void {
|
||||
try {
|
||||
// Native keeps the session active while any wrapper is capturing or playing.
|
||||
native.releaseAudioSession();
|
||||
} finally {
|
||||
lease.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function playAudio(audio: AudioPlaybackSource, signal: AbortSignal): Promise<number> {
|
||||
const bytes = new Uint8Array(await audio.arrayBuffer());
|
||||
if (signal.aborted) throw new Error("Playback stopped");
|
||||
if (audio.type.startsWith("audio/pcm")) {
|
||||
await ensureInitialized();
|
||||
if (signal.aborted) throw new Error("Playback stopped");
|
||||
return playPcm16(bytes, audio.type, signal, native);
|
||||
}
|
||||
return deps.playEncodedAudio(bytes, audio.type, signal, {
|
||||
isCaptureActive: () => refs.captureActive,
|
||||
});
|
||||
}
|
||||
const playback = createPlaybackQueue(playAudio, releaseSessionIfIdle);
|
||||
|
||||
return {
|
||||
async initialize() {
|
||||
await ensureInitialized();
|
||||
},
|
||||
|
||||
async destroy() {
|
||||
if (refs.destroyed) {
|
||||
return;
|
||||
}
|
||||
refs.destroyed = true;
|
||||
try {
|
||||
playback.destroy();
|
||||
if (refs.captureActive) {
|
||||
refs.captureActive = false;
|
||||
native.toggleRecording(false);
|
||||
}
|
||||
refs.muted = false;
|
||||
callbacks.onVolumeLevel(0);
|
||||
} finally {
|
||||
microphoneSubscription.remove();
|
||||
volumeSubscription.remove();
|
||||
interruptionSubscription.remove();
|
||||
if (refs.initialized) {
|
||||
refs.initialized = false;
|
||||
releaseNativeEngine();
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
async startCapture() {
|
||||
if (refs.captureActive) {
|
||||
return;
|
||||
}
|
||||
|
||||
await ensureMicrophonePermission();
|
||||
await ensureInitialized();
|
||||
if (refs.destroyed) {
|
||||
throw new Error("Audio engine destroyed");
|
||||
}
|
||||
if (refs.captureActive) {
|
||||
return;
|
||||
}
|
||||
const isRecording =
|
||||
options?.captureMode === "dictation"
|
||||
? native.startDictation()
|
||||
: native.toggleRecording(true);
|
||||
if (!isRecording) {
|
||||
throw new Error("Microphone capture could not start.");
|
||||
}
|
||||
refs.captureActive = true;
|
||||
},
|
||||
|
||||
async stopCapture() {
|
||||
if (refs.captureActive) {
|
||||
native.toggleRecording(false);
|
||||
}
|
||||
refs.captureActive = false;
|
||||
refs.muted = false;
|
||||
callbacks.onVolumeLevel(0);
|
||||
callbacks.onCaptureAvailabilityChange?.(true);
|
||||
releaseSessionIfIdle();
|
||||
},
|
||||
|
||||
toggleMute() {
|
||||
refs.muted = !refs.muted;
|
||||
if (refs.muted) {
|
||||
callbacks.onVolumeLevel(0);
|
||||
}
|
||||
return refs.muted;
|
||||
},
|
||||
|
||||
isMuted() {
|
||||
return refs.muted;
|
||||
},
|
||||
|
||||
play: playback.play,
|
||||
stop: playback.stop,
|
||||
clearQueue: playback.clearQueue,
|
||||
isPlaying: playback.isPlaying,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import { expect, test } from "vitest";
|
||||
import { createNativeEngineLease, type NativeAudioEngineLifecycle } from "./native-engine-lease";
|
||||
|
||||
// Mirrors the iOS and Android modules: one engine per app, `initialize()` reuses it,
|
||||
// `tearDown()` frees it, and `toggleRecording(true)` returns false once it is gone.
|
||||
function fakeNativeModule(options: { initializeResult?: boolean } = {}) {
|
||||
let engine: object | null = null;
|
||||
const calls = { initialize: 0, tearDown: 0 };
|
||||
let finishInitialize: (() => void) | null = null;
|
||||
let deferInitialize = false;
|
||||
const native: NativeAudioEngineLifecycle & {
|
||||
toggleRecording(value: boolean): boolean;
|
||||
} = {
|
||||
initialize() {
|
||||
calls.initialize += 1;
|
||||
const complete = () => {
|
||||
if (options.initializeResult === false) {
|
||||
return false;
|
||||
}
|
||||
engine ??= {};
|
||||
return true;
|
||||
};
|
||||
if (!deferInitialize) {
|
||||
return Promise.resolve(complete());
|
||||
}
|
||||
return new Promise<boolean>((resolve) => {
|
||||
finishInitialize = () => resolve(complete());
|
||||
});
|
||||
},
|
||||
tearDown() {
|
||||
calls.tearDown += 1;
|
||||
engine = null;
|
||||
},
|
||||
toggleRecording(value: boolean) {
|
||||
return engine !== null && value;
|
||||
},
|
||||
};
|
||||
return {
|
||||
native,
|
||||
calls,
|
||||
deferNextInitialize() {
|
||||
deferInitialize = true;
|
||||
},
|
||||
finishInitialize() {
|
||||
finishInitialize?.();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test("dictation releasing the engine keeps it for voice mode", async () => {
|
||||
const fake = fakeNativeModule();
|
||||
const lease = createNativeEngineLease(fake.native);
|
||||
|
||||
await lease.acquire(); // voice mode
|
||||
await lease.acquire(); // dictation
|
||||
lease.release(); // dictation unmounts
|
||||
|
||||
expect(fake.calls.tearDown).toBe(0);
|
||||
expect(fake.native.toggleRecording(true)).toBe(true);
|
||||
});
|
||||
|
||||
test("the last release tears the native engine down and the next acquire rebuilds it", async () => {
|
||||
const fake = fakeNativeModule();
|
||||
const lease = createNativeEngineLease(fake.native);
|
||||
|
||||
await lease.acquire();
|
||||
lease.release();
|
||||
expect(fake.calls.tearDown).toBe(1);
|
||||
expect(fake.native.toggleRecording(true)).toBe(false);
|
||||
|
||||
await lease.acquire();
|
||||
expect(fake.calls.initialize).toBe(2);
|
||||
expect(fake.native.toggleRecording(true)).toBe(true);
|
||||
});
|
||||
|
||||
test("concurrent acquires share one native initialize", async () => {
|
||||
const fake = fakeNativeModule();
|
||||
fake.deferNextInitialize();
|
||||
const lease = createNativeEngineLease(fake.native);
|
||||
|
||||
const first = lease.acquire();
|
||||
const second = lease.acquire();
|
||||
fake.finishInitialize();
|
||||
await Promise.all([first, second]);
|
||||
|
||||
expect(fake.calls.initialize).toBe(1);
|
||||
lease.release();
|
||||
expect(fake.calls.tearDown).toBe(0);
|
||||
lease.release();
|
||||
expect(fake.calls.tearDown).toBe(1);
|
||||
});
|
||||
|
||||
test("a failed initialize is not counted as a holder and can be retried", async () => {
|
||||
const fake = fakeNativeModule({ initializeResult: false });
|
||||
const lease = createNativeEngineLease(fake.native);
|
||||
|
||||
await expect(lease.acquire()).rejects.toThrow("native initialize() returned false");
|
||||
lease.release();
|
||||
expect(fake.calls.tearDown).toBe(0);
|
||||
|
||||
await expect(lease.acquire()).rejects.toThrow();
|
||||
expect(fake.calls.initialize).toBe(2);
|
||||
});
|
||||
|
||||
test("a release with no holders is a no-op", async () => {
|
||||
const fake = fakeNativeModule();
|
||||
const lease = createNativeEngineLease(fake.native);
|
||||
|
||||
await lease.acquire();
|
||||
lease.release();
|
||||
lease.release();
|
||||
await lease.acquire();
|
||||
lease.release();
|
||||
|
||||
expect(fake.calls.tearDown).toBe(2);
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* The part of `@getpaseo/expo-two-way-audio` that owns the native engine's lifetime.
|
||||
* Each platform keeps a single engine per app: `initialize()` reuses it, `tearDown()` frees it.
|
||||
*/
|
||||
export interface NativeAudioEngineLifecycle {
|
||||
initialize(): Promise<boolean>;
|
||||
tearDown(): void;
|
||||
}
|
||||
|
||||
export interface NativeEngineLease {
|
||||
/** Ensure the native engine exists and count the caller as a holder. */
|
||||
acquire(): Promise<void>;
|
||||
/** Drop one holder; the native engine is torn down when the last holder releases. */
|
||||
release(): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Voice mode and dictation each create their own JS audio engine, but both drive the one
|
||||
* native engine. Counting holders keeps one consumer's teardown from freeing the engine
|
||||
* the other still uses (issue #4147: voice mode on iOS could not start after dictation).
|
||||
*/
|
||||
export function createNativeEngineLease(native: NativeAudioEngineLifecycle): NativeEngineLease {
|
||||
let holders = 0;
|
||||
let ready: Promise<void> | null = null;
|
||||
|
||||
async function initialize(): Promise<void> {
|
||||
const success = await native.initialize();
|
||||
if (!success) {
|
||||
throw new Error("expo-two-way-audio: native initialize() returned false");
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async acquire() {
|
||||
holders += 1;
|
||||
ready ??= initialize();
|
||||
try {
|
||||
await ready;
|
||||
} catch (error) {
|
||||
holders -= 1;
|
||||
ready = null;
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
|
||||
release() {
|
||||
if (holders === 0) {
|
||||
return;
|
||||
}
|
||||
holders -= 1;
|
||||
if (holders === 0) {
|
||||
ready = null;
|
||||
native.tearDown();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1624,6 +1624,8 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
||||
volume: dictationVolume,
|
||||
duration: dictationDuration,
|
||||
error: dictationError,
|
||||
startError: dictationStartError,
|
||||
clearStartError: clearDictationStartError,
|
||||
status: dictationStatus,
|
||||
startDictation,
|
||||
cancelDictation,
|
||||
@@ -1660,17 +1662,29 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
||||
sendAfterTranscriptRef.current = false;
|
||||
}, [dictationStatus, isDictating, isDictationProcessing]);
|
||||
|
||||
const startDictationIfAvailable = useCallback(
|
||||
() =>
|
||||
startDictationIfAvailableImpl({
|
||||
dictationUnavailableMessage,
|
||||
canStartDictation,
|
||||
toast,
|
||||
startDictation,
|
||||
exitFullscreen,
|
||||
}),
|
||||
[canStartDictation, dictationUnavailableMessage, startDictation, toast, exitFullscreen],
|
||||
);
|
||||
useEffect(() => {
|
||||
if (showRealtimeOverlay) {
|
||||
clearDictationStartError();
|
||||
}
|
||||
}, [showRealtimeOverlay, clearDictationStartError]);
|
||||
|
||||
const startDictationIfAvailable = useCallback(() => {
|
||||
clearDictationStartError();
|
||||
return startDictationIfAvailableImpl({
|
||||
dictationUnavailableMessage,
|
||||
canStartDictation,
|
||||
toast,
|
||||
startDictation,
|
||||
exitFullscreen,
|
||||
});
|
||||
}, [
|
||||
canStartDictation,
|
||||
clearDictationStartError,
|
||||
dictationUnavailableMessage,
|
||||
startDictation,
|
||||
toast,
|
||||
exitFullscreen,
|
||||
]);
|
||||
|
||||
const handleVoicePress = useCallback(
|
||||
() =>
|
||||
@@ -2224,43 +2238,50 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
||||
);
|
||||
const toolbar = useMemo(
|
||||
() => (
|
||||
<View style={isFullscreen ? styles.hiddenToolbar : buttonRowStyle}>
|
||||
{/* Toolbar left: attachment button + agent controls */}
|
||||
<View style={styles.leftButtonGroup}>
|
||||
{!isFullscreen && (
|
||||
<AttachmentDropdown
|
||||
visible={mode.showAttachments}
|
||||
isConnected={isConnected}
|
||||
disabled={disabled}
|
||||
attachButtonStyle={attachButtonStyle}
|
||||
renderAttachButtonIcon={renderAttachButtonIcon}
|
||||
attachmentMenuItems={attachmentMenuItems}
|
||||
addAttachmentLabel={t("composer.input.addAttachment")}
|
||||
/>
|
||||
)}
|
||||
{leftContent}
|
||||
</View>
|
||||
|
||||
{/* Right: dictation, then the trailing action (voice mode, send or stop) */}
|
||||
{!isFullscreen && (
|
||||
<View style={styles.rightButtonGroup}>
|
||||
{beforeVoiceContent}
|
||||
<VoiceButtonTooltip
|
||||
visible={mode.showVoice}
|
||||
onVoicePress={handleVoicePress}
|
||||
isDictationStartEnabled={isDictationStartEnabled}
|
||||
voiceButtonAccessibilityLabel={voiceButtonAccessibilityLabel}
|
||||
voiceButtonStyle={voiceButtonStyle}
|
||||
renderVoiceButtonIcon={renderVoiceButtonIcon}
|
||||
voiceTooltipText={voiceTooltipText}
|
||||
isRealtimeVoiceForCurrentAgent={isRealtimeVoiceForCurrentAgent}
|
||||
voiceMuteToggleKeys={voiceMuteToggleKeys}
|
||||
dictationToggleKeys={dictationToggleKeys}
|
||||
/>
|
||||
<View style={styles.trailingDisc}>{sendButton}</View>
|
||||
<>
|
||||
{!isFullscreen && !showOverlay && dictationStartError ? (
|
||||
<Text accessibilityRole="alert" style={styles.dictationStartError}>
|
||||
{t(`message.dictation.${dictationStartError}`)}
|
||||
</Text>
|
||||
) : null}
|
||||
<View style={isFullscreen ? styles.hiddenToolbar : buttonRowStyle}>
|
||||
{/* Toolbar left: attachment button + agent controls */}
|
||||
<View style={styles.leftButtonGroup}>
|
||||
{!isFullscreen && (
|
||||
<AttachmentDropdown
|
||||
visible={mode.showAttachments}
|
||||
isConnected={isConnected}
|
||||
disabled={disabled}
|
||||
attachButtonStyle={attachButtonStyle}
|
||||
renderAttachButtonIcon={renderAttachButtonIcon}
|
||||
attachmentMenuItems={attachmentMenuItems}
|
||||
addAttachmentLabel={t("composer.input.addAttachment")}
|
||||
/>
|
||||
)}
|
||||
{leftContent}
|
||||
</View>
|
||||
)}
|
||||
</View>
|
||||
|
||||
{/* Right: dictation, then the trailing action (voice mode, send or stop) */}
|
||||
{!isFullscreen && (
|
||||
<View style={styles.rightButtonGroup}>
|
||||
{beforeVoiceContent}
|
||||
<VoiceButtonTooltip
|
||||
visible={mode.showVoice}
|
||||
onVoicePress={handleVoicePress}
|
||||
isDictationStartEnabled={isDictationStartEnabled}
|
||||
voiceButtonAccessibilityLabel={voiceButtonAccessibilityLabel}
|
||||
voiceButtonStyle={voiceButtonStyle}
|
||||
renderVoiceButtonIcon={renderVoiceButtonIcon}
|
||||
voiceTooltipText={voiceTooltipText}
|
||||
isRealtimeVoiceForCurrentAgent={isRealtimeVoiceForCurrentAgent}
|
||||
voiceMuteToggleKeys={voiceMuteToggleKeys}
|
||||
dictationToggleKeys={dictationToggleKeys}
|
||||
/>
|
||||
<View style={styles.trailingDisc}>{sendButton}</View>
|
||||
</View>
|
||||
)}
|
||||
</View>
|
||||
</>
|
||||
),
|
||||
[
|
||||
buttonRowStyle,
|
||||
@@ -2275,6 +2296,8 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
||||
leftContent,
|
||||
beforeVoiceContent,
|
||||
mode.showVoice,
|
||||
showOverlay,
|
||||
dictationStartError,
|
||||
handleVoicePress,
|
||||
isDictationStartEnabled,
|
||||
voiceButtonAccessibilityLabel,
|
||||
@@ -2477,6 +2500,12 @@ const styles = StyleSheet.create((theme: Theme) => ({
|
||||
alignItems: "center",
|
||||
justifyContent: "space-between",
|
||||
},
|
||||
dictationStartError: {
|
||||
alignSelf: "flex-end",
|
||||
textAlign: "right",
|
||||
color: theme.colors.destructive,
|
||||
fontSize: theme.fontSize.sm,
|
||||
},
|
||||
leftButtonGroup: {
|
||||
minWidth: 0,
|
||||
flexShrink: 1,
|
||||
|
||||
@@ -130,9 +130,6 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
||||
console.error("[VoiceEngine] Failed to stop after audio interruption:", error);
|
||||
});
|
||||
},
|
||||
onError: (error) => {
|
||||
console.error("[VoiceEngine] Capture error:", error);
|
||||
},
|
||||
});
|
||||
|
||||
runtime = createVoiceRuntime({
|
||||
|
||||
@@ -11,7 +11,6 @@ import type {
|
||||
|
||||
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
|
||||
const onPcmSegmentRef = useRef(config.onPcmSegment);
|
||||
const onErrorRef = useRef(config.onError);
|
||||
const onInterruptionRef = useRef(config.onInterruption);
|
||||
const [volume, setVolume] = useState(0);
|
||||
const [isPaused, setIsPaused] = useState(false);
|
||||
@@ -30,9 +29,6 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
onVolumeLevel: (level) => {
|
||||
setVolume(level);
|
||||
},
|
||||
onError: (error) => {
|
||||
onErrorRef.current?.(error);
|
||||
},
|
||||
onInterruption: () => {
|
||||
onInterruptionRef.current?.();
|
||||
},
|
||||
@@ -45,14 +41,11 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
|
||||
useEffect(() => {
|
||||
onPcmSegmentRef.current = config.onPcmSegment;
|
||||
onErrorRef.current = config.onError;
|
||||
onInterruptionRef.current = config.onInterruption;
|
||||
}, [config.onPcmSegment, config.onError, config.onInterruption]);
|
||||
}, [config.onPcmSegment, config.onInterruption]);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
const engine = getOrCreateEngine();
|
||||
await engine.initialize();
|
||||
await engine.startCapture();
|
||||
await getOrCreateEngine().startCapture();
|
||||
}, [getOrCreateEngine]);
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { AudioCaptureStartError } from "@/audio";
|
||||
import { resampleToPcm16 } from "@/audio/pcm";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { parsePcm16Wav } from "@/utils/pcm16-wav";
|
||||
@@ -133,6 +134,32 @@ async function finalizeRecorderStoppedPromise(input: {
|
||||
}
|
||||
}
|
||||
|
||||
async function requestMicrophone(): Promise<MediaStream> {
|
||||
try {
|
||||
return await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
noiseSuppression: true,
|
||||
echoCancellation: true,
|
||||
autoGainControl: true,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof DOMException) {
|
||||
if (error.name === "NotAllowedError") {
|
||||
throw new AudioCaptureStartError("permissionDenied", error.message, { cause: error });
|
||||
}
|
||||
if (error.name === "NotFoundError") {
|
||||
throw new AudioCaptureStartError("noMicrophone", error.message, { cause: error });
|
||||
}
|
||||
if (error.name === "NotSupportedError") {
|
||||
throw new AudioCaptureStartError("unsupported", error.message, { cause: error });
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
|
||||
const [volume, setVolume] = useState(0);
|
||||
|
||||
@@ -200,50 +227,19 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
}, []);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
const missingNavigator =
|
||||
typeof navigator === "undefined" ||
|
||||
!navigator.mediaDevices ||
|
||||
typeof navigator.mediaDevices.getUserMedia !== "function";
|
||||
|
||||
const secureContext =
|
||||
typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
|
||||
? window.isSecureContext
|
||||
: true;
|
||||
const currentOrigin =
|
||||
typeof window !== "undefined" && window.location ? window.location.origin : "unknown";
|
||||
const isDesktopApp = isElectronRuntime();
|
||||
|
||||
if (missingNavigator) {
|
||||
throw new Error("Microphone capture is not supported in this environment");
|
||||
}
|
||||
if (!secureContext && !isDesktopApp) {
|
||||
throw new Error(
|
||||
`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`,
|
||||
);
|
||||
}
|
||||
if (!secureContext && isDesktopApp) {
|
||||
console.warn(
|
||||
"[DictationAudio][Web] Insecure context reported under Desktop; attempting getUserMedia anyway",
|
||||
{ currentOrigin },
|
||||
if (!window.isSecureContext && !isElectronRuntime()) {
|
||||
throw new AudioCaptureStartError(
|
||||
"insecureContext",
|
||||
`Microphone access requires HTTPS or localhost. Current origin: ${window.location.origin}`,
|
||||
);
|
||||
}
|
||||
|
||||
const AudioContextCtor = getAudioContextCtor();
|
||||
if (!AudioContextCtor) {
|
||||
throw new Error("AudioContext unavailable");
|
||||
if (!navigator.mediaDevices?.getUserMedia || !AudioContextCtor) {
|
||||
throw new AudioCaptureStartError("unsupported", "Browser audio capture is unavailable.");
|
||||
}
|
||||
|
||||
const rawStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
noiseSuppression: true,
|
||||
echoCancellation: true,
|
||||
autoGainControl: true,
|
||||
},
|
||||
});
|
||||
|
||||
const stream = rawStream;
|
||||
|
||||
const stream = await requestMicrophone();
|
||||
const context = new AudioContextCtor();
|
||||
|
||||
try {
|
||||
@@ -431,15 +427,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
|
||||
return useMemo(
|
||||
() => ({
|
||||
start: async () => {
|
||||
try {
|
||||
await start();
|
||||
} catch (err) {
|
||||
const normalized = err instanceof Error ? err : new Error(String(err));
|
||||
onErrorRef.current?.(normalized);
|
||||
throw normalized;
|
||||
}
|
||||
},
|
||||
start,
|
||||
stop,
|
||||
volume,
|
||||
isPaused: false,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { AudioCaptureStartFailure } from "@/audio";
|
||||
import { i18n } from "@/i18n/i18next";
|
||||
|
||||
export type DictationStatus = "idle" | "recording" | "paused" | "uploading" | "failed";
|
||||
@@ -21,6 +22,8 @@ export interface UseDictationResult {
|
||||
volume: number;
|
||||
duration: number;
|
||||
error: string | null;
|
||||
startError: AudioCaptureStartFailure | null;
|
||||
clearStartError: () => void;
|
||||
status: DictationStatus;
|
||||
startDictation: () => Promise<void>;
|
||||
cancelDictation: () => Promise<void>;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
|
||||
import { AudioCaptureStartError, type AudioCaptureStartFailure } from "@/audio";
|
||||
import { DictationStreamSender } from "@/dictation/dictation-stream-sender";
|
||||
import { useDictationAudioSource } from "@/hooks/use-dictation-audio-source";
|
||||
import { generateMessageId } from "@/types/stream";
|
||||
@@ -32,6 +33,8 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
const [partialTranscript, setPartialTranscript] = useState("");
|
||||
const [duration, setDuration] = useState(0);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [startError, setStartError] = useState<AudioCaptureStartFailure | null>(null);
|
||||
const clearStartError = useCallback(() => setStartError(null), []);
|
||||
const [status, setStatus] = useState<DictationStatus>("idle");
|
||||
const latestPartialTranscriptRef = useRef("");
|
||||
|
||||
@@ -265,14 +268,17 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
|
||||
actionGateRef.current.starting = true;
|
||||
setError(null);
|
||||
setStartError(null);
|
||||
setPartialTranscript("");
|
||||
setDuration(0);
|
||||
setIsProcessing(false);
|
||||
setStatus("recording");
|
||||
clearStreamingState();
|
||||
|
||||
let captureStarted = false;
|
||||
try {
|
||||
await audio.start();
|
||||
captureStarted = true;
|
||||
isRecordingRef.current = true;
|
||||
setIsRecording(true);
|
||||
if (enableDuration) {
|
||||
@@ -287,7 +293,12 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
isRecordingRef.current = false;
|
||||
setIsRecording(false);
|
||||
setStatus("idle");
|
||||
reportError(err, "Failed to start dictation");
|
||||
if (captureStarted) {
|
||||
reportError(err, "Failed to start dictation stream");
|
||||
} else {
|
||||
console.error("[useDictation] Failed to start dictation capture", err);
|
||||
setStartError(err instanceof AudioCaptureStartError ? err.reason : "startFailed");
|
||||
}
|
||||
} finally {
|
||||
actionGateRef.current.starting = false;
|
||||
}
|
||||
@@ -438,6 +449,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
stopDurationTracking();
|
||||
setDuration(0);
|
||||
setError(null);
|
||||
setStartError(null);
|
||||
setStatus("idle");
|
||||
clearStreamingState();
|
||||
}, [clearStreamingState, stopDurationTracking]);
|
||||
@@ -461,6 +473,8 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
volume: audio.volume,
|
||||
duration,
|
||||
error,
|
||||
startError,
|
||||
clearStartError,
|
||||
status: status === "recording" && audio.isPaused ? "paused" : status,
|
||||
startDictation,
|
||||
cancelDictation,
|
||||
|
||||
@@ -377,6 +377,11 @@ export const ar: TranslationResources = {
|
||||
details: "تفاصيل",
|
||||
},
|
||||
dictation: {
|
||||
insecureContext: "افتح Paseo عبر HTTPS أو localhost لاستخدام الميكروفون.",
|
||||
unsupported: "الإملاء غير متاح في هذا المتصفح.",
|
||||
noMicrophone: "صِل ميكروفونًا وحاول مرة أخرى.",
|
||||
permissionDenied: "اسمح بالوصول إلى الميكروفون وحاول مرة أخرى.",
|
||||
startFailed: "تعذر بدء الإملاء. حاول مرة أخرى.",
|
||||
paused: "تمت مقاطعة الميكروفون. في انتظار الاستئناف…",
|
||||
start: "بدء الإملاء الصوتي",
|
||||
cancel: "إلغاء الإملاء",
|
||||
|
||||
@@ -376,6 +376,11 @@ export const en = {
|
||||
details: "Details",
|
||||
},
|
||||
dictation: {
|
||||
insecureContext: "Open Paseo over HTTPS or localhost to use the microphone.",
|
||||
unsupported: "Dictation is unavailable in this browser.",
|
||||
noMicrophone: "Connect a microphone and try again.",
|
||||
permissionDenied: "Allow microphone access and try again.",
|
||||
startFailed: "Could not start dictation. Try again.",
|
||||
paused: "Microphone interrupted. Waiting to resume…",
|
||||
start: "Start voice dictation",
|
||||
cancel: "Cancel dictation",
|
||||
|
||||
@@ -382,6 +382,11 @@ export const es: TranslationResources = {
|
||||
details: "Detalles",
|
||||
},
|
||||
dictation: {
|
||||
insecureContext: "Abre Paseo por HTTPS o en localhost para usar el micrófono.",
|
||||
unsupported: "El dictado no está disponible en este navegador.",
|
||||
noMicrophone: "Conecta un micrófono e inténtalo de nuevo.",
|
||||
permissionDenied: "Permite el acceso al micrófono e inténtalo de nuevo.",
|
||||
startFailed: "No se pudo iniciar el dictado. Inténtalo de nuevo.",
|
||||
paused: "Micrófono interrumpido. Esperando para reanudar…",
|
||||
start: "Iniciar dictado de voz",
|
||||
cancel: "Cancelar dictado",
|
||||
|
||||
@@ -382,6 +382,11 @@ export const fr: TranslationResources = {
|
||||
details: "Détails",
|
||||
},
|
||||
dictation: {
|
||||
insecureContext: "Ouvrez Paseo via HTTPS ou sur localhost pour utiliser le microphone.",
|
||||
unsupported: "La dictée est indisponible dans ce navigateur.",
|
||||
noMicrophone: "Connectez un microphone, puis réessayez.",
|
||||
permissionDenied: "Autorisez l’accès au microphone, puis réessayez.",
|
||||
startFailed: "Impossible de lancer la dictée. Réessayez.",
|
||||
paused: "Microphone interrompu. En attente de reprise…",
|
||||
start: "Démarrer la dictée vocale",
|
||||
cancel: "Annuler la dictée",
|
||||
|
||||
@@ -382,6 +382,11 @@ export const ja: TranslationResources = {
|
||||
details: "詳細",
|
||||
},
|
||||
dictation: {
|
||||
insecureContext: "マイクを使用するには、HTTPS または localhost で Paseo を開いてください。",
|
||||
unsupported: "このブラウザーでは音声入力を利用できません。",
|
||||
noMicrophone: "マイクを接続して、もう一度お試しください。",
|
||||
permissionDenied: "マイクへのアクセスを許可して、もう一度お試しください。",
|
||||
startFailed: "音声入力を開始できませんでした。もう一度お試しください。",
|
||||
paused: "マイクが中断されました。再開を待っています…",
|
||||
start: "音声入力を開始",
|
||||
cancel: "音声入力をキャンセル",
|
||||
|
||||
@@ -378,6 +378,11 @@ export const ko: TranslationResources = {
|
||||
details: "세부 정보",
|
||||
},
|
||||
dictation: {
|
||||
insecureContext: "마이크를 사용하려면 HTTPS 또는 localhost에서 Paseo를 여세요.",
|
||||
unsupported: "이 브라우저에서는 받아쓰기를 사용할 수 없습니다.",
|
||||
noMicrophone: "마이크를 연결한 후 다시 시도하세요.",
|
||||
permissionDenied: "마이크 접근을 허용한 후 다시 시도하세요.",
|
||||
startFailed: "받아쓰기를 시작할 수 없습니다. 다시 시도하세요.",
|
||||
paused: "마이크가 중단되었습니다. 재개를 기다리는 중…",
|
||||
start: "음성 받아쓰기 시작",
|
||||
cancel: "받아쓰기 취소",
|
||||
|
||||
@@ -381,6 +381,11 @@ export const ptBR: TranslationResources = {
|
||||
details: "Detalhes",
|
||||
},
|
||||
dictation: {
|
||||
insecureContext: "Abra o Paseo por HTTPS ou em localhost para usar o microfone.",
|
||||
unsupported: "O ditado não está disponível neste navegador.",
|
||||
noMicrophone: "Conecte um microfone e tente novamente.",
|
||||
permissionDenied: "Permita o acesso ao microfone e tente novamente.",
|
||||
startFailed: "Não foi possível iniciar o ditado. Tente novamente.",
|
||||
paused: "Microfone interrompido. Aguardando para retomar…",
|
||||
start: "Iniciar ditado por voz",
|
||||
cancel: "Cancelar ditado",
|
||||
|
||||
@@ -381,6 +381,11 @@ export const ru: TranslationResources = {
|
||||
details: "Подробности",
|
||||
},
|
||||
dictation: {
|
||||
insecureContext: "Чтобы использовать микрофон, откройте Paseo по HTTPS или на localhost.",
|
||||
unsupported: "Диктовка недоступна в этом браузере.",
|
||||
noMicrophone: "Подключите микрофон и попробуйте снова.",
|
||||
permissionDenied: "Разрешите доступ к микрофону и попробуйте снова.",
|
||||
startFailed: "Не удалось начать диктовку. Попробуйте снова.",
|
||||
paused: "Микрофон прерван. Ожидание возобновления…",
|
||||
start: "Начать диктовку голосом",
|
||||
cancel: "Отменить диктовку",
|
||||
|
||||
@@ -377,6 +377,11 @@ export const zhCN: TranslationResources = {
|
||||
details: "详情",
|
||||
},
|
||||
dictation: {
|
||||
insecureContext: "请通过 HTTPS 或 localhost 打开 Paseo,以使用麦克风。",
|
||||
unsupported: "此浏览器无法使用听写。",
|
||||
noMicrophone: "请连接麦克风,然后重试。",
|
||||
permissionDenied: "请允许访问麦克风,然后重试。",
|
||||
startFailed: "无法开始听写。请重试。",
|
||||
paused: "麦克风已中断,正在等待恢复…",
|
||||
start: "开始语音听写",
|
||||
cancel: "取消听写",
|
||||
|
||||
Reference in New Issue
Block a user