From 42806cdbf9ee979d3500111b2df17cf60850c131 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Sat, 3 Oct 2026 16:56:59 +0100 Subject: [PATCH] Let plugins play audio on every client (#5976) * feat(plugins): play audio files on every client * fix(audio): preserve voice output and isolate waiting cues * fix(plugins): normalize audio MIME parameters * refactor(audio): share PCM conversion and keep plugin playback file-only --- .../app/e2e/browser/plugin-host-ui.spec.ts | 32 +- .../{voice => audio}/audio-engine-types.ts | 2 +- .../src/{voice => audio}/audio-engine.d.ts | 0 packages/app/src/audio/audio-engine.native.ts | 224 +++++++++++ .../src/{voice => audio}/audio-engine.web.ts | 190 +++------- packages/app/src/audio/file-playback.ts | 61 +++ packages/app/src/audio/index.ts | 3 + packages/app/src/audio/pcm/index.ts | 124 +++++++ packages/app/src/audio/play-audio.test.ts | 202 ++++++++++ packages/app/src/audio/play-audio.ts | 38 ++ packages/app/src/audio/playback.test.ts | 136 +++++++ packages/app/src/audio/playback.ts | 81 ++++ packages/app/src/contexts/session-context.tsx | 2 +- packages/app/src/contexts/voice-context.tsx | 4 +- .../use-dictation-audio-source.native.ts | 2 +- .../hooks/use-dictation-audio-source.web.ts | 36 +- packages/app/src/plugins/catalog-sync.tsx | 7 +- packages/app/src/plugins/client-runtime.ts | 7 +- packages/app/src/plugins/evaluate.ts | 1 + .../src/plugins/registry-lifecycle.test.ts | 22 +- .../src/plugins/registry-requirements.test.ts | 8 +- packages/app/src/plugins/registry.test.ts | 4 +- packages/app/src/plugins/registry.ts | 4 +- .../app/src/plugins/timeline/view.test.tsx | 7 +- .../workspace-panels/locations.test.ts | 3 + packages/app/src/voice/audio-engine.native.ts | 350 ------------------ packages/app/src/voice/voice-runtime.test.ts | 7 +- packages/app/src/voice/voice-runtime.ts | 40 +- packages/plugin/src/client/contracts.ts | 2 + public-docs/plugins/reference.md | 23 ++ 30 files changed, 1042 insertions(+), 580 deletions(-) rename packages/app/src/{voice => audio}/audio-engine-types.ts (88%) rename packages/app/src/{voice => audio}/audio-engine.d.ts (100%) create mode 100644 packages/app/src/audio/audio-engine.native.ts rename packages/app/src/{voice => audio}/audio-engine.web.ts (63%) create mode 100644 packages/app/src/audio/file-playback.ts create mode 100644 packages/app/src/audio/index.ts create mode 100644 packages/app/src/audio/pcm/index.ts create mode 100644 packages/app/src/audio/play-audio.test.ts create mode 100644 packages/app/src/audio/play-audio.ts create mode 100644 packages/app/src/audio/playback.test.ts create mode 100644 packages/app/src/audio/playback.ts delete mode 100644 packages/app/src/voice/audio-engine.native.ts diff --git a/packages/app/e2e/browser/plugin-host-ui.spec.ts b/packages/app/e2e/browser/plugin-host-ui.spec.ts index bd53c59272..84d786033a 100644 --- a/packages/app/e2e/browser/plugin-host-ui.spec.ts +++ b/packages/app/e2e/browser/plugin-host-ui.spec.ts @@ -1,5 +1,5 @@ import { pluginRequirements } from "../support/helpers/plugin-fixture"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { expect, test, type Page } from "../support/fixtures"; @@ -36,9 +36,18 @@ function ModalBody({ onSaved }) { ; } -function Surface() { +function Surface({ playAudio }) { + const [audioStatus, setAudioStatus] = useState("Audio ready"); + async function play(base64) { + setAudioStatus("Playing audio"); + try { await playAudio({ base64, mimeType: "audio/wav" }); setAudioStatus("Audio finished"); } + catch { setAudioStatus("Audio rejected"); } + } const [open, setOpen] = useState(false); return + {audioStatus} + play(AUDIO_BASE64)}>Play plugin audio + play("UklGRg==")}>Play invalid audio setOpen(true)}> @@ -59,7 +68,7 @@ function Surface() { } export default function contribute(plugin) { - plugin.addSurface("main", Surface); + plugin.addSurface("main", () => ); plugin.addSidebarItem({ id: "main", title: "Host UI", @@ -123,7 +132,11 @@ test("plugin modal adapts its presentation and preserves host contexts", async ( path.join(directory, "paseo-plugin.json"), JSON.stringify({ id: PLUGIN_ID, requirements: pluginRequirements }), ); - await writeFile(path.join(directory, "index.client.tsx"), PLUGIN_SOURCE); + const audio = await readFile(path.resolve(__dirname, "../../assets/audio/thinking-tone.wav")); + await writeFile( + path.join(directory, "index.client.tsx"), + `const AUDIO_BASE64 = ${JSON.stringify(audio.toString("base64"))};\n${PLUGIN_SOURCE}`, + ); try { await client.patchDaemonConfig({ pluginsEnabled: true }); @@ -131,6 +144,12 @@ test("plugin modal adapts its presentation and preserves host contexts", async ( await useNonCompactLayout(page); await openHostUiPlugin(page); + await test.step("plugin audio finishes, rejects corrupt files, and recovers", async () => { + await playPluginAudio(page, "Play plugin audio", "Audio finished"); + await playPluginAudio(page, "Play invalid audio", "Audio rejected"); + await playPluginAudio(page, "Play plugin audio", "Audio finished"); + }); + await test.step("non-compact layouts use a centered dialog", async () => { await openPluginModal(page); await expectCenteredPluginDialog(page); @@ -153,3 +172,8 @@ test("plugin modal adapts its presentation and preserves host contexts", async ( await rm(directory, { recursive: true, force: true }); } }); + +async function playPluginAudio(page: Page, button: string, result: string) { + await page.getByRole("button", { name: button, exact: true }).click(); + await expect(page.getByText(result, { exact: true })).toBeVisible(); +} diff --git a/packages/app/src/voice/audio-engine-types.ts b/packages/app/src/audio/audio-engine-types.ts similarity index 88% rename from packages/app/src/voice/audio-engine-types.ts rename to packages/app/src/audio/audio-engine-types.ts index 03108b5cf6..90bec732fb 100644 --- a/packages/app/src/voice/audio-engine-types.ts +++ b/packages/app/src/audio/audio-engine-types.ts @@ -20,7 +20,7 @@ export interface AudioEngine { toggleMute(): boolean; isMuted(): boolean; - play(audio: AudioPlaybackSource): Promise; + play(audio: AudioPlaybackSource, signal?: AbortSignal): Promise; stop(): void; clearQueue(): void; isPlaying(): boolean; diff --git a/packages/app/src/voice/audio-engine.d.ts b/packages/app/src/audio/audio-engine.d.ts similarity index 100% rename from packages/app/src/voice/audio-engine.d.ts rename to packages/app/src/audio/audio-engine.d.ts diff --git a/packages/app/src/audio/audio-engine.native.ts b/packages/app/src/audio/audio-engine.native.ts new file mode 100644 index 0000000000..86a6fc3d26 --- /dev/null +++ b/packages/app/src/audio/audio-engine.native.ts @@ -0,0 +1,224 @@ +import type { AudioEngine, AudioEngineCallbacks, AudioPlaybackSource } from "./audio-engine-types"; + +import { createAudioPlayer, setAudioModeAsync } from "expo-audio"; +import { File, Paths } from "expo-file-system"; +import { createPlaybackQueue } from "./playback"; +import { playFile } from "./file-playback"; +import { playPcm16 } from "./pcm"; + +export function createAudioEngine( + callbacks: AudioEngineCallbacks, + _options?: { traceLabel?: string }, +): AudioEngine { + const native = require("@getpaseo/expo-two-way-audio"); + + const refs: { + initialized: boolean; + captureActive: boolean; + muted: boolean; + destroyed: boolean; + } = { + initialized: false, + captureActive: false, + muted: false, + destroyed: false, + }; + + const microphoneSubscription = native.addExpoTwoWayAudioEventListener( + "onMicrophoneData", + (event: { data: Uint8Array }) => { + if (!refs.captureActive || refs.muted) { + return; + } + const pcm = event.data; + callbacks.onCaptureData(pcm); + }, + ); + const volumeSubscription = native.addExpoTwoWayAudioEventListener( + "onInputVolumeLevelData", + (event: { data: number }) => { + if (!refs.captureActive) { + return; + } + const level = refs.muted ? 0 : event.data; + callbacks.onVolumeLevel(level); + }, + ); + const interruptionSubscription = native.addExpoTwoWayAudioEventListener( + "onAudioInterruption", + (event: { data: string }) => { + if (event.data !== "blocked") { + return; + } + const wasCaptureActive = refs.captureActive; + refs.captureActive = false; + refs.muted = false; + callbacks.onVolumeLevel(0); + if (wasCaptureActive) { + callbacks.onInterruption?.(); + } + }, + ); + + async function ensureInitialized(): Promise { + if (refs.initialized) { + return; + } + const success = await native.initialize(); + if (!success) { + throw new Error("expo-two-way-audio: native initialize() returned false"); + } + refs.initialized = true; + } + + /** + * Release the OS audio session as soon as we are neither capturing nor playing. + * Holding it keeps the user's background music paused — on iOS the non-mixing + * `.playAndRecord` category survives backgrounding and is re-asserted on every + * foreground, so an unreleased session means their music never comes back. + */ + function releaseSessionIfIdle(): void { + if (!refs.initialized || refs.destroyed) { + return; + } + if (refs.captureActive || playback.isPlaying()) { + return; + } + // The wrapper no-ops on binaries whose native module predates this function. + native.releaseAudioSession(); + } + + async function ensureMicrophonePermission(): Promise { + let permission = await native.getMicrophonePermissionsAsync().catch(() => null); + if (!permission?.granted) { + permission = await native.requestMicrophonePermissionsAsync().catch(() => null); + } + if (!permission?.granted) { + throw new Error( + "Microphone permission is required to capture audio. Please enable microphone access in system settings.", + ); + } + } + + let nextFileId = 0; + async function playAudio(audio: AudioPlaybackSource, signal: AbortSignal): Promise { + const bytes = new Uint8Array(await audio.arrayBuffer()); + if (signal.aborted) throw new Error("Playback stopped"); + if (audio.type.startsWith("audio/pcm")) { + await ensureInitialized(); + return playPcm16(bytes, audio.type, signal, native); + } + // Capture owns its audio session while active. File playback alone must not + // initialize the microphone or the native two-way engine. + if (!refs.captureActive) { + await setAudioModeAsync({ + playsInSilentMode: true, + allowsRecording: false, + interruptionMode: "duckOthers", + interruptionModeAndroid: "duckOthers", + }); + } + if (signal.aborted) throw new Error("Playback stopped"); + // AVPlayer needs a file extension to recognize local encoded audio on iOS. + const extension = + { + "audio/wav": "wav", + "audio/x-wav": "wav", + "audio/wave": "wav", + "audio/mpeg": "mp3", + "audio/mp3": "mp3", + "audio/mp4": "m4a", + "audio/aac": "aac", + "audio/ogg": "ogg", + "audio/flac": "flac", + }[audio.type.split(";")[0].trim()] ?? "audio"; + const file = new File(Paths.cache, `paseo-audio-${Date.now()}-${nextFileId++}.${extension}`); + try { + file.write(bytes); + const player = createAudioPlayer(file.uri, { + updateInterval: 100, + keepAudioSessionActive: refs.captureActive, + }); + return await playFile(player, signal); + } finally { + if (file.exists) file.delete(); + } + } + const playback = createPlaybackQueue(playAudio, releaseSessionIfIdle); + + return { + async initialize() { + await ensureInitialized(); + }, + + async destroy() { + if (refs.destroyed) { + return; + } + refs.destroyed = true; + playback.destroy(); + if (refs.captureActive) { + native.toggleRecording(false); + refs.captureActive = false; + } + refs.muted = false; + callbacks.onVolumeLevel(0); + if (refs.initialized) { + native.tearDown(); + refs.initialized = false; + } + microphoneSubscription.remove(); + volumeSubscription.remove(); + interruptionSubscription.remove(); + }, + + async startCapture() { + if (refs.captureActive) { + return; + } + + try { + await ensureMicrophonePermission(); + await ensureInitialized(); + const isRecording = native.toggleRecording(true); + if (!isRecording) { + throw new Error( + "Microphone capture could not start because Android audio focus is unavailable.", + ); + } + refs.captureActive = true; + } catch (error) { + const wrapped = error instanceof Error ? error : new Error(String(error)); + callbacks.onError?.(wrapped); + throw wrapped; + } + }, + + async stopCapture() { + if (refs.captureActive) { + native.toggleRecording(false); + } + refs.captureActive = false; + refs.muted = false; + callbacks.onVolumeLevel(0); + releaseSessionIfIdle(); + }, + + toggleMute() { + refs.muted = !refs.muted; + if (refs.muted) { + callbacks.onVolumeLevel(0); + } + return refs.muted; + }, + + isMuted() { + return refs.muted; + }, + + play: playback.play, + stop: playback.stop, + clearQueue: playback.clearQueue, + isPlaying: playback.isPlaying, + }; +} diff --git a/packages/app/src/voice/audio-engine.web.ts b/packages/app/src/audio/audio-engine.web.ts similarity index 63% rename from packages/app/src/voice/audio-engine.web.ts rename to packages/app/src/audio/audio-engine.web.ts index 0453c0ae33..dc0f049ca7 100644 --- a/packages/app/src/voice/audio-engine.web.ts +++ b/packages/app/src/audio/audio-engine.web.ts @@ -1,15 +1,8 @@ +import { parsePcmSampleRate, resampleToPcm16 } from "./pcm"; import { isElectronRuntime } from "@/desktop/host"; -import type { - AudioEngine, - AudioEngineCallbacks, - AudioPlaybackSource, -} from "@/voice/audio-engine-types"; +import type { AudioEngine, AudioEngineCallbacks, AudioPlaybackSource } from "./audio-engine-types"; -interface QueuedAudio { - audio: AudioPlaybackSource; - resolve: (duration: number) => void; - reject: (error: Error) => void; -} +import { createPlaybackQueue } from "./playback"; function getAudioContextCtor(): typeof AudioContext | null { if (typeof window === "undefined") { @@ -21,40 +14,6 @@ function getAudioContextCtor(): typeof AudioContext | null { return browserWindow.AudioContext ?? browserWindow.webkitAudioContext ?? null; } -function floatToInt16(sample: number): number { - const clamped = Math.max(-1, Math.min(1, sample)); - return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff); -} - -function resampleToPcm16(input: Float32Array, inputRate: number, outputRate: number): Uint8Array { - if (input.length === 0) { - return new Uint8Array(0); - } - - const ratio = inputRate / outputRate; - const outputLength = Math.max(1, Math.round(input.length / ratio)); - const output = new Int16Array(outputLength); - for (let i = 0; i < outputLength; i += 1) { - const sourceIndex = i * ratio; - const i0 = Math.floor(sourceIndex); - const i1 = Math.min(input.length - 1, i0 + 1); - const frac = sourceIndex - i0; - const sample = input[i0] * (1 - frac) + input[i1] * frac; - output[i] = floatToInt16(sample); - } - - return new Uint8Array(output.buffer, output.byteOffset, output.byteLength); -} - -function parsePcmSampleRate(mimeType: string): number | null { - const match = /rate=(\d+)/i.exec(mimeType); - if (!match) { - return null; - } - const rate = Number(match[1]); - return Number.isFinite(rate) && rate > 0 ? rate : null; -} - function pcm16LeToAudioBuffer( context: AudioContext, bytes: Uint8Array, @@ -98,14 +57,6 @@ export function createAudioEngine( gain: GainNode | null; started: boolean; muted: boolean; - queue: QueuedAudio[]; - processingQueue: boolean; - activePlayback: { - source: AudioBufferSourceNode; - resolve: (duration: number) => void; - reject: (error: Error) => void; - settled: boolean; - } | null; } = { playbackContext: null, captureContext: null, @@ -115,29 +66,31 @@ export function createAudioEngine( gain: null, started: false, muted: false, - queue: [], - processingQueue: false, - activePlayback: null, }; async function ensurePlaybackContext(): Promise { - if (refs.playbackContext) { - if (refs.playbackContext.state === "suspended") { - await refs.playbackContext.resume().catch(() => undefined); - } - return refs.playbackContext; + if (!refs.playbackContext) { + const AudioContextCtor = getAudioContextCtor(); + if (!AudioContextCtor) throw new Error("AudioContext unavailable"); + refs.playbackContext = new AudioContextCtor(); } - - const AudioContextCtor = getAudioContextCtor(); - if (!AudioContextCtor) { - throw new Error("AudioContext unavailable"); - } - - const context = new AudioContextCtor(); + const context = refs.playbackContext; if (context.state === "suspended") { - await context.resume().catch(() => undefined); + let timeout: ReturnType | undefined; + try { + await Promise.race([ + context.resume(), + new Promise((_resolve, reject) => { + timeout = setTimeout( + () => reject(new Error("Audio playback requires a user interaction")), + 2000, + ); + }), + ]); + } finally { + clearTimeout(timeout); + } } - refs.playbackContext = context; return context; } @@ -162,7 +115,7 @@ export function createAudioEngine( return context; } - async function playAudio(audio: AudioPlaybackSource): Promise { + async function playAudio(audio: AudioPlaybackSource, signal: AbortSignal): Promise { const context = await ensurePlaybackContext(); const arrayBuffer = await audio.arrayBuffer(); const type = (audio.type || "").toLowerCase(); @@ -174,53 +127,39 @@ export function createAudioEngine( ) : await decodeAudioData(context, arrayBuffer); + if (signal.aborted) throw new Error("Playback stopped"); const durationSec = audioBuffer.duration; const source = context.createBufferSource(); source.buffer = audioBuffer; source.connect(context.destination); return await new Promise((resolve, reject) => { - refs.activePlayback = { source, resolve, reject, settled: false }; - - const settle = (fn: () => void) => { - const active = refs.activePlayback; - if (!active || active.source !== source || active.settled) { - return; - } - active.settled = true; - refs.activePlayback = null; - fn(); + const abort = () => { + source.stop(); + source.disconnect(); + reject(new Error("Playback stopped")); }; - - source.addEventListener("ended", () => { - settle(() => resolve(durationSec)); - }); - + signal.addEventListener("abort", abort, { once: true }); + source.addEventListener( + "ended", + () => { + signal.removeEventListener("abort", abort); + source.disconnect(); + resolve(durationSec); + }, + { once: true }, + ); try { source.start(); } catch (error) { - settle(() => reject(error instanceof Error ? error : new Error(String(error)))); + signal.removeEventListener("abort", abort); + source.disconnect(); + reject(error); } }); } - async function processQueue(): Promise { - if (refs.processingQueue || refs.queue.length === 0) { - return; - } - - refs.processingQueue = true; - while (refs.queue.length > 0) { - const item = refs.queue.shift()!; - try { - const duration = await playAudio(item.audio); - item.resolve(duration); - } catch (error) { - item.reject(error instanceof Error ? error : new Error(String(error))); - } - } - refs.processingQueue = false; - } + const playback = createPlaybackQueue(playAudio); async function stopCapture(): Promise { refs.started = false; @@ -263,8 +202,7 @@ export function createAudioEngine( }, async destroy() { - this.stop(); - this.clearQueue(); + playback.destroy(); await stopCapture(); const playbackContext = refs.playbackContext; @@ -334,7 +272,8 @@ export function createAudioEngine( return; } - callbacks.onCaptureData(resampleToPcm16(input, context.sampleRate, 16000)); + const pcm = resampleToPcm16(input, context.sampleRate, 16000); + callbacks.onCaptureData(new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength)); }; source.connect(processor); @@ -370,40 +309,9 @@ export function createAudioEngine( return refs.muted; }, - async play(audio: AudioPlaybackSource) { - return await new Promise((resolve, reject) => { - refs.queue.push({ audio, resolve, reject }); - if (!refs.processingQueue) { - void processQueue(); - } - }); - }, - - stop() { - if (refs.activePlayback) { - const active = refs.activePlayback; - refs.activePlayback = null; - try { - active.source.stop(); - } catch { - // Ignore best-effort stop errors. - } - if (!active.settled) { - active.settled = true; - active.reject(new Error("Playback stopped")); - } - } - }, - - clearQueue() { - while (refs.queue.length > 0) { - refs.queue.shift()!.reject(new Error("Playback stopped")); - } - refs.processingQueue = false; - }, - - isPlaying() { - return refs.activePlayback !== null; - }, + play: playback.play, + stop: playback.stop, + clearQueue: playback.clearQueue, + isPlaying: playback.isPlaying, }; } diff --git a/packages/app/src/audio/file-playback.ts b/packages/app/src/audio/file-playback.ts new file mode 100644 index 0000000000..6d4370bd6f --- /dev/null +++ b/packages/app/src/audio/file-playback.ts @@ -0,0 +1,61 @@ +export interface FilePlaybackStatus { + playbackState: string; + isLoaded: boolean; + didJustFinish: boolean; + duration: number; +} + +export interface FilePlayer { + play(): void; + remove(): void; + addListener( + event: "playbackStatusUpdate", + listener: (status: FilePlaybackStatus) => void, + ): { remove(): void }; +} + +/** Own completion, load failure and resource release for the native file backend. */ +export function playFile(player: FilePlayer, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + let settled = false; + // expo-audio 1.x does not report Android decoder errors. A local file that + // never becomes loaded must still release its queue slot and temporary file. + let timeout: ReturnType | undefined; + const finish = (error?: unknown, duration = 0) => { + if (settled) return; + settled = true; + clearTimeout(timeout); + signal.removeEventListener("abort", abort); + subscription.remove(); + player.remove(); + if (error) reject(error); + else resolve(duration); + }; + const abort = () => finish(new Error("Playback stopped")); + let loaded = false; + const subscription = player.addListener("playbackStatusUpdate", (status) => { + if (status.playbackState === "error" || status.playbackState === "failed") { + finish(new Error("Audio file could not be decoded")); + } else if (status.didJustFinish) { + finish(undefined, status.duration); + } else if (status.isLoaded && status.duration > 0 && !loaded) { + loaded = true; + clearTimeout(timeout); + timeout = setTimeout( + () => finish(new Error("Audio playback timed out")), + (status.duration + 30) * 1000, + ); + } + }); + timeout = setTimeout(() => finish(new Error("Audio file could not be loaded")), 15000); + signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) abort(); + else { + try { + player.play(); + } catch (error) { + finish(error); + } + } + }); +} diff --git a/packages/app/src/audio/index.ts b/packages/app/src/audio/index.ts new file mode 100644 index 0000000000..fd5bbfd221 --- /dev/null +++ b/packages/app/src/audio/index.ts @@ -0,0 +1,3 @@ +export { createAudioEngine } from "./audio-engine"; +export type { AudioEngine, AudioEngineCallbacks, AudioPlaybackSource } from "./audio-engine-types"; +export { createPlayAudio } from "./play-audio"; diff --git a/packages/app/src/audio/pcm/index.ts b/packages/app/src/audio/pcm/index.ts new file mode 100644 index 0000000000..fe1164b8ce --- /dev/null +++ b/packages/app/src/audio/pcm/index.ts @@ -0,0 +1,124 @@ +const floatToInt16 = (sample: number): number => { + const clamped = Math.max(-1, Math.min(1, sample)); + return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff); +}; + +export const resampleToPcm16 = ( + input: Float32Array, + inputRate: number, + outputRate: number, +): Int16Array => { + if (input.length === 0) { + return new Int16Array(0); + } + if (inputRate === outputRate) { + const out = new Int16Array(input.length); + for (let i = 0; i < input.length; i++) { + out[i] = floatToInt16(input[i]); + } + return out; + } + + const ratio = inputRate / outputRate; + const outputLength = Math.max(1, Math.round(input.length / ratio)); + const out = new Int16Array(outputLength); + for (let i = 0; i < outputLength; i++) { + const sourceIndex = i * ratio; + const i0 = Math.floor(sourceIndex); + const i1 = Math.min(input.length - 1, i0 + 1); + const frac = sourceIndex - i0; + const sample = input[i0] * (1 - frac) + input[i1] * frac; + out[i] = floatToInt16(sample); + } + return out; +}; + +export function parsePcmSampleRate(mimeType: string): number | null { + const match = /rate=(\d+)/i.exec(mimeType); + if (!match) { + return null; + } + const rate = Number(match[1]); + return Number.isFinite(rate) && rate > 0 ? rate : null; +} + +function resamplePcm16(pcm: Uint8Array, fromRate: number, toRate: number): Uint8Array { + if (fromRate === toRate) { + return pcm; + } + + const inputSamples = Math.floor(pcm.length / 2); + const outputSamples = Math.floor((inputSamples * toRate) / fromRate); + const out = new Uint8Array(outputSamples * 2); + const ratio = fromRate / toRate; + + const readInt16 = (sampleIndex: number): number => { + const i = sampleIndex * 2; + if (i + 1 >= pcm.length) { + return 0; + } + const lo = pcm[i]; + const hi = pcm[i + 1]; + let value = (hi << 8) | lo; + if (value & 0x8000) { + value = value - 0x10000; + } + return value; + }; + + const writeInt16 = (sampleIndex: number, value: number): void => { + const clamped = Math.max(-32768, Math.min(32767, Math.round(value))); + const i = sampleIndex * 2; + out[i] = clamped & 0xff; + out[i + 1] = (clamped >> 8) & 0xff; + }; + + for (let i = 0; i < outputSamples; i++) { + const srcPos = i * ratio; + const i0 = Math.floor(srcPos); + const frac = srcPos - i0; + const s0 = readInt16(i0); + const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1)); + writeInt16(i, s0 + (s1 - s0) * frac); + } + + return out; +} + +interface PcmOutput { + resumePlayback(): void; + playPCMData(bytes: Uint8Array): void; + stopPlayback(): void; +} + +/** Keep voice output on the native communication engine and its echo reference. */ +export function playPcm16( + bytes: Uint8Array, + mimeType: string, + signal: AbortSignal, + output: PcmOutput, +): Promise { + if (signal.aborted) return Promise.reject(new Error("Playback stopped")); + const pcm = resamplePcm16(bytes, parsePcmSampleRate(mimeType) ?? 24000, 16000); + const duration = pcm.length / 2 / 16000; + return new Promise((resolve, reject) => { + let timeout: ReturnType | undefined; + const abort = () => { + clearTimeout(timeout); + output.stopPlayback(); + reject(new Error("Playback stopped")); + }; + signal.addEventListener("abort", abort, { once: true }); + try { + output.resumePlayback(); + output.playPCMData(pcm); + timeout = setTimeout(() => { + signal.removeEventListener("abort", abort); + resolve(duration); + }, duration * 1000); + } catch (error) { + signal.removeEventListener("abort", abort); + reject(error); + } + }); +} diff --git a/packages/app/src/audio/play-audio.test.ts b/packages/app/src/audio/play-audio.test.ts new file mode 100644 index 0000000000..a47b5c0ecc --- /dev/null +++ b/packages/app/src/audio/play-audio.test.ts @@ -0,0 +1,202 @@ +import { expect, test } from "vitest"; +import { createPlayAudio } from "./play-audio"; +import { playPcm16, parsePcmSampleRate, resampleToPcm16 } from "./pcm"; +import { playFile, type FilePlaybackStatus, type FilePlayer } from "./file-playback"; + +const source = { base64: "UklGRg==", mimeType: "audio/wav" }; +test("passes RPC file bytes to the shared engine and awaits completion", async () => { + const owner = new AbortController(); + let finish!: () => void; + const played = new Promise((resolve) => { + finish = resolve; + }); + let completed = false; + const play = createPlayAudio( + { + async play(audio, signal) { + expect(signal).toBe(owner.signal); + expect(audio.type).toBe("audio/wav"); + expect(audio.size).toBe(4); + expect(new Uint8Array(await audio.arrayBuffer())).toEqual(new Uint8Array([82, 73, 70, 70])); + await played; + return 1; + }, + }, + owner.signal, + ); + const result = play(source).then(() => { + completed = true; + return undefined; + }); + await Promise.resolve(); + expect(completed).toBe(false); + finish(); + await result; + expect(completed).toBe(true); +}); + +test.each([ + { ...source, base64: "garbage!" }, + { ...source, base64: "" }, + { ...source, mimeType: "text/plain" }, +])("rejects invalid input before reaching a decoder: %o", async (input) => { + const play = createPlayAudio( + { + play() { + throw new Error("Reached decoder"); + }, + }, + new AbortController().signal, + ); + await expect(play(input)).rejects.toThrow(/^Audio must/); +}); + +test("rejects calls from an unloaded plugin", async () => { + const owner = new AbortController(); + owner.abort(new Error("Plugin unloaded")); + const play = createPlayAudio( + { + play() { + throw new Error("Reached decoder"); + }, + }, + owner.signal, + ); + await expect(play(source)).rejects.toThrow("Playback stopped"); +}); + +test("voice PCM retains its native output and 16 kHz samples", async () => { + const pcm = new Uint8Array([0, 0, 255, 127, 0, 128]); + let written: Uint8Array | undefined; + const duration = await playPcm16( + pcm, + "audio/pcm;rate=16000;bits=16", + new AbortController().signal, + { + resumePlayback() {}, + playPCMData(bytes) { + written = bytes; + }, + stopPlayback() {}, + }, + ); + expect(written).toEqual(pcm); + expect(duration).toBe(3 / 16000); +}); + +test("voice PCM cancellation stops native output and rejects", async () => { + let stopped = false; + const owner = new AbortController(); + const result = playPcm16(new Uint8Array(32000), "audio/pcm;rate=16000;bits=16", owner.signal, { + resumePlayback() {}, + playPCMData() {}, + stopPlayback() { + stopped = true; + }, + }); + owner.abort(); + await expect(result).rejects.toThrow("Playback stopped"); + expect(stopped).toBe(true); +}); + +function filePlayer() { + let listener: (status: FilePlaybackStatus) => void = () => {}; + let removed = false; + let started = false; + const player: FilePlayer = { + play() { + started = true; + }, + remove() { + removed = true; + }, + addListener(_event, callback) { + listener = callback; + return { + remove() { + listener = () => {}; + }, + }; + }, + }; + return { + player, + status: (status: Partial) => + listener({ + playbackState: "ready", + duration: 1, + didJustFinish: false, + isLoaded: true, + ...status, + }), + removed: () => removed, + started: () => started, + }; +} + +test("native file playback waits for completion and releases its player", async () => { + const h = filePlayer(); + const promise = playFile(h.player, new AbortController().signal); + expect(h.started()).toBe(true); + h.status({ didJustFinish: true }); + await expect(promise).resolves.toBe(1); + expect(h.removed()).toBe(true); +}); + +test("native decoder errors reject and release the player", async () => { + const h = filePlayer(); + const promise = playFile(h.player, new AbortController().signal); + h.status({ playbackState: "failed" }); + await expect(promise).rejects.toThrow("could not be decoded"); + expect(h.removed()).toBe(true); +}); + +test("native cancellation releases playback and settles the caller", async () => { + const h = filePlayer(); + const owner = new AbortController(); + const promise = playFile(h.player, owner.signal); + owner.abort(new Error("Plugin unloaded")); + await expect(promise).rejects.toThrow("Playback stopped"); + expect(h.removed()).toBe(true); +}); + +test.each(["audio/pcm;rate=16000;bits=16", "AUDIO/PCM; rate=16000"])( + "rejects raw PCM instead of initializing voice capture: %s", + async (mimeType) => { + const play = createPlayAudio( + { + play: async () => { + throw new Error("Reached engine"); + }, + }, + new AbortController().signal, + ); + await expect(play({ base64: "AAAAAA==", mimeType })).rejects.toThrow( + "Pass an audio file, such as WAV or MP3", + ); + }, +); + +// Voice capture and browser dictation use this same PCM boundary. +test("capture PCM preserves clipping, signed samples, and empty input", () => { + expect(resampleToPcm16(new Float32Array([-2, -1, -0.5, 0, 0.5, 1, 2]), 16000, 16000)).toEqual( + new Int16Array([-32768, -32768, -16384, 0, 16384, 32767, 32767]), + ); + expect(resampleToPcm16(new Float32Array(), 48000, 16000)).toEqual(new Int16Array()); +}); + +test("capture PCM resamples with interpolation and preserves the last sample", () => { + expect(resampleToPcm16(new Float32Array([-1, 0, 1]), 24000, 16000)).toEqual( + new Int16Array([-32768, 16384]), + ); + expect(resampleToPcm16(new Float32Array([0, 1]), 8000, 16000)).toEqual( + new Int16Array([0, 16384, 32767, 32767]), + ); +}); + +test("voice playback shares PCM sample-rate parsing", () => { + expect(parsePcmSampleRate("audio/pcm;rate=16000;bits=16")).toBe(16000); + expect(parsePcmSampleRate("audio/PCM;RATE=48000")).toBe(48000); + expect(parsePcmSampleRate("audio/pcm")).toBeNull(); + expect(parsePcmSampleRate("audio/pcm;rate=0")).toBeNull(); +}); diff --git a/packages/app/src/audio/play-audio.ts b/packages/app/src/audio/play-audio.ts new file mode 100644 index 0000000000..e5d831de17 --- /dev/null +++ b/packages/app/src/audio/play-audio.ts @@ -0,0 +1,38 @@ +import { Buffer } from "buffer"; +import type { AudioEngine } from "./audio-engine-types"; + +/** RPC-friendly file input. No browser Blob, URL, or platform knowledge is needed. */ +export function createPlayAudio(engine: Pick, signal: AbortSignal) { + return async (source: { base64: string; mimeType: string }): Promise => { + if (signal.aborted) throw new Error("Playback stopped"); + if ( + !source || + typeof source.base64 !== "string" || + !/^[A-Za-z0-9+/]+={0,2}$/.test(source.base64) || + source.base64.length % 4 !== 0 || + !source.base64.length + ) { + throw new Error("Audio must contain a non-empty base64-encoded file"); + } + if ( + typeof source.mimeType !== "string" || + !/^audio\/[\w.+-]+(?:[ \t]*;[^\r\n]+)?$/i.test(source.mimeType) + ) { + throw new Error("Audio must have an audio MIME type, such as audio/wav"); + } + const bytes = Uint8Array.from(Buffer.from(source.base64, "base64")); + const [mediaType, ...parameters] = source.mimeType.split(";"); + const type = [mediaType.trim().toLowerCase(), ...parameters].join(";"); + if (mediaType.trim().toLowerCase() === "audio/pcm") { + throw new Error("Pass an audio file, such as WAV or MP3; raw PCM is not supported"); + } + await engine.play( + { + type, + size: bytes.byteLength, + arrayBuffer: async () => bytes.buffer, + }, + signal, + ); + }; +} diff --git a/packages/app/src/audio/playback.test.ts b/packages/app/src/audio/playback.test.ts new file mode 100644 index 0000000000..2627d899cf --- /dev/null +++ b/packages/app/src/audio/playback.test.ts @@ -0,0 +1,136 @@ +import { expect, test } from "vitest"; +import { createPlaybackQueue } from "./playback"; + +function harness() { + const started: string[] = []; + const finishes: (() => void)[] = []; + const queue = createPlaybackQueue((source, signal) => { + started.push(source); + return new Promise((resolve, reject) => { + finishes.push(() => resolve(1)); + signal.addEventListener("abort", () => reject(new Error("Playback stopped")), { once: true }); + }); + }); + return { queue, started, finishes }; +} + +test("plays in order and resolves only after playback completes", async () => { + const h = harness(); + const first = h.queue.play("first"); + const second = h.queue.play("second"); + expect(h.started).toEqual(["first"]); + h.finishes[0](); + await expect(first).resolves.toBe(1); + expect(h.started).toEqual(["first", "second"]); + h.finishes[1](); + await expect(second).resolves.toBe(1); +}); + +test("cancels one owner's queued and active work without stopping another owner", async () => { + const h = harness(); + const owner = new AbortController(); + const first = h.queue.play("first", owner.signal); + const queued = h.queue.play("cancelled", owner.signal); + const other = h.queue.play("other"); + const rejected = Promise.all([ + expect(first).rejects.toThrow("Playback stopped"), + expect(queued).rejects.toThrow("Playback stopped"), + ]); + owner.abort(new Error("unloaded")); + await rejected; + expect(h.started).toEqual(["first", "other"]); + h.finishes[1](); + await expect(other).resolves.toBe(1); +}); + +test("stop during loading cannot start cancelled audio or strand the next call", async () => { + const h = harness(); + const first = h.queue.play("loading"); + const rejected = expect(first).rejects.toThrow("Playback stopped"); + h.queue.stop(); + await rejected; + const second = h.queue.play("next"); + h.finishes[1](); + await expect(second).resolves.toBe(1); +}); + +test("a failed file does not block the queue", async () => { + const queue = createPlaybackQueue(async (source) => { + if (source === "invalid") throw new Error("Invalid audio"); + return 2; + }); + const failed = queue.play("invalid"); + const next = queue.play("valid"); + await expect(failed).rejects.toThrow("Invalid audio"); + await expect(next).resolves.toBe(2); +}); + +test("ending the voice waiting cue leaves a plugin's active audio playing", async () => { + const { createVoiceRuntime } = await import("../voice/voice-runtime"); + let finishPlugin!: () => void; + let cueQueued!: () => void; + const queued = new Promise((resolve) => { + cueQueued = resolve; + }); + const output = createPlaybackQueue<{ type: string }>( + (_source, signal) => + new Promise((resolve, reject) => { + finishPlugin = () => resolve(1); + signal.addEventListener("abort", () => reject(new Error("Playback stopped")), { + once: true, + }); + }), + ); + const engine = { + initialize: async () => {}, + destroy: async () => { + output.destroy(); + }, + startCapture: async () => {}, + stopCapture: async () => {}, + toggleMute: () => false, + isMuted: () => false, + play(source: { type: string }, signal?: AbortSignal) { + if (source.type.startsWith("audio/pcm")) cueQueued(); + return output.play(source, signal); + }, + stop: output.stop, + clearQueue: output.clearQueue, + isPlaying: output.isPlaying, + }; + const runtime = createVoiceRuntime({ + engine, + getServerInfo: () => ({ + serverId: "host", + hostname: "host", + version: "1.0.0", + capabilities: { + voice: { dictation: { enabled: true, reason: "" }, voice: { enabled: true, reason: "" } }, + }, + }), + activateKeepAwake: async () => {}, + deactivateKeepAwake: async () => {}, + }); + runtime.registerSession({ + serverId: "host", + setVoiceMode: async () => {}, + sendVoiceAudioChunk: async () => {}, + audioPlayed: async () => {}, + abortRequest: async () => {}, + setAssistantAudioPlaying: () => {}, + }); + try { + await runtime.startVoice("host", "agent"); + const result = engine.play({ type: "audio/wav" }).then( + () => "finished", + () => "stopped", + ); + runtime.onTurnEvent("host", "agent", "turn_started"); + await queued; + runtime.onAssistantAudioStarted("host"); + finishPlugin(); + expect(await result).toBe("finished"); + } finally { + await runtime.destroy(); + } +}); diff --git a/packages/app/src/audio/playback.ts b/packages/app/src/audio/playback.ts new file mode 100644 index 0000000000..8d0744b79c --- /dev/null +++ b/packages/app/src/audio/playback.ts @@ -0,0 +1,81 @@ +/** One queue for voice and plugin audio, including cancellation while loading. */ +export function createPlaybackQueue( + start: (source: Source, signal: AbortSignal) => Promise, + onIdle: () => void = () => {}, +) { + interface Item { + source: Source; + controller: AbortController; + resolve: (duration: number) => void; + reject: (error: unknown) => void; + release: () => void; + } + const pending: Item[] = []; + let active: Item | null = null; + let destroyed = false; + + async function drain() { + if (active) return; + const item = pending.shift(); + if (!item) { + onIdle(); + return; + } + active = item; + try { + if (item.controller.signal.aborted) throw new Error("Playback stopped"); + item.resolve(await start(item.source, item.controller.signal)); + } catch (error) { + item.reject(error); + } finally { + item.release(); + active = null; + void drain(); + } + } + + const stopped = () => new Error("Playback stopped"); + return { + play(source: Source, signal?: AbortSignal): Promise { + if (destroyed) return Promise.reject(new Error("Audio engine destroyed")); + if (signal?.aborted) return Promise.reject(new Error("Playback stopped")); + return new Promise((resolve, reject) => { + const controller = new AbortController(); + const cancel = () => { + controller.abort(); + const index = pending.indexOf(item); + if (index !== -1) { + pending.splice(index, 1); + item.release(); + reject(new Error("Playback stopped")); + } + }; + const item: Item = { + source, + controller, + resolve, + reject, + release: () => signal?.removeEventListener("abort", cancel), + }; + signal?.addEventListener("abort", cancel, { once: true }); + pending.push(item); + void drain(); + }); + }, + stop() { + active?.controller.abort(); + }, + clearQueue() { + for (const item of pending.splice(0)) { + item.release(); + item.reject(stopped()); + } + }, + isPlaying: () => active !== null, + destroy() { + destroyed = true; + this.clearQueue(); + this.stop(); + }, + }; +} diff --git a/packages/app/src/contexts/session-context.tsx b/packages/app/src/contexts/session-context.tsx index 6a2c42a617..3e8165146c 100644 --- a/packages/app/src/contexts/session-context.tsx +++ b/packages/app/src/contexts/session-context.tsx @@ -29,7 +29,7 @@ import type { DaemonClient } from "@getpaseo/client/internal/daemon-client"; import type { AgentPermissionResponse } from "@getpaseo/protocol/agent-types"; import { getHostRuntimeStore, useHostRuntimeIsConnected } from "@/runtime/host-runtime"; import { useVoiceAudioEngineOptional, useVoiceRuntimeOptional } from "@/contexts/voice-context"; -import type { AudioPlaybackSource } from "@/voice/audio-engine-types"; +import type { AudioPlaybackSource } from "@/audio"; import { selectAgentTimelineState, toDaemonServerInfo, diff --git a/packages/app/src/contexts/voice-context.tsx b/packages/app/src/contexts/voice-context.tsx index 7bb0b9b7e1..4c8d04e2f1 100644 --- a/packages/app/src/contexts/voice-context.tsx +++ b/packages/app/src/contexts/voice-context.tsx @@ -9,8 +9,8 @@ import { } from "react"; import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; import { useSessionStore } from "@/stores/session-store"; -import { createAudioEngine } from "@/voice/audio-engine"; -import type { AudioEngine } from "@/voice/audio-engine-types"; +import { createAudioEngine } from "@/audio"; +import type { AudioEngine } from "@/audio"; import { createVoiceRuntime, type VoiceRuntime, diff --git a/packages/app/src/hooks/use-dictation-audio-source.native.ts b/packages/app/src/hooks/use-dictation-audio-source.native.ts index aa50a8eead..293412fe1a 100644 --- a/packages/app/src/hooks/use-dictation-audio-source.native.ts +++ b/packages/app/src/hooks/use-dictation-audio-source.native.ts @@ -2,7 +2,7 @@ import { useCallback, useEffect, useRef } from "react"; import { Buffer } from "buffer"; import { useState } from "react"; -import { createAudioEngine } from "@/voice/audio-engine"; +import { createAudioEngine } from "@/audio"; import type { DictationAudioSource, diff --git a/packages/app/src/hooks/use-dictation-audio-source.web.ts b/packages/app/src/hooks/use-dictation-audio-source.web.ts index afb4b44499..c482bece18 100644 --- a/packages/app/src/hooks/use-dictation-audio-source.web.ts +++ b/packages/app/src/hooks/use-dictation-audio-source.web.ts @@ -1,3 +1,4 @@ +import { resampleToPcm16 } from "@/audio/pcm"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { parsePcm16Wav } from "@/utils/pcm16-wav"; import { isElectronRuntime } from "@/desktop/host"; @@ -17,41 +18,6 @@ const getAudioContextCtor = (): typeof AudioContext | null => { return ctor ?? null; }; -const floatToInt16 = (sample: number): number => { - const clamped = Math.max(-1, Math.min(1, sample)); - return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff); -}; - -const resampleToPcm16 = ( - input: Float32Array, - inputRate: number, - outputRate: number, -): Int16Array => { - if (input.length === 0) { - return new Int16Array(0); - } - if (inputRate === outputRate) { - const out = new Int16Array(input.length); - for (let i = 0; i < input.length; i++) { - out[i] = floatToInt16(input[i]); - } - return out; - } - - const ratio = inputRate / outputRate; - const outputLength = Math.max(1, Math.round(input.length / ratio)); - const out = new Int16Array(outputLength); - for (let i = 0; i < outputLength; i++) { - const sourceIndex = i * ratio; - const i0 = Math.floor(sourceIndex); - const i1 = Math.min(input.length - 1, i0 + 1); - const frac = sourceIndex - i0; - const sample = input[i0] * (1 - frac) + input[i1] * frac; - out[i] = floatToInt16(sample); - } - return out; -}; - const concatInt16 = (a: Int16Array, b: Int16Array): Int16Array => { if (a.length === 0) { return b; diff --git a/packages/app/src/plugins/catalog-sync.tsx b/packages/app/src/plugins/catalog-sync.tsx index 484124971e..0eedfde3ab 100644 --- a/packages/app/src/plugins/catalog-sync.tsx +++ b/packages/app/src/plugins/catalog-sync.tsx @@ -1,3 +1,4 @@ +import { useVoiceAudioEngineOptional } from "@/contexts/voice-context"; import { pluginSettingsKey } from "./settings/use-settings"; import { useEffect } from "react"; import type { DaemonClient } from "@getpaseo/client/internal/daemon-client"; @@ -12,13 +13,14 @@ export function PluginCatalogSync({ serverId: string; client: DaemonClient; }) { + const audio = useVoiceAudioEngineOptional(); const connected = useHostRuntimeIsConnected(serverId); const supported = useHostFeature(serverId, "plugins"); useEffect(() => { let cancelled = false; let refreshQueue = Promise.resolve(); - if (!supported) { + if (!supported || !audio) { pluginRegistry.removeHost(serverId); return; } @@ -35,6 +37,7 @@ export function PluginCatalogSync({ pluginRegistry.installCatalog(serverId, catalog, { replacePluginId, client, + audio, }); } return undefined; @@ -79,7 +82,7 @@ export function PluginCatalogSync({ .release() .catch((error) => console.warn("[Plugins] Failed to release catalog", error)); }; - }, [client, connected, serverId, supported]); + }, [audio, client, connected, serverId, supported]); useEffect(() => () => pluginRegistry.removeHost(serverId), [serverId]); return null; diff --git a/packages/app/src/plugins/client-runtime.ts b/packages/app/src/plugins/client-runtime.ts index 68534a0579..f9074383bd 100644 --- a/packages/app/src/plugins/client-runtime.ts +++ b/packages/app/src/plugins/client-runtime.ts @@ -1,3 +1,4 @@ +import { createPlayAudio, type AudioEngine } from "@/audio"; import { getHostRuntimeStore } from "@/runtime/host-runtime"; import { createPluginHosts } from "./hosts"; import type { PluginClientOpenPanelOptions } from "@getpaseo/plugin/client"; @@ -12,7 +13,10 @@ import { createPluginNavigation } from "./navigation"; import { pluginButtonStore } from "./buttons"; import type { InstalledPlugin } from "./types"; -export function createPluginClientRuntime(installation: InstalledPlugin): PluginClientRuntime { +export function createPluginClientRuntime( + installation: InstalledPlugin, + audio: Pick, +): PluginClientRuntime { const state = createPluginClientStateSource(installation.serverId); const capabilities = createPluginCapabilities( installation, @@ -20,6 +24,7 @@ export function createPluginClientRuntime(installation: InstalledPlugin): Plugin ); return { ...capabilities, + playAudio: createPlayAudio(audio, installation.lifetime.signal), hosts: createPluginHosts(getHostRuntimeStore(), installation.lifetime.signal), addComposerPill(contribution) { return pluginButtonStore.addComposerPill(installation, contribution); diff --git a/packages/app/src/plugins/evaluate.ts b/packages/app/src/plugins/evaluate.ts index ecb8f89084..a2703b6ad2 100644 --- a/packages/app/src/plugins/evaluate.ts +++ b/packages/app/src/plugins/evaluate.ts @@ -91,6 +91,7 @@ function requireScreenTitle(screenId: string, title: unknown): PluginScreenTitle export type PluginClientRuntime = Pick< PluginClientContext, + | "playAudio" | "paseo" | "rpc" | "openSettings" diff --git a/packages/app/src/plugins/registry-lifecycle.test.ts b/packages/app/src/plugins/registry-lifecycle.test.ts index 445269c87e..ecec420e3c 100644 --- a/packages/app/src/plugins/registry-lifecycle.test.ts +++ b/packages/app/src/plugins/registry-lifecycle.test.ts @@ -4,6 +4,8 @@ import type { PaseoApi } from "@getpaseo/client"; import { expect, test } from "vitest"; import { PluginRegistry } from "./registry"; +const audio = { play: async () => 0 }; + /** A host client that records open event observations by their first event name. */ function observingClient() { const open = new Set(); @@ -47,6 +49,7 @@ function registry() { openScreen: () => {}, openSurface: () => {}, openSettings: () => {}, + playAudio: async () => {}, openPanel: () => {}, addComposerPill: () => ({ update() {}, remove() {} }), addHeaderButton: () => ({ update() {}, remove() {} }), @@ -67,7 +70,7 @@ function catalog(id: string, body = "return function() {};") { test("setup and every surface share the installation's one Paseo client", () => { const h = registry(); - h.plugins.installCatalog("host", [catalog("deploys")], { client: h.client }); + h.plugins.installCatalog("host", [catalog("deploys")], { client: h.client, audio }); const [installation] = h.plugins.getSnapshot(); expect(h.setupClients.get("deploys")).toBe(installation.paseo); @@ -76,11 +79,14 @@ test("setup and every surface share the installation's one Paseo client", () => test("teardown ends every subscription the plugin still holds", async () => { const h = registry(); - h.plugins.installCatalog("host", [catalog("disabled"), catalog("kept")], { client: h.client }); + h.plugins.installCatalog("host", [catalog("disabled"), catalog("kept")], { + client: h.client, + audio, + }); expect(h.open).toEqual(new Set(["disabled", "kept"])); // Disabling one plugin removes it from the catalog. - h.plugins.installCatalog("host", [catalog("kept")], { client: h.client }); + h.plugins.installCatalog("host", [catalog("kept")], { client: h.client, audio }); await expect.poll(() => [...h.open]).toEqual(["kept"]); h.plugins.removeHost("host"); @@ -89,11 +95,12 @@ test("teardown ends every subscription the plugin still holds", async () => { test("reloading a plugin ends the old installation's subscriptions", async () => { const h = registry(); - h.plugins.installCatalog("host", [catalog("reloaded")], { client: h.client }); + h.plugins.installCatalog("host", [catalog("reloaded")], { client: h.client, audio }); const [first] = h.plugins.getSnapshot(); h.plugins.installCatalog("host", [catalog("reloaded")], { client: h.client, + audio, replacePluginId: "reloaded", }); @@ -109,6 +116,7 @@ test("failed plugin initialization ends the subscriptions it opened", async () = const h = registry(); h.plugins.installCatalog("host", [catalog("failed", 'throw new Error("setup failed");')], { client: h.client, + audio, }); await expect.poll(() => h.open.size).toBe(0); expect(h.plugins.getSnapshot()).toEqual([]); @@ -118,8 +126,8 @@ test("unloading ends the subscriptions even when plugin cleanup throws, preservi const h = registry(); const failing = catalog("failing", 'return function() { throw new Error("cleanup failed"); };'); const surviving = catalog("surviving", "return function() {};"); - h.plugins.installCatalog("host", [failing, surviving], { client: h.client }); - h.plugins.installCatalog("host", [surviving], { client: h.client }); + h.plugins.installCatalog("host", [failing, surviving], { client: h.client, audio }); + h.plugins.installCatalog("host", [surviving], { client: h.client, audio }); await expect.poll(() => [...h.open]).toEqual(["surviving"]); expect(h.plugins.getSnapshot().map((plugin) => plugin.id)).toEqual(["surviving"]); h.plugins.removeHost("host"); @@ -131,7 +139,7 @@ test("an invalid async client entry is disposed and its rejected continuation is h.plugins.installCatalog( "host", [catalog("async-entry", 'return Promise.reject(new Error("asynchronous setup failed"));')], - { client: h.client }, + { client: h.client, audio }, ); await expect.poll(() => h.open.size).toBe(0); await new Promise((resolve) => setTimeout(resolve, 0)); diff --git a/packages/app/src/plugins/registry-requirements.test.ts b/packages/app/src/plugins/registry-requirements.test.ts index 055fe207cd..eb141e94c2 100644 --- a/packages/app/src/plugins/registry-requirements.test.ts +++ b/packages/app/src/plugins/registry-requirements.test.ts @@ -3,6 +3,8 @@ import { afterEach, expect, it } from "vitest"; import { DaemonClient } from "@getpaseo/client/internal/daemon-client"; import { PluginRegistry } from "./registry"; +const audio = { play: async () => 0 }; + const client = new DaemonClient({ url: "ws://unused.test", clientId: "plugin-requirements-test" }); const registries: PluginRegistry[] = []; function registry(version: string) { @@ -31,6 +33,7 @@ function registry(version: string) { }, openScreen() {}, openSurface() {}, + async playAudio() {}, openPanel() {}, addComposerPill: () => ({ update() {}, remove() {} }), addHeaderButton: () => ({ update() {}, remove() {} }), @@ -57,7 +60,7 @@ it("checks the app version before creating a runtime or evaluating plugin code", clientBundle: "throw new Error('executed')", }, ], - { client }, + { client, audio }, ); expect(starts()).toBe(0); expect(result.getSnapshot()).toEqual([]); @@ -68,7 +71,7 @@ it("checks the app version before creating a runtime or evaluating plugin code", it("rejects catalogs without requirements from pre-0.8 daemons", () => { const { result, starts } = registry("0.8.0"); - result.installCatalog("host", [{ id: "example", clientBundle }], { client }); + result.installCatalog("host", [{ id: "example", clientBundle }], { client, audio }); expect(starts()).toBe(0); expect(result.getEvaluationError("host", "example")).toContain( "https://paseo.sh/docs/plugins/migration", @@ -80,6 +83,7 @@ it("unloads on a requirement-only edit and recovers after correction", () => { const install = (paseo: string) => result.installCatalog("host", [{ id: "example", clientBundle, requirements: { paseo } }], { client, + audio, }); install("^0.8.0"); expect(result.getSnapshot().map(({ id }) => id)).toEqual(["example"]); diff --git a/packages/app/src/plugins/registry.test.ts b/packages/app/src/plugins/registry.test.ts index 45e1ba546f..9e8d246a95 100644 --- a/packages/app/src/plugins/registry.test.ts +++ b/packages/app/src/plugins/registry.test.ts @@ -3,6 +3,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { DaemonClient } from "@getpaseo/client/internal/daemon-client"; import { pluginRegistry as registry } from "./registry"; +const audio = { play: async () => 0 }; + vi.mock("./navigation", () => ({ createPluginNavigation: () => ({}), })); @@ -30,7 +32,7 @@ const pluginRegistry = { return registry.installCatalog( serverId, catalog.map((entry) => ({ ...entry, requirements: { paseo: `>=${appPackage.version}` } })), - { ...options, client: daemonClient }, + { ...options, client: daemonClient, audio }, ); }, }; diff --git a/packages/app/src/plugins/registry.ts b/packages/app/src/plugins/registry.ts index eed76a530f..e349446bfe 100644 --- a/packages/app/src/plugins/registry.ts +++ b/packages/app/src/plugins/registry.ts @@ -1,3 +1,4 @@ +import type { AudioEngine } from "@/audio"; import { useMemo, useSyncExternalStore } from "react"; import { QueryClient } from "@tanstack/react-query"; import { createPaseoApi } from "@getpaseo/client"; @@ -41,6 +42,7 @@ export class PluginRegistry { options: { replacePluginId?: string; client: DaemonClient; + audio: Pick; }, ): boolean { const previous = this.byHost.get(serverId) ?? []; @@ -99,7 +101,7 @@ export class PluginRegistry { timelineTransformers: [], timelineRenderers: [], }; - const runtime = this.dependencies.createRuntime(installation); + const runtime = this.dependencies.createRuntime(installation, options.audio); const evaluated = runPluginClientBundle(entry.id, entry.clientBundle, runtime, () => this.publish(), ); diff --git a/packages/app/src/plugins/timeline/view.test.tsx b/packages/app/src/plugins/timeline/view.test.tsx index af1e711eb5..d915216105 100644 --- a/packages/app/src/plugins/timeline/view.test.tsx +++ b/packages/app/src/plugins/timeline/view.test.tsx @@ -39,6 +39,8 @@ vi.mock("../icons", () => ({ import { pluginRegistry } from "../registry"; import { PluginTimelineItemView } from "./view"; +const audio = { play: async () => 0 }; + const bundle = `(function(require) { const React = require("react"); return { default: function(plugin) { @@ -122,6 +124,7 @@ describe("PluginTimelineItemView", () => { [{ id: "reports", requirements: { paseo: `>=${appPackage.version}` }, clientBundle: bundle }], { client: daemonClient, + audio, }, ); @@ -151,6 +154,7 @@ describe("PluginTimelineItemView", () => { ], { client: daemonClient, + audio, }, ); const container = document.createElement("div"); @@ -188,6 +192,7 @@ describe("PluginTimelineItemView", () => { ], { client: daemonClient, + audio, }, ); const container = document.createElement("div"); @@ -310,7 +315,7 @@ it("a renderer releases its observation on the plugin's client when it crashes, clientBundle: liveBundle, }, ], - { client }, + { client, audio }, ); const container = document.createElement("div"); containers.push(container); diff --git a/packages/app/src/plugins/workspace-panels/locations.test.ts b/packages/app/src/plugins/workspace-panels/locations.test.ts index b475dd501d..af5cade055 100644 --- a/packages/app/src/plugins/workspace-panels/locations.test.ts +++ b/packages/app/src/plugins/workspace-panels/locations.test.ts @@ -4,6 +4,8 @@ import type { DaemonClient } from "@getpaseo/client/internal/daemon-client"; import { pluginRegistry } from "../registry"; import { panelTargetSupportsHost, resolvePluginPanelOpenLocation } from "./locations"; +const audio = { play: async () => 0 }; + vi.mock("../navigation", () => ({ createPluginNavigation: () => ({}), })); @@ -37,6 +39,7 @@ function install(locations: readonly ("workspace" | "explorer")[]) { [{ id: "review", requirements: { paseo: `>=${appPackage.version}` }, clientBundle: bundle }], { client: {} as DaemonClient, + audio, }, ); return pluginRegistry.getSnapshot()[0]!; diff --git a/packages/app/src/voice/audio-engine.native.ts b/packages/app/src/voice/audio-engine.native.ts deleted file mode 100644 index 465bab3c73..0000000000 --- a/packages/app/src/voice/audio-engine.native.ts +++ /dev/null @@ -1,350 +0,0 @@ -import type { - AudioEngine, - AudioEngineCallbacks, - AudioPlaybackSource, -} from "@/voice/audio-engine-types"; - -interface QueuedAudio { - audio: AudioPlaybackSource; - resolve: (duration: number) => void; - reject: (error: Error) => void; -} - -interface AudioEngineTraceOptions { - traceLabel?: string; -} - -function parsePcmSampleRate(mimeType: string): number | null { - const match = /rate=(\d+)/i.exec(mimeType); - if (!match) { - return null; - } - const rate = Number(match[1]); - return Number.isFinite(rate) && rate > 0 ? rate : null; -} - -function resamplePcm16(pcm: Uint8Array, fromRate: number, toRate: number): Uint8Array { - if (fromRate === toRate) { - return pcm; - } - - const inputSamples = Math.floor(pcm.length / 2); - const outputSamples = Math.floor((inputSamples * toRate) / fromRate); - const out = new Uint8Array(outputSamples * 2); - const ratio = fromRate / toRate; - - const readInt16 = (sampleIndex: number): number => { - const i = sampleIndex * 2; - if (i + 1 >= pcm.length) { - return 0; - } - const lo = pcm[i]; - const hi = pcm[i + 1]; - let value = (hi << 8) | lo; - if (value & 0x8000) { - value = value - 0x10000; - } - return value; - }; - - const writeInt16 = (sampleIndex: number, value: number): void => { - const clamped = Math.max(-32768, Math.min(32767, Math.round(value))); - const i = sampleIndex * 2; - out[i] = clamped & 0xff; - out[i + 1] = (clamped >> 8) & 0xff; - }; - - for (let i = 0; i < outputSamples; i++) { - const srcPos = i * ratio; - const i0 = Math.floor(srcPos); - const frac = srcPos - i0; - const s0 = readInt16(i0); - const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1)); - writeInt16(i, s0 + (s1 - s0) * frac); - } - - return out; -} - -export function createAudioEngine( - callbacks: AudioEngineCallbacks, - _options?: AudioEngineTraceOptions, -): AudioEngine { - const native = require("@getpaseo/expo-two-way-audio"); - - const refs: { - initialized: boolean; - captureActive: boolean; - muted: boolean; - queue: QueuedAudio[]; - processingQueue: boolean; - playbackTimeout: ReturnType | null; - activePlayback: { - resolve: (duration: number) => void; - reject: (error: Error) => void; - settled: boolean; - } | null; - destroyed: boolean; - } = { - initialized: false, - captureActive: false, - muted: false, - queue: [], - processingQueue: false, - playbackTimeout: null, - activePlayback: null, - destroyed: false, - }; - - const microphoneSubscription = native.addExpoTwoWayAudioEventListener( - "onMicrophoneData", - (event: { data: Uint8Array }) => { - if (!refs.captureActive || refs.muted) { - return; - } - const pcm = event.data; - callbacks.onCaptureData(pcm); - }, - ); - const volumeSubscription = native.addExpoTwoWayAudioEventListener( - "onInputVolumeLevelData", - (event: { data: number }) => { - if (!refs.captureActive) { - return; - } - const level = refs.muted ? 0 : event.data; - callbacks.onVolumeLevel(level); - }, - ); - const interruptionSubscription = native.addExpoTwoWayAudioEventListener( - "onAudioInterruption", - (event: { data: string }) => { - if (event.data !== "blocked") { - return; - } - const wasCaptureActive = refs.captureActive; - refs.captureActive = false; - refs.muted = false; - callbacks.onVolumeLevel(0); - if (wasCaptureActive) { - callbacks.onInterruption?.(); - } - }, - ); - - async function ensureInitialized(): Promise { - if (refs.initialized) { - return; - } - const success = await native.initialize(); - if (!success) { - throw new Error("expo-two-way-audio: native initialize() returned false"); - } - refs.initialized = true; - } - - /** - * Release the OS audio session as soon as we are neither capturing nor playing. - * Holding it keeps the user's background music paused — on iOS the non-mixing - * `.playAndRecord` category survives backgrounding and is re-asserted on every - * foreground, so an unreleased session means their music never comes back. - */ - function releaseSessionIfIdle(): void { - if (!refs.initialized || refs.destroyed) { - return; - } - if (refs.captureActive || refs.activePlayback || refs.queue.length > 0) { - return; - } - // The wrapper no-ops on binaries whose native module predates this function. - native.releaseAudioSession(); - } - - async function ensureMicrophonePermission(): Promise { - let permission = await native.getMicrophonePermissionsAsync().catch(() => null); - if (!permission?.granted) { - permission = await native.requestMicrophonePermissionsAsync().catch(() => null); - } - if (!permission?.granted) { - throw new Error( - "Microphone permission is required to capture audio. Please enable microphone access in system settings.", - ); - } - } - - function clearPlaybackTimeout(): void { - if (refs.playbackTimeout) { - clearTimeout(refs.playbackTimeout); - refs.playbackTimeout = null; - } - } - - async function playAudio(audio: AudioPlaybackSource): Promise { - await ensureInitialized(); - - return await new Promise((resolve, reject) => { - refs.activePlayback = { resolve, reject, settled: false }; - - audio - .arrayBuffer() - .then((arrayBuffer) => { - const pcm = new Uint8Array(arrayBuffer); - const inputRate = parsePcmSampleRate(audio.type || "") ?? 24000; - - // Native AudioEngine expects 16kHz PCM16 - const pcm16k = resamplePcm16(pcm, inputRate, 16000); - const durationSec = pcm16k.length / 2 / 16000; - - native.resumePlayback(); - native.playPCMData(pcm16k); - - clearPlaybackTimeout(); - refs.playbackTimeout = setTimeout(() => { - clearPlaybackTimeout(); - const active = refs.activePlayback; - if (!active || active.settled) { - return; - } - active.settled = true; - refs.activePlayback = null; - resolve(durationSec); - }, durationSec * 1000); - return undefined; - }) - .catch((error: unknown) => { - clearPlaybackTimeout(); - const active = refs.activePlayback; - if (active && !active.settled) { - active.settled = true; - refs.activePlayback = null; - reject(error instanceof Error ? error : new Error(String(error))); - } - }); - }); - } - - async function processQueue(): Promise { - if (refs.processingQueue || refs.queue.length === 0) { - return; - } - - refs.processingQueue = true; - while (refs.queue.length > 0) { - const item = refs.queue.shift()!; - try { - const duration = await playAudio(item.audio); - item.resolve(duration); - } catch (error) { - item.reject(error instanceof Error ? error : new Error(String(error))); - } - } - refs.processingQueue = false; - releaseSessionIfIdle(); - } - - return { - async initialize() { - await ensureInitialized(); - }, - - async destroy() { - if (refs.destroyed) { - return; - } - refs.destroyed = true; - this.stop(); - this.clearQueue(); - if (refs.captureActive) { - native.toggleRecording(false); - refs.captureActive = false; - } - clearPlaybackTimeout(); - refs.muted = false; - callbacks.onVolumeLevel(0); - if (refs.initialized) { - native.tearDown(); - refs.initialized = false; - } - microphoneSubscription.remove(); - volumeSubscription.remove(); - interruptionSubscription.remove(); - }, - - async startCapture() { - if (refs.captureActive) { - return; - } - - try { - await ensureMicrophonePermission(); - await ensureInitialized(); - const isRecording = native.toggleRecording(true); - if (!isRecording) { - throw new Error( - "Microphone capture could not start because Android audio focus is unavailable.", - ); - } - refs.captureActive = true; - } catch (error) { - const wrapped = error instanceof Error ? error : new Error(String(error)); - callbacks.onError?.(wrapped); - throw wrapped; - } - }, - - async stopCapture() { - if (refs.captureActive) { - native.toggleRecording(false); - } - refs.captureActive = false; - refs.muted = false; - callbacks.onVolumeLevel(0); - releaseSessionIfIdle(); - }, - - toggleMute() { - refs.muted = !refs.muted; - if (refs.muted) { - callbacks.onVolumeLevel(0); - } - return refs.muted; - }, - - isMuted() { - return refs.muted; - }, - - async play(audio: AudioPlaybackSource) { - return await new Promise((resolve, reject) => { - refs.queue.push({ audio, resolve, reject }); - if (!refs.processingQueue) { - void processQueue(); - } - }); - }, - - stop() { - native.stopPlayback(); - clearPlaybackTimeout(); - const active = refs.activePlayback; - refs.activePlayback = null; - if (active && !active.settled) { - active.settled = true; - active.reject(new Error("Playback stopped")); - } - releaseSessionIfIdle(); - }, - - clearQueue() { - while (refs.queue.length > 0) { - refs.queue.shift()!.reject(new Error("Playback stopped")); - } - refs.processingQueue = false; - releaseSessionIfIdle(); - }, - - isPlaying() { - return refs.activePlayback !== null; - }, - }; -} diff --git a/packages/app/src/voice/voice-runtime.test.ts b/packages/app/src/voice/voice-runtime.test.ts index eb0e8c88f3..189140c7f6 100644 --- a/packages/app/src/voice/voice-runtime.test.ts +++ b/packages/app/src/voice/voice-runtime.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { DaemonServerInfo } from "@/stores/session-store"; -import type { AudioEngine } from "@/voice/audio-engine-types"; +import type { AudioEngine } from "@/audio"; import { createVoiceRuntime, type VoiceSessionAdapter } from "@/voice/voice-runtime"; import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config"; @@ -307,8 +307,9 @@ describe("voice runtime", () => { runtime.onServerSpeechStateChanged("server-1", true); - expect(engine.stop).toHaveBeenCalledTimes(1); - expect(engine.clearQueue).toHaveBeenCalledTimes(1); + expect(engine.stop).not.toHaveBeenCalled(); + expect(engine.clearQueue).not.toHaveBeenCalled(); + expect(vi.mocked(engine.play).mock.calls[0][1]?.aborted).toBe(true); resolvePlay(0.1); }); diff --git a/packages/app/src/voice/voice-runtime.ts b/packages/app/src/voice/voice-runtime.ts index a8281acbc3..3b7843e2f7 100644 --- a/packages/app/src/voice/voice-runtime.ts +++ b/packages/app/src/voice/voice-runtime.ts @@ -2,7 +2,7 @@ import { Buffer } from "buffer"; import type { AgentStreamEventPayload, SessionOutboundMessage } from "@getpaseo/protocol/messages"; import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities"; import type { DaemonServerInfo } from "@/stores/session-store"; -import type { AudioEngine } from "@/voice/audio-engine-types"; +import type { AudioEngine } from "@/audio"; import { THINKING_TONE_NATIVE_PCM_BASE64, THINKING_TONE_NATIVE_PCM_DURATION_MS, @@ -118,10 +118,8 @@ interface RuntimePlaybackState { } interface CueState { - active: boolean; - token: number; + controller: AbortController | null; timeout: ReturnType | null; - playing: boolean; } const INITIAL_SNAPSHOT: VoiceRuntimeSnapshot = { @@ -210,10 +208,8 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { generation: 0, }; const cue: CueState = { - active: false, - token: 0, + controller: null, timeout: null, - playing: false, }; const cuePcm16 = Uint8Array.from(Buffer.from(THINKING_TONE_NATIVE_PCM_BASE64, "base64")); const cueSource = { @@ -442,18 +438,12 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { } function stopCue(): void { - const hadActive = cue.active || cue.timeout !== null || cue.playing; - cue.active = false; - cue.token += 1; + cue.controller?.abort(); + cue.controller = null; if (cue.timeout) { clearTimeout(cue.timeout); cue.timeout = null; } - cue.playing = false; - if (hadActive) { - deps.engine.stop(); - deps.engine.clearQueue(); - } } function resetCaptureTelemetry(): void { @@ -467,29 +457,26 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { stopCue(); return; } - if (cue.active) { + if (cue.controller) { return; } - cue.active = true; - cue.token += 1; - const token = cue.token; + const controller = new AbortController(); + cue.controller = controller; const playNext = () => { - if (!cue.active || cue.token !== token) { + if (controller.signal.aborted) { return; } - cue.playing = true; void deps.engine - .play(cueSource) + .play(cueSource, controller.signal) .catch((error) => { - if (cue.token !== token) { + if (controller.signal.aborted) { return; } console.warn(`[VoiceRuntime#${instanceId}] Cue playback failed:`, error); }) .finally(() => { - cue.playing = false; - if (!cue.active || cue.token !== token) { + if (controller.signal.aborted) { return; } cue.timeout = setTimeout( @@ -920,10 +907,9 @@ export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime { if (isSpeaking) { const shouldInterruptPlayback = state.snapshot.phase === "playing" || playback.groups.size > 0; - const hadCue = cue.active || cue.timeout !== null || cue.playing; resetPlaybackState(); stopCue(); - if (shouldInterruptPlayback && !hadCue) { + if (shouldInterruptPlayback) { deps.engine.stop(); deps.engine.clearQueue(); } diff --git a/packages/plugin/src/client/contracts.ts b/packages/plugin/src/client/contracts.ts index 239b3580ee..5bc82d8a58 100644 --- a/packages/plugin/src/client/contracts.ts +++ b/packages/plugin/src/client/contracts.ts @@ -131,6 +131,8 @@ interface PluginClientContextAliases { } export interface PluginClientContext extends PluginCommandCapabilities, PluginClientContextAliases { + /** Play a base64-encoded audio file on this client; resolves when playback ends. */ + playAudio(source: { base64: string; mimeType: string }): Promise; addSettingsScreen(contribution: PluginSettingsScreenContribution): PluginCleanup; addScreen(contribution: PluginScreenContribution): PluginCleanup; addSidebarHeaderItem(contribution: PluginSidebarItemContribution): PluginCleanup; diff --git a/public-docs/plugins/reference.md b/public-docs/plugins/reference.md index fc473d99d0..7eece2810c 100644 --- a/public-docs/plugins/reference.md +++ b/public-docs/plugins/reference.md @@ -166,6 +166,29 @@ The scaffold's `tsconfig.json` omits the DOM library. Keep DOM globals out of cr components; do not add `/// ` or `"DOM"` to `lib`. `layout.platform` carries the same value as React Native's `Platform.OS` for rendering decisions. +### Play audio + +Call `client.playAudio({ base64, mimeType }): Promise` to play an audio file on the +current client (browser, Electron, iOS, or Android). Pass the base64 file returned by your +plugin RPC; no browser globals or platform checks are needed. + +```ts +const audio = await client.rpc(renderSpeech, { text: "Your agent needs you" }); +// renderSpeech returns { base64: string, mimeType: "audio/wav" }. +await client.playAudio(audio); +``` + +The promise resolves when playback finishes and rejects if the file is invalid, playback +fails, or the plugin unloads. Calls share Paseo's voice playback queue and play in order. +Unloading a plugin cancels its active and queued audio. Voice playback controls can also +interrupt that shared queue. Playback does not request microphone permission. + +Use PCM WAV or MP3 for portable files. Other codecs depend on the client's decoder. +MIME parameters are accepted. Pass a complete audio file; raw PCM samples are not supported. +Browsers require user interaction before allowing sound; handle rejection and offer a +play button. The function plays on the device running the plugin client, not on the daemon, +and does not promise delivery while the app is suspended or closed. + ### External links and workspace browsers Use `ExternalLink` to open documentation outside Paseo: