From b1bbb7a1022e89bb5fc9a5a6ac4bcb222df07340 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Tue, 1 Sep 2026 14:54:40 +0200 Subject: [PATCH] fix(voice): initialize manual speech on demand --- docs/architecture.md | 9 ++- packages/server/src/server/bootstrap.ts | 5 +- .../providers/manual/provider.test.ts | 80 +++++++++++++++++++ .../voice-chat/providers/manual/provider.ts | 38 +++++---- 4 files changed, 111 insertions(+), 21 deletions(-) create mode 100644 packages/server/src/server/voice-chat/providers/manual/provider.test.ts diff --git a/docs/architecture.md b/docs/architecture.md index e0e32921cd..62bd802b1c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -124,10 +124,11 @@ Cross-platform React Native app that connects to one or more daemons. provider and matching client transport. Providers own prompts, credentials, hidden resources, tools, media integration, and cleanup. The manual provider creates a hidden orchestrator through the atomic required-Paseo-tools capability with an authoritative provider prompt, and owns a - separate speech runtime configured from the top-level `manualVoice` block. Dictation keeps its own - speech runtime. A complete manual orchestrator provider and mode selection is required; partial - configuration does not fall back to another provider or mode. Cleanup detaches work delegated by - the hidden orchestrator before archiving it. + separate speech runtime configured from the top-level `manualVoice` block. That runtime is created + on the first valid manual call and reused; ordinary daemon startup does not load or download voice + models. Dictation keeps its own speech runtime. A complete manual orchestrator provider and mode + selection is required; partial configuration does not fall back to another provider or mode. + Cleanup detaches work delegated by the hidden orchestrator before archiving it. - Voice calls end as soon as the final client socket disconnects. The containing client session may remain reconnectable, but calls are not replayed or rehydrated in this protocol slice. The sidebar footer starts the call and hosts its controls; routes and visible agents do not own its lifetime. diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index d0b566899c..754a16964c 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -1791,10 +1791,9 @@ export async function createPaseoDaemon( } }); - // Start speech service after listening so synchronous Sherpa native + // Start dictation after listening so synchronous Sherpa native // model loading doesn't block the server from accepting connections. dictationSpeechService.start(); - manualVoiceProvider.startHarness(); scriptHealthMonitor.start(); } catch (error) { unsubscribePluginProviders(); @@ -1825,7 +1824,7 @@ export async function createPaseoDaemon( await agentProviderRuntime.shutdown(); terminalManager.killAll(); await voiceChat?.close(); - await manualVoiceProvider.stopHarness(); + await manualVoiceProvider.close(); await dictationSpeechService.stop(); await scheduleService.stop().catch(() => undefined); await relayRuntime?.stop().catch(() => undefined); diff --git a/packages/server/src/server/voice-chat/providers/manual/provider.test.ts b/packages/server/src/server/voice-chat/providers/manual/provider.test.ts new file mode 100644 index 0000000000..202ec235ca --- /dev/null +++ b/packages/server/src/server/voice-chat/providers/manual/provider.test.ts @@ -0,0 +1,80 @@ +import pino from "pino"; +import { describe, expect, test, vi } from "vitest"; + +import type { AgentManager } from "../../../agent/agent-manager.js"; +import type { AgentStorage } from "../../../agent/agent-storage.js"; +import type { SpeechService } from "../../../speech/speech-runtime.js"; +import type { WorkspaceRegistry } from "../../../workspace-registry.js"; +import type { VoiceProviderStartInput } from "../../internal/provider.js"; +import { resolveManualVoiceConfig } from "./config.js"; +import { createManualVoiceProvider } from "./provider.js"; + +function createStartInput( + transportOffers: VoiceProviderStartInput["transportOffers"], +): VoiceProviderStartInput { + return { + callId: "voice-call", + context: { workspaceId: "workspace-id", agentId: null }, + transportOffers, + signal: new AbortController().signal, + emit: vi.fn(), + }; +} + +describe("manual voice provider speech lifecycle", () => { + test("starts one speech runtime only when the first valid call begins", async () => { + const startSpeech = vi.fn(); + const stopSpeech = vi.fn(async () => undefined); + const speech = { + start: startSpeech, + stop: stopSpeech, + ready: Promise.resolve(), + } as unknown as SpeechService; + const speechFactory = vi.fn(() => speech); + const launchInternalAgentWithRequiredPaseoTools = vi.fn(async () => { + throw new Error("stop after speech initialization"); + }); + const provider = createManualVoiceProvider({ + config: resolveManualVoiceConfig({ + paseoHome: "/tmp/manual-voice-provider-test", + persisted: { + orchestrator: { provider: "codex", modeId: "full-access" }, + }, + providers: undefined, + }), + agentManager: { + getAgent: vi.fn(() => undefined), + launchInternalAgentWithRequiredPaseoTools, + } as unknown as AgentManager, + agentStorage: {} as AgentStorage, + workspaceRegistry: { + get: vi.fn(async () => ({ + workspaceId: "workspace-id", + cwd: "/tmp/workspace", + archivedAt: null, + })), + } as unknown as Pick, + logger: pino({ level: "silent" }), + speechFactory, + }); + + expect(provider.getReadiness()).toEqual({ ready: true }); + expect(speechFactory).not.toHaveBeenCalled(); + + await expect(provider.start(createStartInput([]))).rejects.toThrow( + "The client does not support daemon audio", + ); + expect(speechFactory).not.toHaveBeenCalled(); + + const validInput = createStartInput([{ kind: "daemon-audio" }]); + await expect(provider.start(validInput)).rejects.toThrow("stop after speech initialization"); + await expect(provider.start(validInput)).rejects.toThrow("stop after speech initialization"); + + expect(speechFactory).toHaveBeenCalledOnce(); + expect(startSpeech).toHaveBeenCalledOnce(); + expect(launchInternalAgentWithRequiredPaseoTools).toHaveBeenCalledTimes(2); + + await provider.close(); + expect(stopSpeech).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/server/src/server/voice-chat/providers/manual/provider.ts b/packages/server/src/server/voice-chat/providers/manual/provider.ts index 4042539140..402095313a 100644 --- a/packages/server/src/server/voice-chat/providers/manual/provider.ts +++ b/packages/server/src/server/voice-chat/providers/manual/provider.ts @@ -35,12 +35,12 @@ export interface CreateManualVoiceProviderOptions { agentStorage: AgentStorage; workspaceRegistry: Pick; logger: Logger; + speechFactory?: typeof createSpeechService; } export interface ManualVoiceProvider extends VoiceCallProvider { configureOrchestrator(orchestrator: ManualVoiceOrchestratorSettings): void; - startHarness(): void; - stopHarness(): Promise; + close(): Promise; } const ManualClientTransportMessageSchema = z.discriminatedUnion("type", [ @@ -149,20 +149,27 @@ export function createManualVoiceProvider( options: CreateManualVoiceProviderOptions, ): ManualVoiceProvider { let orchestratorSettings = options.config.orchestrator; - const speech = createSpeechService({ - logger: options.logger, - openaiConfig: options.config.openai, - speechConfig: options.config.speech, - }); + let speech: ReturnType | null = null; + + function getSpeech() { + if (!speech) { + speech = (options.speechFactory ?? createSpeechService)({ + logger: options.logger, + openaiConfig: options.config.openai, + speechConfig: options.config.speech, + }); + speech.start(); + } + return speech; + } + return { configureOrchestrator(orchestrator) { orchestratorSettings = orchestrator; }, - startHarness() { - speech.start(); - }, - stopHarness() { - return speech.stop(); + async close() { + await speech?.stop(); + speech = null; }, getReadiness() { if (!resolveManualVoiceOrchestratorConfig(orchestratorSettings)) { @@ -171,14 +178,14 @@ export function createManualVoiceProvider( return { ready: true }; }, async start(input) { - return startManualVoiceCall(options, speech, orchestratorSettings, input); + return startManualVoiceCall(options, getSpeech, orchestratorSettings, input); }, }; } async function startManualVoiceCall( options: CreateManualVoiceProviderOptions, - speech: ReturnType, + getSpeech: () => ReturnType, orchestratorSettings: ManualVoiceOrchestratorSettings, input: VoiceProviderStartInput, ) { @@ -195,6 +202,9 @@ async function startManualVoiceCall( const orchestrator = resolveManualVoiceOrchestratorConfig(orchestratorSettings); if (!orchestrator) throw new Error("Configure the manual voice orchestrator and mode."); if (input.signal.aborted) throw new Error("Voice call start was canceled."); + const speech = getSpeech(); + await speech.ready; + if (input.signal.aborted) throw new Error("Voice call start was canceled."); const agentId = uuidv4(); let context = input.context;