fix(voice): initialize manual speech on demand

docs/architecture.md

@@ -124,10 +124,11 @@ Cross-platform React Native app that connects to one or more daemons.
   provider and matching client transport. Providers own prompts, credentials, hidden resources,
   tools, media integration, and cleanup. The manual provider creates a hidden orchestrator through
   the atomic required-Paseo-tools capability with an authoritative provider prompt, and owns a
-  separate speech runtime configured from the top-level `manualVoice` block. Dictation keeps its own
-  speech runtime. A complete manual orchestrator provider and mode selection is required; partial
-  configuration does not fall back to another provider or mode. Cleanup detaches work delegated by
-  the hidden orchestrator before archiving it.
+  separate speech runtime configured from the top-level `manualVoice` block. That runtime is created
+  on the first valid manual call and reused; ordinary daemon startup does not load or download voice
+  models. Dictation keeps its own speech runtime. A complete manual orchestrator provider and mode
+  selection is required; partial configuration does not fall back to another provider or mode.
+  Cleanup detaches work delegated by the hidden orchestrator before archiving it.
 - Voice calls end as soon as the final client socket disconnects. The containing client session may
   remain reconnectable, but calls are not replayed or rehydrated in this protocol slice. The sidebar
   footer starts the call and hosts its controls; routes and visible agents do not own its lifetime.

packages/server/src/server/bootstrap.ts

@@ -1791,10 +1791,9 @@ export async function createPaseoDaemon(
         }
       });
 
-      // Start speech service after listening so synchronous Sherpa native
+      // Start dictation after listening so synchronous Sherpa native
       // model loading doesn't block the server from accepting connections.
       dictationSpeechService.start();
-      manualVoiceProvider.startHarness();
       scriptHealthMonitor.start();
     } catch (error) {
       unsubscribePluginProviders();
@@ -1825,7 +1824,7 @@ export async function createPaseoDaemon(
     await agentProviderRuntime.shutdown();
     terminalManager.killAll();
     await voiceChat?.close();
-    await manualVoiceProvider.stopHarness();
+    await manualVoiceProvider.close();
     await dictationSpeechService.stop();
     await scheduleService.stop().catch(() => undefined);
     await relayRuntime?.stop().catch(() => undefined);

packages/server/src/server/voice-chat/providers/manual/provider.test.ts

@@ -0,0 +1,80 @@
+import pino from "pino";
+import { describe, expect, test, vi } from "vitest";
+
+import type { AgentManager } from "../../../agent/agent-manager.js";
+import type { AgentStorage } from "../../../agent/agent-storage.js";
+import type { SpeechService } from "../../../speech/speech-runtime.js";
+import type { WorkspaceRegistry } from "../../../workspace-registry.js";
+import type { VoiceProviderStartInput } from "../../internal/provider.js";
+import { resolveManualVoiceConfig } from "./config.js";
+import { createManualVoiceProvider } from "./provider.js";
+
+function createStartInput(
+  transportOffers: VoiceProviderStartInput["transportOffers"],
+): VoiceProviderStartInput {
+  return {
+    callId: "voice-call",
+    context: { workspaceId: "workspace-id", agentId: null },
+    transportOffers,
+    signal: new AbortController().signal,
+    emit: vi.fn(),
+  };
+}
+
+describe("manual voice provider speech lifecycle", () => {
+  test("starts one speech runtime only when the first valid call begins", async () => {
+    const startSpeech = vi.fn();
+    const stopSpeech = vi.fn(async () => undefined);
+    const speech = {
+      start: startSpeech,
+      stop: stopSpeech,
+      ready: Promise.resolve(),
+    } as unknown as SpeechService;
+    const speechFactory = vi.fn(() => speech);
+    const launchInternalAgentWithRequiredPaseoTools = vi.fn(async () => {
+      throw new Error("stop after speech initialization");
+    });
+    const provider = createManualVoiceProvider({
+      config: resolveManualVoiceConfig({
+        paseoHome: "/tmp/manual-voice-provider-test",
+        persisted: {
+          orchestrator: { provider: "codex", modeId: "full-access" },
+        },
+        providers: undefined,
+      }),
+      agentManager: {
+        getAgent: vi.fn(() => undefined),
+        launchInternalAgentWithRequiredPaseoTools,
+      } as unknown as AgentManager,
+      agentStorage: {} as AgentStorage,
+      workspaceRegistry: {
+        get: vi.fn(async () => ({
+          workspaceId: "workspace-id",
+          cwd: "/tmp/workspace",
+          archivedAt: null,
+        })),
+      } as unknown as Pick<WorkspaceRegistry, "get">,
+      logger: pino({ level: "silent" }),
+      speechFactory,
+    });
+
+    expect(provider.getReadiness()).toEqual({ ready: true });
+    expect(speechFactory).not.toHaveBeenCalled();
+
+    await expect(provider.start(createStartInput([]))).rejects.toThrow(
+      "The client does not support daemon audio",
+    );
+    expect(speechFactory).not.toHaveBeenCalled();
+
+    const validInput = createStartInput([{ kind: "daemon-audio" }]);
+    await expect(provider.start(validInput)).rejects.toThrow("stop after speech initialization");
+    await expect(provider.start(validInput)).rejects.toThrow("stop after speech initialization");
+
+    expect(speechFactory).toHaveBeenCalledOnce();
+    expect(startSpeech).toHaveBeenCalledOnce();
+    expect(launchInternalAgentWithRequiredPaseoTools).toHaveBeenCalledTimes(2);
+
+    await provider.close();
+    expect(stopSpeech).toHaveBeenCalledOnce();
+  });
+});

packages/server/src/server/voice-chat/providers/manual/provider.ts

@@ -35,12 +35,12 @@ export interface CreateManualVoiceProviderOptions {
   agentStorage: AgentStorage;
   workspaceRegistry: Pick<WorkspaceRegistry, "get">;
   logger: Logger;
+  speechFactory?: typeof createSpeechService;
 }
 
 export interface ManualVoiceProvider extends VoiceCallProvider {
   configureOrchestrator(orchestrator: ManualVoiceOrchestratorSettings): void;
-  startHarness(): void;
-  stopHarness(): Promise<void>;
+  close(): Promise<void>;
 }
 
 const ManualClientTransportMessageSchema = z.discriminatedUnion("type", [
@@ -149,20 +149,27 @@ export function createManualVoiceProvider(
   options: CreateManualVoiceProviderOptions,
 ): ManualVoiceProvider {
   let orchestratorSettings = options.config.orchestrator;
-  const speech = createSpeechService({
-    logger: options.logger,
-    openaiConfig: options.config.openai,
-    speechConfig: options.config.speech,
-  });
+  let speech: ReturnType<typeof createSpeechService> | null = null;
+
+  function getSpeech() {
+    if (!speech) {
+      speech = (options.speechFactory ?? createSpeechService)({
+        logger: options.logger,
+        openaiConfig: options.config.openai,
+        speechConfig: options.config.speech,
+      });
+      speech.start();
+    }
+    return speech;
+  }
+
   return {
     configureOrchestrator(orchestrator) {
       orchestratorSettings = orchestrator;
     },
-    startHarness() {
-      speech.start();
-    },
-    stopHarness() {
-      return speech.stop();
+    async close() {
+      await speech?.stop();
+      speech = null;
     },
     getReadiness() {
       if (!resolveManualVoiceOrchestratorConfig(orchestratorSettings)) {
@@ -171,14 +178,14 @@ export function createManualVoiceProvider(
       return { ready: true };
     },
     async start(input) {
-      return startManualVoiceCall(options, speech, orchestratorSettings, input);
+      return startManualVoiceCall(options, getSpeech, orchestratorSettings, input);
     },
   };
 }
 
 async function startManualVoiceCall(
   options: CreateManualVoiceProviderOptions,
-  speech: ReturnType<typeof createSpeechService>,
+  getSpeech: () => ReturnType<typeof createSpeechService>,
   orchestratorSettings: ManualVoiceOrchestratorSettings,
   input: VoiceProviderStartInput,
 ) {
@@ -195,6 +202,9 @@ async function startManualVoiceCall(
   const orchestrator = resolveManualVoiceOrchestratorConfig(orchestratorSettings);
   if (!orchestrator) throw new Error("Configure the manual voice orchestrator and mode.");
   if (input.signal.aborted) throw new Error("Voice call start was canceled.");
+  const speech = getSpeech();
+  await speech.ready;
+  if (input.signal.aborted) throw new Error("Voice call start was canceled.");
 
   const agentId = uuidv4();
   let context = input.context;