From 34ba33402a7748df7b71681fa393eadb9f9eb2a1 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 15:55:34 +0700 Subject: [PATCH 01/21] Voice MCP policy: required prompt, cwd override, and restricted tool path --- .../src/server/agent/mcp-server.test.ts | 128 ++++++- .../server/src/server/agent/mcp-server.ts | 230 +++++++++---- .../server/src/server/bootstrap.smoke.test.ts | 11 +- packages/server/src/server/bootstrap.ts | 106 +++++- packages/server/src/server/config.ts | 30 ++ .../server/src/server/persisted-config.ts | 4 +- packages/server/src/server/session.ts | 324 +++++++++++++++++- .../local/sherpa/sherpa-onnx-loader.ts | 30 ++ .../src/server/test-utils/paseo-daemon.ts | 9 +- .../src/server/voice-local-agent.e2e.test.ts | 124 +++++++ .../server/src/server/websocket-server.ts | 55 +++ .../server/src/test-utils/vitest-setup.ts | 3 +- 12 files changed, 970 insertions(+), 84 deletions(-) create mode 100644 packages/server/src/server/voice-local-agent.e2e.test.ts diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index afbfbbf64..ce7186091 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -58,6 +58,7 @@ describe("create_agent MCP tool", () => { const missingTitle = await tool.inputSchema.safeParseAsync({ cwd: "/tmp/repo", initialMode: "default", + initialPrompt: "test", }); expect(missingTitle.success).toBe(false); expect(missingTitle.error.issues[0].path).toEqual(["title"]); @@ -66,6 +67,7 @@ describe("create_agent MCP tool", () => { cwd: "/tmp/repo", initialMode: "default", title: "x".repeat(61), + initialPrompt: "test", }); expect(tooLong.success).toBe(false); expect(tooLong.error.issues[0].path).toEqual(["title"]); @@ -74,10 +76,24 @@ describe("create_agent MCP tool", () => { cwd: "/tmp/repo", initialMode: "default", title: "Short title", + initialPrompt: "test", }); expect(ok.success).toBe(true); }); + it("requires initialPrompt", async () => { + const { agentManager, agentStorage } = createTestDeps(); + const server = await createAgentMcpServer({ agentManager, agentStorage, logger }); + const tool = (server as any)._registeredTools["create_agent"]; + const parsed = await tool.inputSchema.safeParseAsync({ + cwd: "/tmp/repo", + initialMode: "default", + title: "Short title", + }); + expect(parsed.success).toBe(false); + expect(parsed.error.issues.some((issue: { path: string[] }) => issue.path[0] === "initialPrompt")).toBe(true); + }); + it("passes caller-provided titles directly into createAgent", async () => { const { agentManager, agentStorage, spies } = createTestDeps(); spies.agentManager.createAgent.mockResolvedValue({ @@ -86,6 +102,7 @@ describe("create_agent MCP tool", () => { lifecycle: "idle", currentModeId: null, availableModes: [], + config: { title: "Fix auth bug" }, } as ManagedAgent); const server = await createAgentMcpServer({ agentManager, agentStorage, logger }); @@ -93,13 +110,16 @@ describe("create_agent MCP tool", () => { await tool.callback({ cwd: "/tmp/repo", title: " Fix auth bug ", + initialPrompt: "Do work", }); expect(spies.agentManager.createAgent).toHaveBeenCalledWith( expect.objectContaining({ cwd: "/tmp/repo", title: "Fix auth bug", - }) + }), + undefined, + undefined ); }); @@ -111,6 +131,7 @@ describe("create_agent MCP tool", () => { lifecycle: "idle", currentModeId: null, availableModes: [], + config: { title: "Fix auth" }, } as ManagedAgent); const server = await createAgentMcpServer({ agentManager, agentStorage, logger }); @@ -118,12 +139,115 @@ describe("create_agent MCP tool", () => { await tool.callback({ cwd: "/tmp/repo", title: " Fix auth ", + initialPrompt: "Do work", }); expect(spies.agentManager.createAgent).toHaveBeenCalledWith( expect.objectContaining({ title: "Fix auth", - }) + }), + undefined, + undefined + ); + }); + + it("allows caller agents to override cwd and applies caller context labels", async () => { + const { agentManager, agentStorage, spies } = createTestDeps(); + spies.agentManager.getAgent.mockReturnValue({ + id: "voice-agent", + cwd: "/tmp/voice", + provider: "codex", + currentModeId: "full-access", + } as ManagedAgent); + spies.agentManager.createAgent.mockResolvedValue({ + id: "child-agent", + cwd: "/tmp/voice/subdir", + lifecycle: "idle", + currentModeId: null, + availableModes: [], + config: { title: "Child" }, + } as ManagedAgent); + + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + callerAgentId: "voice-agent", + resolveCallerContext: () => ({ + childAgentDefaultLabels: { ui: "true" }, + allowCustomCwd: true, + }), + logger, + }); + + const tool = (server as any)._registeredTools["create_agent"]; + await tool.callback({ + cwd: "subdir", + title: "Child", + agentType: "codex", + initialPrompt: "Do work", + }); + + expect(spies.agentManager.createAgent).toHaveBeenCalledWith( + expect.objectContaining({ + cwd: "/tmp/voice/subdir", + }), + undefined, + { labels: { ui: "true" } } ); }); }); + +describe("speak MCP tool", () => { + const logger = createTestLogger(); + + it("invokes registered speak handler for caller agent", async () => { + const { agentManager, agentStorage } = createTestDeps(); + const speak = vi.fn().mockResolvedValue(undefined); + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + callerAgentId: "voice-agent-1", + enableVoiceTools: true, + resolveSpeakHandler: () => speak, + logger, + }); + const tool = (server as any)._registeredTools["speak"]; + expect(tool).toBeDefined(); + + await tool.callback({ text: "Hello from voice agent." }); + expect(speak).toHaveBeenCalledWith( + expect.objectContaining({ + text: "Hello from voice agent.", + callerAgentId: "voice-agent-1", + }) + ); + }); + + it("fails when no speak handler exists", async () => { + const { agentManager, agentStorage } = createTestDeps(); + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + callerAgentId: "voice-agent-2", + enableVoiceTools: true, + resolveSpeakHandler: () => null, + logger, + }); + const tool = (server as any)._registeredTools["speak"]; + await expect(tool.callback({ text: "Hello." })).rejects.toThrow( + "No speak handler registered for caller agent" + ); + }); + + it("does not register speak tool unless voice tools are enabled", async () => { + const { agentManager, agentStorage } = createTestDeps(); + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + callerAgentId: "agent-no-voice", + logger, + }); + const tool = (server as any)._registeredTools["speak"]; + expect(tool).toBeUndefined(); + }); +}); diff --git a/packages/server/src/server/agent/mcp-server.ts b/packages/server/src/server/agent/mcp-server.ts index ee7d4d70a..83d09a9ab 100644 --- a/packages/server/src/server/agent/mcp-server.ts +++ b/packages/server/src/server/agent/mcp-server.ts @@ -38,6 +38,22 @@ export interface AgentMcpServerOptions { * Used for cwd/mode inheritance when agents spawn child agents. */ callerAgentId?: string; + /** + * Optional resolver for session-bound speak handlers. + * Used by hidden voice agents to narrate through daemon-managed TTS. + */ + resolveSpeakHandler?: ( + callerAgentId: string + ) => ((params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise) | null; + resolveCallerContext?: ( + callerAgentId: string + ) => { + childAgentDefaultLabels?: Record; + lockedCwd?: string; + allowCustomCwd?: boolean; + enableVoiceTools?: boolean; + } | null; + enableVoiceTools?: boolean; logger: Logger; } @@ -252,9 +268,17 @@ async function serializeSnapshotWithMetadata( export async function createAgentMcpServer( options: AgentMcpServerOptions ): Promise { - const { agentManager, agentStorage, callerAgentId, logger } = options; + const { + agentManager, + agentStorage, + callerAgentId, + resolveSpeakHandler, + resolveCallerContext, + logger, + } = options; const childLogger = logger.child({ module: "agent", component: "mcp-server" }); const waitTracker = new WaitForAgentTracker(logger); + const callerContext = callerAgentId ? resolveCallerContext?.(callerAgentId) ?? null : null; const server = new McpServer({ name: "agent-mcp", @@ -262,6 +286,12 @@ export async function createAgentMcpServer( }); const agentToAgentInputSchema = { + cwd: z + .string() + .optional() + .describe( + "Optional working directory. Defaults to the caller agent working directory." + ), title: z .string() .trim() @@ -275,9 +305,10 @@ export async function createAgentMcpServer( ), initialPrompt: z .string() - .optional() + .trim() + .min(1, "initialPrompt is required") .describe( - "Optional task to start immediately after creation (non-blocking)." + "Required first task to run immediately after creation." ), background: z .boolean() @@ -307,9 +338,10 @@ export async function createAgentMcpServer( ), initialPrompt: z .string() - .optional() + .trim() + .min(1, "initialPrompt is required") .describe( - "Optional task to start immediately after creation (non-blocking)." + "Required first task to run immediately after creation." ), initialMode: z .string() @@ -339,6 +371,45 @@ export async function createAgentMcpServer( ? agentToAgentInputSchema : topLevelInputSchema; + if (options.enableVoiceTools || callerContext?.enableVoiceTools) { + server.registerTool( + "speak", + { + title: "Speak", + description: + "Speak text to the user via daemon-managed voice output. Blocks until playback completes.", + inputSchema: { + text: z + .string() + .trim() + .min(1, "text is required") + .max(4000, "text must be 4000 characters or fewer"), + }, + outputSchema: { + ok: z.boolean(), + }, + }, + async (args, context) => { + if (!callerAgentId) { + throw new Error("speak is only available to agent-scoped MCP sessions"); + } + const handler = resolveSpeakHandler?.(callerAgentId) ?? null; + if (!handler) { + throw new Error(`No speak handler registered for caller agent '${callerAgentId}'`); + } + await handler({ + text: args.text, + callerAgentId, + signal: (context as { signal?: AbortSignal } | undefined)?.signal, + }); + return { + content: [], + structuredContent: ensureValidJson({ ok: true }), + }; + } + ); + } + server.registerTool( "create_agent", { @@ -363,7 +434,7 @@ export async function createAgentMcpServer( permission: AgentPermissionRequestPayloadSchema.nullable().optional(), }, }, - async (args) => { + async (args: unknown) => { const { agentType, initialPrompt, @@ -372,7 +443,7 @@ export async function createAgentMcpServer( } = args as { cwd?: string; agentType?: AgentProvider; - initialPrompt?: string; + initialPrompt: string; initialMode?: string; worktreeName?: string; background?: boolean; @@ -387,7 +458,19 @@ export async function createAgentMcpServer( if (!parentAgent) { throw new Error(`Parent agent ${callerAgentId} not found`); } - resolvedCwd = parentAgent.cwd; + const callerArgs = args as unknown as { cwd?: string }; + const requestedCwd = callerArgs.cwd?.trim(); + const lockedCwd = callerContext?.lockedCwd?.trim(); + if (lockedCwd) { + resolvedCwd = expandPath(lockedCwd); + } else if (requestedCwd && (callerContext?.allowCustomCwd ?? true)) { + resolvedCwd = + requestedCwd.startsWith("/") || requestedCwd.startsWith("~") + ? expandPath(requestedCwd) + : resolve(parentAgent.cwd, requestedCwd); + } else { + resolvedCwd = parentAgent.cwd; + } const provider: AgentProvider = agentType ?? "claude"; const parentMode = parentAgent.currentModeId; @@ -433,73 +516,78 @@ export async function createAgentMcpServer( const provider: AgentProvider = agentType ?? "claude"; const normalizedTitle = title?.trim() ?? null; - const snapshot = await agentManager.createAgent({ - provider, - cwd: resolvedCwd, - modeId: resolvedMode, - title: normalizedTitle ?? undefined, + const childAgentDefaultLabels = + callerAgentId && callerContext?.childAgentDefaultLabels + ? callerContext.childAgentDefaultLabels + : undefined; + const snapshot = await agentManager.createAgent( + { + provider, + cwd: resolvedCwd, + modeId: resolvedMode, + title: normalizedTitle ?? undefined, + }, + undefined, + childAgentDefaultLabels ? { labels: childAgentDefaultLabels } : undefined + ); + + const trimmedPrompt = initialPrompt.trim(); + scheduleAgentMetadataGeneration({ + agentManager, + agentId: snapshot.id, + cwd: snapshot.cwd, + initialPrompt: trimmedPrompt, + explicitTitle: snapshot.config.title, + paseoHome: options.paseoHome, + logger: childLogger, }); - const trimmedPrompt = initialPrompt?.trim(); - if (trimmedPrompt) { - scheduleAgentMetadataGeneration({ - agentManager, - agentId: snapshot.id, - cwd: snapshot.cwd, - initialPrompt: trimmedPrompt, - explicitTitle: snapshot.config.title, - paseoHome: options.paseoHome, - logger: childLogger, - }); - - try { - agentManager.recordUserMessage(snapshot.id, trimmedPrompt); - } catch (error) { - childLogger.error( - { err: error, agentId: snapshot.id }, - "Failed to record initial prompt" - ); - } - - try { - startAgentRun(agentManager, snapshot.id, trimmedPrompt, childLogger); - - // If not running in background, wait for completion - if (!background) { - const result = await waitForAgentWithTimeout( - agentManager, - snapshot.id, - { waitForActive: true } - ); - - const responseData = { - agentId: snapshot.id, - type: provider, - status: result.status, - cwd: snapshot.cwd, - currentModeId: snapshot.currentModeId, - availableModes: snapshot.availableModes, - lastMessage: result.lastMessage, - permission: sanitizePermissionRequest(result.permission), - }; - const validJson = ensureValidJson(responseData); - - const response = { - content: [], - structuredContent: validJson, - }; - return response; - } - } catch (error) { - childLogger.error( - { err: error, agentId: snapshot.id }, - "Failed to run initial prompt" - ); - } - } else { + try { + agentManager.recordUserMessage(snapshot.id, trimmedPrompt); + } catch (error) { + childLogger.error( + { err: error, agentId: snapshot.id }, + "Failed to record initial prompt" + ); } - // Return immediately if background=true or no initialPrompt + try { + startAgentRun(agentManager, snapshot.id, trimmedPrompt, childLogger); + + // If not running in background, wait for completion + if (!background) { + const result = await waitForAgentWithTimeout( + agentManager, + snapshot.id, + { waitForActive: true } + ); + + const responseData = { + agentId: snapshot.id, + type: provider, + status: result.status, + cwd: snapshot.cwd, + currentModeId: snapshot.currentModeId, + availableModes: snapshot.availableModes, + lastMessage: result.lastMessage, + permission: sanitizePermissionRequest(result.permission), + }; + const validJson = ensureValidJson(responseData); + + const response = { + content: [], + structuredContent: validJson, + }; + return response; + } + } catch (error) { + childLogger.error( + { err: error, agentId: snapshot.id }, + "Failed to run initial prompt" + ); + } + + // Return immediately if background=true const response = { content: [], structuredContent: ensureValidJson({ diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 65834a8b6..8ee4a3d14 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -9,8 +9,15 @@ import { createTestPaseoDaemon } from "./test-utils/paseo-daemon.js"; import { createTestAgentClients } from "./test-utils/fake-agent-client.js"; describe("paseo daemon bootstrap", () => { - test("starts and serves health endpoint", async () => { - const daemonHandle = await createTestPaseoDaemon(); + test.runIf(Boolean(process.env.OPENAI_API_KEY))("starts and serves health endpoint", async () => { + const daemonHandle = await createTestPaseoDaemon({ + openai: { apiKey: process.env.OPENAI_API_KEY! }, + speech: { + dictationSttProvider: "openai", + voiceSttProvider: "openai", + voiceTtsProvider: "openai", + }, + }); try { const response = await fetch( `http://127.0.0.1:${daemonHandle.port}/api/health`, diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index df688172f..081e5d8f7 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -77,6 +77,8 @@ import { acquirePidLock, releasePidLock } from "./pid-lock.js"; import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js"; type AgentMcpTransportMap = Map; +type VoiceAgentProvider = "claude" | "codex" | "opencode"; +const VOICE_AGENT_FALLBACK_ORDER: VoiceAgentProvider[] = ["claude", "codex", "opencode"]; export type PaseoOpenAIConfig = { apiKey?: string; @@ -121,6 +123,8 @@ export type PaseoDaemonConfig = { openai?: PaseoOpenAIConfig; speech?: PaseoSpeechConfig; openrouterApiKey?: string | null; + voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null; + voiceLlmProviderExplicit?: boolean; voiceLlmModel?: string | null; dictationFinalTimeoutMs?: number; downloadTokenTtlMs?: number; @@ -282,12 +286,88 @@ export async function createPaseoDaemon( `Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand` ); + const requestedVoiceLlmProvider = config.voiceLlmProvider ?? null; + const voiceLlmProviderExplicit = config.voiceLlmProviderExplicit ?? false; + logger.info( + { + requestedVoiceLlmProvider, + voiceLlmProviderExplicit, + }, + "Voice LLM provider reconciliation started" + ); + + const providerClients = createAllClients(logger); + const voiceLlmAvailability: Record = { + claude: false, + codex: false, + opencode: false, + }; + for (const provider of VOICE_AGENT_FALLBACK_ORDER) { + try { + voiceLlmAvailability[provider] = await providerClients[provider].isAvailable(); + } catch (error) { + logger.warn({ err: error, provider }, "Voice LLM provider availability check failed"); + voiceLlmAvailability[provider] = false; + } + } + + const voiceLlmDefaultProvider = + VOICE_AGENT_FALLBACK_ORDER.find((provider) => voiceLlmAvailability[provider]) ?? null; + + if (requestedVoiceLlmProvider === "openrouter") { + const openrouterApiKey = + config.openrouterApiKey ?? process.env.OPENROUTER_API_KEY ?? null; + if (!openrouterApiKey) { + logger.error("voiceMode.llm.provider is openrouter but no OpenRouter API key is configured"); + throw new Error("Missing OpenRouter API key for voiceMode.llm.provider=openrouter"); + } + } else if ( + requestedVoiceLlmProvider === "claude" || + requestedVoiceLlmProvider === "codex" || + requestedVoiceLlmProvider === "opencode" + ) { + if (!voiceLlmAvailability[requestedVoiceLlmProvider]) { + logger.error( + { provider: requestedVoiceLlmProvider, voiceLlmAvailability }, + "Configured voice LLM provider is unavailable" + ); + throw new Error(`Configured voice LLM provider '${requestedVoiceLlmProvider}' is unavailable`); + } + } else if (!voiceLlmDefaultProvider) { + logger.error( + { requestedVoiceLlmProvider, voiceLlmAvailability }, + "No local voice LLM provider available for fallback" + ); + throw new Error("No local voice LLM provider available (claude/codex/opencode)"); + } + + logger.info( + { + requestedVoiceLlmProvider, + voiceLlmProviderExplicit, + voiceLlmAvailability, + voiceLlmDefaultProvider, + }, + "Voice LLM provider reconciliation completed" + ); + if (listenTarget.type !== "tcp" && requestedVoiceLlmProvider !== "openrouter") { + logger.error( + { listen: config.listen, requestedVoiceLlmProvider }, + "Local voice agent mode requires TCP listen target for HTTP MCP bridge" + ); + throw new Error("Local voice agent mode requires TCP listen target"); + } + let wsServer: VoiceAssistantWebSocketServer | null = null; + // Create in-memory transport for Session's Agent MCP client (voice assistant tools) const createInMemoryAgentMcpTransport = async (): Promise => { const agentMcpServer = await createAgentMcpServer({ agentManager, agentStorage, paseoHome: config.paseoHome, + enableVoiceTools: false, + resolveSpeakHandler: (callerAgentId) => wsServer?.resolveVoiceSpeakHandler(callerAgentId) ?? null, + resolveCallerContext: (callerAgentId) => wsServer?.resolveVoiceCallerContext(callerAgentId) ?? null, logger, }); @@ -309,6 +389,9 @@ export async function createPaseoDaemon( agentStorage, paseoHome: config.paseoHome, callerAgentId, + enableVoiceTools: false, + resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null, + resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null, logger, }); @@ -792,7 +875,12 @@ export async function createPaseoDaemon( ); } - const wsServer = new VoiceAssistantWebSocketServer( + const voiceAgentMcpUrl = + listenTarget.type === "tcp" + ? `http://127.0.0.1:${listenTarget.port}/mcp/agents` + : null; + + wsServer = new VoiceAssistantWebSocketServer( httpServer, logger, serverId, @@ -806,7 +894,12 @@ export async function createPaseoDaemon( terminalManager, { openrouterApiKey: config.openrouterApiKey ?? null, + voiceLlmProvider: config.voiceLlmProvider ?? null, + voiceLlmProviderExplicit, + voiceLlmDefaultProvider, voiceLlmModel: config.voiceLlmModel ?? null, + voiceLlmAvailability, + voiceAgentMcpUrl, }, { finalTimeoutMs: config.dictationFinalTimeoutMs, @@ -870,7 +963,12 @@ export async function createPaseoDaemon( relayTransport?.stop().catch(() => undefined); relayTransport = startRelayTransport({ logger, - attachSocket: (ws) => wsServer.attachExternalSocket(ws), + attachSocket: (ws) => { + if (!wsServer) { + throw new Error("WebSocket server not initialized"); + } + return wsServer.attachExternalSocket(ws); + }, relayEndpoint, serverId, daemonKeyPair: daemonKeyPair.keyPair, @@ -911,7 +1009,9 @@ export async function createPaseoDaemon( sherpaOnline?.free(); sherpaOffline?.free(); await relayTransport?.stop().catch(() => undefined); - await wsServer.close(); + if (wsServer) { + await wsServer.close(); + } await new Promise((resolve) => { httpServer.close(() => resolve()); }); diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index 5f0825a05..47c1cd6f8 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -13,6 +13,14 @@ import { const DEFAULT_PORT = 6767; const DEFAULT_RELAY_ENDPOINT = "relay.paseo.sh:443"; const DEFAULT_APP_BASE_URL = "https://app.paseo.sh"; +const VOICE_LLM_PROVIDER_IDS = [ + "openrouter", + "local-agent", + "claude", + "codex", + "opencode", +] as const; +type VoiceLlmProviderId = (typeof VOICE_LLM_PROVIDER_IDS)[number]; function getDefaultListen(): string { // Main HTTP server defaults to TCP @@ -89,6 +97,19 @@ function parseSpeechProviderId(value: unknown): "openai" | "local" | null { return null; } +function parseVoiceLlmProviderId(value: unknown): VoiceLlmProviderId | null { + if (typeof value !== "string") { + return null; + } + const normalized = value.trim().toLowerCase(); + if (!normalized) { + return null; + } + return (VOICE_LLM_PROVIDER_IDS as readonly string[]).includes(normalized) + ? (normalized as VoiceLlmProviderId) + : null; +} + function normalizeSherpaSttPreset(value: string): string { const raw = value.trim(); const normalized = raw.toLowerCase(); @@ -246,6 +267,13 @@ export function loadConfig( const openrouterApiKey = env.OPENROUTER_API_KEY ?? persisted.providers?.openrouter?.apiKey ?? null; + const envVoiceLlmProvider = parseVoiceLlmProviderId(env.PASEO_VOICE_LLM_PROVIDER); + const persistedVoiceLlmProvider = parseVoiceLlmProviderId( + persisted.features?.voiceMode?.llm?.provider + ); + const voiceLlmProvider = envVoiceLlmProvider ?? persistedVoiceLlmProvider ?? null; + const voiceLlmProviderExplicit = + envVoiceLlmProvider !== null || persistedVoiceLlmProvider !== null; const voiceLlmModel = persisted.features?.voiceMode?.llm?.model ?? null; return { @@ -272,6 +300,8 @@ export function loadConfig( ...(sherpaOnnx ? { sherpaOnnx } : {}), }, openrouterApiKey, + voiceLlmProvider, + voiceLlmProviderExplicit, voiceLlmModel, }; } diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index 4b34cbbcd..be1293241 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -74,7 +74,9 @@ const FeatureVoiceModeSchema = z .object({ llm: z .object({ - provider: z.enum(["openrouter"]).optional(), + provider: z + .enum(["openrouter", "local-agent", "claude", "codex", "opencode"]) + .optional(), model: z.string().min(1).optional(), }) .strict() diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 7cff12a80..409587d5e 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -49,6 +49,15 @@ import { experimental_createMCPClient } from "ai"; import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js"; export type AgentMcpTransportFactory = () => Promise; +type VoiceLlmProvider = "openrouter" | "local-agent" | "claude" | "codex" | "opencode"; +type VoiceAgentProvider = Exclude; +type VoiceSpeakHandler = (params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise; +type VoiceCallerContext = { + childAgentDefaultLabels?: Record; + lockedCwd?: string; + allowCustomCwd?: boolean; + enableVoiceTools?: boolean; +}; import { buildProviderRegistry } from "./agent/provider-registry.js"; import { AgentManager } from "./agent/agent-manager.js"; import type { ManagedAgent } from "./agent/agent-manager.js"; @@ -59,6 +68,7 @@ import { generateStructuredAgentResponse, } from "./agent/agent-response-loop.js"; import type { + AgentPermissionRequest, AgentPermissionResponse, AgentPromptContentBlock, AgentPromptInput, @@ -130,6 +140,24 @@ const RESTART_EXIT_DELAY_MS = 250; * Uses Claude Haiku for speed and cost efficiency. */ const AUTO_GEN_MODEL = "haiku"; +const VOICE_AGENT_FALLBACK_ORDER: VoiceAgentProvider[] = ["claude", "codex", "opencode"]; +const VOICE_AGENT_DEFAULT_MODE: Record = { + claude: "default", + codex: "read-only", + opencode: "default", +}; +const VOICE_AGENT_DEFAULT_MODEL: Partial> = { + claude: "haiku", + codex: "gpt-5.2-mini", +}; +const VOICE_AGENT_SYSTEM_INSTRUCTION = [ + "You are the Paseo voice assistant.", + "The user cannot see your chat messages or tool calls.", + "Always narrate everything through the speak tool.", + "Use concise plain language suitable for speech output.", + "Never use bash, file-edit, or web tools directly.", + "Only use the paseo MCP tools.", +].join(" "); type ProcessingPhase = "idle" | "transcribing" | "llm"; @@ -320,7 +348,23 @@ export class Session { private readonly terminalManager: TerminalManager | null; private terminalSubscriptions: Map void> = new Map(); private readonly openrouterApiKey: string | null; + private readonly voiceLlmProvider: VoiceLlmProvider | null; + private readonly voiceLlmProviderExplicit: boolean; + private readonly voiceLlmDefaultProvider: VoiceAgentProvider | null; private readonly voiceLlmModel: string | null; + private readonly voiceLlmAvailability: Record | null; + private readonly voiceAgentMcpUrl: string | null; + private readonly registerVoiceSpeakHandler?: ( + agentId: string, + handler: VoiceSpeakHandler + ) => void; + private readonly unregisterVoiceSpeakHandler?: (agentId: string) => void; + private readonly registerVoiceCallerContext?: ( + agentId: string, + context: VoiceCallerContext + ) => void; + private readonly unregisterVoiceCallerContext?: (agentId: string) => void; + private voiceAssistantAgentId: string | null = null; constructor( clientId: string, @@ -338,7 +382,18 @@ export class Session { voiceConversationStore: VoiceConversationStore, voice?: { openrouterApiKey?: string | null; + voiceLlmProvider?: VoiceLlmProvider | null; + voiceLlmProviderExplicit?: boolean; + voiceLlmDefaultProvider?: VoiceAgentProvider | null; voiceLlmModel?: string | null; + voiceLlmAvailability?: Record | null; + voiceAgentMcpUrl?: string | null; + }, + voiceBridge?: { + registerVoiceSpeakHandler?: (agentId: string, handler: VoiceSpeakHandler) => void; + unregisterVoiceSpeakHandler?: (agentId: string) => void; + registerVoiceCallerContext?: (agentId: string, context: VoiceCallerContext) => void; + unregisterVoiceCallerContext?: (agentId: string) => void; }, dictation?: { finalTimeoutMs?: number; @@ -357,7 +412,16 @@ export class Session { this.terminalManager = terminalManager; this.voiceConversationStore = voiceConversationStore; this.openrouterApiKey = voice?.openrouterApiKey ?? null; + this.voiceLlmProvider = voice?.voiceLlmProvider ?? null; + this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false; + this.voiceLlmDefaultProvider = voice?.voiceLlmDefaultProvider ?? null; this.voiceLlmModel = voice?.voiceLlmModel ?? null; + this.voiceLlmAvailability = voice?.voiceLlmAvailability ?? null; + this.voiceAgentMcpUrl = voice?.voiceAgentMcpUrl ?? null; + this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler; + this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler; + this.registerVoiceCallerContext = voiceBridge?.registerVoiceCallerContext; + this.unregisterVoiceCallerContext = voiceBridge?.unregisterVoiceCallerContext; this.abortController = new AbortController(); this.sessionLogger = logger.child({ module: "session", @@ -4241,7 +4305,7 @@ export class Session { this.messages.push({ role: "user", content: text }); // Process through LLM (TTS enabled in voice mode for voice conversations) - this.currentStreamPromise = this.processWithLLM(this.isVoiceMode); + this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, text); await this.currentStreamPromise; } @@ -4518,7 +4582,7 @@ export class Session { // Set phase to LLM and process (TTS enabled in voice mode for voice conversations) this.clearSpeechInProgress("transcription complete"); this.setPhase("llm"); - this.currentStreamPromise = this.processWithLLM(this.isVoiceMode); + this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, result.text); await this.currentStreamPromise; this.setPhase("idle"); } catch (error: any) { @@ -4537,10 +4601,249 @@ export class Session { } } + /** + * Resolve the effective voice LLM provider. + * - explicit provider => strict + * - local-agent / unset => fallback order + */ + private resolveVoiceAgentProvider(): VoiceAgentProvider { + const configured = this.voiceLlmProvider; + const availability = this.voiceLlmAvailability ?? { + claude: true, + codex: true, + opencode: true, + }; + + if (configured === "openrouter") { + throw new Error("voiceLlmProvider=openrouter cannot be used in local-agent flow"); + } + + if (configured === "claude" || configured === "codex" || configured === "opencode") { + if (!availability[configured]) { + throw new Error(`Configured voice LLM provider '${configured}' is unavailable`); + } + return configured; + } + + const fallbackOrder = + this.voiceLlmDefaultProvider && availability[this.voiceLlmDefaultProvider] + ? [this.voiceLlmDefaultProvider, ...VOICE_AGENT_FALLBACK_ORDER.filter((id) => id !== this.voiceLlmDefaultProvider)] + : VOICE_AGENT_FALLBACK_ORDER; + + for (const provider of fallbackOrder) { + if (availability[provider]) { + return provider; + } + } + + throw new Error("No local voice LLM provider is available (claude/codex/opencode)"); + } + + private getVoiceAgentModel(provider: VoiceAgentProvider): string | undefined { + const configured = this.voiceLlmModel?.trim(); + if (configured) { + return configured; + } + return VOICE_AGENT_DEFAULT_MODEL[provider]; + } + + private async ensureVoiceAssistantAgent(): Promise { + if (this.voiceAssistantAgentId) { + const existing = this.agentManager.getAgent(this.voiceAssistantAgentId); + if (existing) { + return existing.id; + } + this.voiceAssistantAgentId = null; + } + + const provider = this.resolveVoiceAgentProvider(); + const voiceAgentId = `voice-${uuidv4()}`; + const cwd = join(this.paseoHome, "voice-agent-workspace"); + await mkdir(cwd, { recursive: true }); + + const mcpUrl = this.voiceAgentMcpUrl; + if (!mcpUrl) { + throw new Error("Voice MCP URL is not configured"); + } + + const model = this.getVoiceAgentModel(provider); + const config: AgentSessionConfig = { + provider, + cwd, + modeId: VOICE_AGENT_DEFAULT_MODE[provider], + ...(model ? { model } : {}), + internal: true, + mcpServers: { + paseo: { + type: "http", + url: `${mcpUrl}?callerAgentId=${encodeURIComponent(voiceAgentId)}`, + }, + }, + }; + + const created = await this.agentManager.createAgent(config, voiceAgentId, { + labels: { + surface: "voice", + ui: "false", + }, + }); + this.voiceAssistantAgentId = created.id; + + this.registerVoiceSpeakHandler?.(created.id, async ({ text, signal }) => { + const abortSignal = signal ?? this.abortController.signal; + await this.ttsManager.generateAndWaitForPlayback( + text, + (msg) => this.emit(msg), + abortSignal, + true + ); + this.emit({ + type: "activity_log", + payload: { + id: uuidv4(), + timestamp: new Date(), + type: "assistant", + content: text, + }, + }); + }); + this.registerVoiceCallerContext?.(created.id, { + childAgentDefaultLabels: { ui: "true" }, + allowCustomCwd: true, + enableVoiceTools: true, + }); + + this.sessionLogger.info( + { + voiceAssistantAgentId: created.id, + provider, + model: model ?? null, + providerExplicit: this.voiceLlmProviderExplicit, + }, + "Voice assistant agent initialized" + ); + return created.id; + } + + private buildVoiceAgentPrompt(userText: string): string { + return [ + VOICE_AGENT_SYSTEM_INSTRUCTION, + "", + `User said: ${userText.trim()}`, + ].join("\n"); + } + + private shouldAllowVoicePermission(request: AgentPermissionRequest): boolean { + const name = request.name.toLowerCase(); + if (name.includes("mcp") || name.includes("paseo") || name.includes("speak")) { + return true; + } + if (name === "codextool") { + const metadata = request.metadata ?? {}; + const rawQuestions = metadata.questions; + if (Array.isArray(rawQuestions)) { + const text = JSON.stringify(rawQuestions).toLowerCase(); + return text.includes("mcp") || text.includes("paseo") || text.includes("speak"); + } + return false; + } + return false; + } + + private async processWithVoiceAgent(userText: string): Promise { + const agentId = await this.ensureVoiceAssistantAgent(); + + await this.interruptAgentIfRunning(agentId); + + const prompt = this.buildVoiceAgentPrompt(userText); + this.agentManager.recordUserMessage(agentId, userText); + + let sawSpeakToolCall = false; + const assistantTextChunks: string[] = []; + const iterator = this.agentManager.streamAgent(agentId, prompt); + for await (const event of iterator) { + if (event.type === "turn_failed") { + throw new Error(event.error); + } + if (event.type === "timeline") { + if (event.item.type === "tool_call" && typeof event.item.name === "string") { + if (event.item.name.toLowerCase().includes("speak")) { + sawSpeakToolCall = true; + } + } + if (event.item.type === "assistant_message" && event.item.text.trim().length > 0) { + assistantTextChunks.push(event.item.text.trim()); + } + } + if (event.type === "permission_requested") { + if (this.shouldAllowVoicePermission(event.request)) { + await this.agentManager.respondToPermission(agentId, event.request.id, { + behavior: "allow", + }); + } else { + await this.agentManager.respondToPermission(agentId, event.request.id, { + behavior: "deny", + message: "Voice assistant policy only allows MCP paseo tools.", + interrupt: true, + }); + throw new Error( + `Voice assistant denied non-MCP tool request: ${event.request.name}` + ); + } + } + } + + if (!sawSpeakToolCall && assistantTextChunks.length > 0) { + const fallbackText = assistantTextChunks.join(" ").trim(); + await this.ttsManager.generateAndWaitForPlayback( + fallbackText, + (msg) => this.emit(msg), + this.abortController.signal, + true + ); + this.emit({ + type: "activity_log", + payload: { + id: uuidv4(), + timestamp: new Date(), + type: "assistant", + content: fallbackText, + }, + }); + this.sessionLogger.warn( + { voiceAssistantAgentId: agentId }, + "Voice agent responded without speak tool; used fallback TTS from assistant text" + ); + } + } + /** * Process user message through LLM with streaming and tool execution */ - private async processWithLLM(enableTTS: boolean): Promise { + private async processWithLLM(enableTTS: boolean, latestUserText?: string): Promise { + if (enableTTS && this.voiceLlmProvider !== "openrouter") { + const text = + typeof latestUserText === "string" && latestUserText.trim().length > 0 + ? latestUserText + : (() => { + const lastUser = [...this.messages] + .reverse() + .find((message) => message.role === "user"); + if (!lastUser) { + return ""; + } + return typeof lastUser.content === "string" + ? lastUser.content + : JSON.stringify(lastUser.content); + })(); + const normalized = text.trim(); + if (!normalized) { + return; + } + await this.processWithVoiceAgent(normalized); + return; + } + let assistantResponse = ""; let pendingTTS: Promise | null = null; let textBuffer = ""; @@ -5180,6 +5483,21 @@ export class Session { this.agentTools = null; } + if (this.voiceAssistantAgentId) { + try { + await this.agentManager.closeAgent(this.voiceAssistantAgentId); + } catch (error) { + this.sessionLogger.warn( + { err: error, voiceAssistantAgentId: this.voiceAssistantAgentId }, + "Failed to close voice assistant agent" + ); + } finally { + this.unregisterVoiceSpeakHandler?.(this.voiceAssistantAgentId); + this.unregisterVoiceCallerContext?.(this.voiceAssistantAgentId); + this.voiceAssistantAgentId = null; + } + } + // Unsubscribe from all terminals for (const unsubscribe of this.terminalSubscriptions.values()) { unsubscribe(); diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts index 80bf1abe2..680d16c64 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts @@ -1,4 +1,6 @@ import { createRequire } from "node:module"; +import os from "node:os"; +import path from "node:path"; export type SherpaOnnxModule = { createOnlineRecognizer: (config: any) => any; @@ -8,11 +10,39 @@ export type SherpaOnnxModule = { let cached: SherpaOnnxModule | null = null; +function ensureSherpaNativeLibraryPath(requireFn: NodeRequire): void { + const platform = os.platform(); + if (platform !== "darwin" && platform !== "linux") { + return; + } + + const platformArch = `${platform}-${os.arch()}`; + const packageName = `sherpa-onnx-${platformArch}`; + + let nativeDir: string; + try { + const binaryPath = requireFn.resolve(`${packageName}/sherpa-onnx.node`); + nativeDir = path.dirname(binaryPath); + } catch { + return; + } + + const envKey = platform === "darwin" ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH"; + const current = process.env[envKey]?.trim() ?? ""; + const entries = current.length > 0 ? current.split(":").filter(Boolean) : []; + if (entries.includes(nativeDir)) { + return; + } + + process.env[envKey] = entries.length > 0 ? `${nativeDir}:${entries.join(":")}` : nativeDir; +} + export function loadSherpaOnnx(): SherpaOnnxModule { if (cached) { return cached; } const require = createRequire(import.meta.url); + ensureSherpaNativeLibraryPath(require); cached = require("sherpa-onnx") as SherpaOnnxModule; return cached; } diff --git a/packages/server/src/server/test-utils/paseo-daemon.ts b/packages/server/src/server/test-utils/paseo-daemon.ts index 7024991b8..74da63fab 100644 --- a/packages/server/src/server/test-utils/paseo-daemon.ts +++ b/packages/server/src/server/test-utils/paseo-daemon.ts @@ -21,6 +21,10 @@ type TestPaseoDaemonOptions = { cleanup?: boolean; openai?: PaseoOpenAIConfig; speech?: PaseoSpeechConfig; + openrouterApiKey?: string | null; + voiceLlmProvider?: PaseoDaemonConfig["voiceLlmProvider"]; + voiceLlmProviderExplicit?: boolean; + voiceLlmModel?: string | null; dictationFinalTimeoutMs?: number; }; @@ -78,7 +82,10 @@ export async function createTestPaseoDaemon( appBaseUrl: "https://app.paseo.sh", openai: options.openai, speech: options.speech, - openrouterApiKey: null, + openrouterApiKey: options.openrouterApiKey ?? null, + voiceLlmProvider: options.voiceLlmProvider ?? null, + voiceLlmProviderExplicit: options.voiceLlmProviderExplicit ?? false, + voiceLlmModel: options.voiceLlmModel ?? null, dictationFinalTimeoutMs: options.dictationFinalTimeoutMs, downloadTokenTtlMs: options.downloadTokenTtlMs, }; diff --git a/packages/server/src/server/voice-local-agent.e2e.test.ts b/packages/server/src/server/voice-local-agent.e2e.test.ts new file mode 100644 index 000000000..0de10e6a1 --- /dev/null +++ b/packages/server/src/server/voice-local-agent.e2e.test.ts @@ -0,0 +1,124 @@ +import { afterAll, beforeAll, describe, expect, test } from "vitest"; + +import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js"; + +const openaiApiKey = process.env.OPENAI_API_KEY ?? null; +const shouldRun = + process.env.PASEO_VOICE_LOCAL_AGENT_E2E === "1" && + Boolean(openaiApiKey) && + !process.env.CI; + +function waitForSignal( + timeoutMs: number, + setup: ( + resolve: (value: T) => void, + reject: (error: Error) => void + ) => () => void +): Promise { + return new Promise((resolve, reject) => { + let cleanup: (() => void) | null = null; + const timeout = setTimeout(() => { + cleanup?.(); + reject(new Error(`Timeout waiting for event after ${timeoutMs}ms`)); + }, timeoutMs); + + cleanup = setup( + (value) => { + clearTimeout(timeout); + cleanup?.(); + resolve(value); + }, + (error) => { + clearTimeout(timeout); + cleanup?.(); + reject(error); + } + ); + }); +} + +(shouldRun ? describe : describe.skip)( + "voice local-agent e2e", + () => { + let ctx: DaemonTestContext; + + beforeAll(async () => { + ctx = await createDaemonTestContext({ + agentClients: {}, + openai: { apiKey: openaiApiKey! }, + speech: { + dictationSttProvider: "openai", + voiceSttProvider: "openai", + voiceTtsProvider: "openai", + }, + voiceLlmProvider: "codex", + voiceLlmProviderExplicit: true, + voiceLlmModel: "gpt-5.2-mini", + }); + }, 120000); + + afterAll(async () => { + await ctx.cleanup(); + }, 60000); + + test( + "routes voice turns through local agent speak tool", + async () => { + await ctx.client.setVoiceConversation(true, `voice-local-agent-${Date.now()}`); + + const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => { + const offAudio = ctx.client.on("audio_output", (msg) => { + if (msg.type !== "audio_output") return; + resolve({ chunkId: msg.payload.id }); + }); + const offError = ctx.client.on("activity_log", (msg) => { + if (msg.type !== "activity_log") return; + if (msg.payload.type !== "error") return; + reject(new Error(String(msg.payload.content))); + }); + return () => { + offAudio(); + offError(); + }; + }); + + const assistantLogPromise = waitForSignal(120000, (resolve, reject) => { + const offLog = ctx.client.on("activity_log", (msg) => { + if (msg.type !== "activity_log") return; + if (msg.payload.type !== "assistant") return; + const content = String(msg.payload.content ?? ""); + if (!content.trim()) return; + resolve(content); + }); + const offError = ctx.client.on("activity_log", (msg) => { + if (msg.type !== "activity_log") return; + if (msg.payload.type !== "error") return; + reject(new Error(String(msg.payload.content))); + }); + return () => { + offLog(); + offError(); + }; + }); + + ctx.client.sendUserMessage( + "Use the speak tool and say exactly: local voice agent path is working." + ); + + const [{ chunkId }, assistantText] = await Promise.all([ + audioPromise, + assistantLogPromise, + ]); + + expect(chunkId.length).toBeGreaterThan(0); + expect(assistantText.toLowerCase()).toContain("local voice agent path is working"); + + const agents = await ctx.client.fetchAgents(); + expect( + agents.some((agent) => String(agent.labels?.surface ?? "") === "voice") + ).toBe(false); + }, + 180000 + ); + } +); diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 4d347694e..6ac67ff41 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -23,6 +23,7 @@ import { VoiceConversationStore } from "./voice-conversation-store.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; export type AgentMcpTransportFactory = () => Promise; +type VoiceAgentProvider = "claude" | "codex" | "opencode"; type WebSocketServerConfig = { allowedOrigins: Set; @@ -76,8 +77,26 @@ export class VoiceAssistantWebSocketServer { } | null; private readonly voice: { openrouterApiKey?: string | null; + voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null; + voiceLlmProviderExplicit?: boolean; + voiceLlmDefaultProvider?: VoiceAgentProvider | null; voiceLlmModel?: string | null; + voiceLlmAvailability?: Record | null; + voiceAgentMcpUrl?: string | null; } | null; + private readonly voiceSpeakHandlers = new Map< + string, + (params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise + >(); + private readonly voiceCallerContexts = new Map< + string, + { + childAgentDefaultLabels?: Record; + lockedCwd?: string; + allowCustomCwd?: boolean; + enableVoiceTools?: boolean; + } + >(); constructor( server: HTTPServer, @@ -93,7 +112,12 @@ export class VoiceAssistantWebSocketServer { terminalManager?: TerminalManager | null, voice?: { openrouterApiKey?: string | null; + voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null; + voiceLlmProviderExplicit?: boolean; + voiceLlmDefaultProvider?: VoiceAgentProvider | null; voiceLlmModel?: string | null; + voiceLlmAvailability?: Record | null; + voiceAgentMcpUrl?: string | null; }, dictation?: { finalTimeoutMs?: number; @@ -225,6 +249,20 @@ export class VoiceAssistantWebSocketServer { this.terminalManager, this.voiceConversationStore, this.voice ?? undefined, + { + registerVoiceSpeakHandler: (agentId, handler) => { + this.voiceSpeakHandlers.set(agentId, handler); + }, + unregisterVoiceSpeakHandler: (agentId) => { + this.voiceSpeakHandlers.delete(agentId); + }, + registerVoiceCallerContext: (agentId, context) => { + this.voiceCallerContexts.set(agentId, context); + }, + unregisterVoiceCallerContext: (agentId) => { + this.voiceCallerContexts.delete(agentId); + }, + }, this.dictation ?? undefined ); @@ -263,6 +301,23 @@ export class VoiceAssistantWebSocketServer { }); } + public resolveVoiceSpeakHandler( + callerAgentId: string + ): ((params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise) | null { + return this.voiceSpeakHandlers.get(callerAgentId) ?? null; + } + + public resolveVoiceCallerContext( + callerAgentId: string + ): { + childAgentDefaultLabels?: Record; + lockedCwd?: string; + allowCustomCwd?: boolean; + enableVoiceTools?: boolean; + } | null { + return this.voiceCallerContexts.get(callerAgentId) ?? null; + } + private async detachSocket( ws: WebSocketLike, connectionLogger: pino.Logger, diff --git a/packages/server/src/test-utils/vitest-setup.ts b/packages/server/src/test-utils/vitest-setup.ts index 3ecf822c6..8f0beb4ad 100644 --- a/packages/server/src/test-utils/vitest-setup.ts +++ b/packages/server/src/test-utils/vitest-setup.ts @@ -1,7 +1,8 @@ import path from "node:path"; import dotenv from "dotenv"; -// Load repo-root .env for integration/E2E tests (OpenAI, etc.) +// Load package-local .env.test first for integration/E2E credentials, then repo-root .env fallback. +dotenv.config({ path: path.resolve(process.cwd(), ".env.test"), override: true }); dotenv.config({ path: path.resolve(process.cwd(), "../.env") }); process.env.GIT_TERMINAL_PROMPT = "0"; From 7d388b456f1557edac92437151fca6a2cfe6ba12 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 15:58:02 +0700 Subject: [PATCH 02/21] Tune web voice VAD for short utterances --- packages/app/src/contexts/voice-context.tsx | 12 ++++--- .../app/src/voice/speech-segmenter.test.ts | 33 +++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/packages/app/src/contexts/voice-context.tsx b/packages/app/src/contexts/voice-context.tsx index 83fa5deee..0f4cb147c 100644 --- a/packages/app/src/contexts/voice-context.tsx +++ b/packages/app/src/contexts/voice-context.tsx @@ -8,6 +8,10 @@ import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; const VOICE_CONVERSATION_ID_STORAGE_KEY = "@paseo:voice-conversation-id"; const KEEP_AWAKE_TAG = "paseo:voice"; +const VOICE_VAD_VOLUME_THRESHOLD = 0.18; +const VOICE_VAD_SILENCE_DURATION_MS = 1400; +const VOICE_VAD_SPEECH_CONFIRMATION_MS = 120; +const VOICE_VAD_DETECTION_GRACE_PERIOD_MS = 700; interface VoiceContextValue { isVoiceMode: boolean; @@ -123,10 +127,10 @@ export function VoiceProvider({ children }: VoiceProviderProps) { console.error("[Voice] Cannot handle error - setMessages not available from SessionState"); } }, - volumeThreshold: 0.3, - silenceDuration: 2000, - speechConfirmationDuration: 300, - detectionGracePeriod: 200, + volumeThreshold: VOICE_VAD_VOLUME_THRESHOLD, + silenceDuration: VOICE_VAD_SILENCE_DURATION_MS, + speechConfirmationDuration: VOICE_VAD_SPEECH_CONFIRMATION_MS, + detectionGracePeriod: VOICE_VAD_DETECTION_GRACE_PERIOD_MS, }); useEffect(() => { diff --git a/packages/app/src/voice/speech-segmenter.test.ts b/packages/app/src/voice/speech-segmenter.test.ts index c9b0278a5..a02428a26 100644 --- a/packages/app/src/voice/speech-segmenter.test.ts +++ b/packages/app/src/voice/speech-segmenter.test.ts @@ -88,4 +88,37 @@ describe("SpeechSegmenter", () => { expect(lastCall.isLast).toBe(true); expect(lastCall.audioData.length).toBeGreaterThan(0); }); + + it("keeps detection alive across a brief pause and confirms short utterances", () => { + const onSpeechStart = vi.fn(); + const detectingChanges: boolean[] = []; + + const segmenter = new SpeechSegmenter( + { + enableContinuousStreaming: false, + volumeThreshold: 0.18, + silenceDurationMs: 1400, + speechConfirmationMs: 120, + detectionGracePeriodMs: 700, + minChunkDurationMs: 100, + pcmSampleRate: 1000, + }, + { + onSpeechStart, + onDetectingChange: (v) => detectingChanges.push(v), + } + ); + + const t0 = 20_000; + + segmenter.pushVolumeLevel(0.4, t0); + segmenter.pushPcmChunk(mkPcmBytes(20)); + segmenter.pushVolumeLevel(0.0, t0 + 80); + segmenter.pushPcmChunk(mkPcmBytes(20)); + segmenter.pushVolumeLevel(0.4, t0 + 140); + segmenter.pushPcmChunk(mkPcmBytes(20)); + + expect(detectingChanges).toContain(true); + expect(onSpeechStart).toHaveBeenCalledTimes(1); + }); }); From bc494501f6a5bfa0cb49914d25dda139bf2a3677 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 16:07:18 +0700 Subject: [PATCH 03/21] Expose voice assistant agent in standard listings --- packages/server/src/server/session.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 409587d5e..c245db56a 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -4672,7 +4672,6 @@ export class Session { cwd, modeId: VOICE_AGENT_DEFAULT_MODE[provider], ...(model ? { model } : {}), - internal: true, mcpServers: { paseo: { type: "http", From 217f4b1cb5008b1f1ab53f7594367b0d758f4b4d Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 16:21:41 +0700 Subject: [PATCH 04/21] Unify cwd validation and improve MCP tool log visibility --- .../src/server/agent/activity-curator.test.ts | 32 ++++++++++++ .../src/server/agent/activity-curator.ts | 34 +++++++++++++ .../src/server/agent/mcp-server.test.ts | 42 ++++++++++++---- .../server/src/server/agent/mcp-server.ts | 3 ++ .../agent/working-directory-validation.ts | 20 ++++++++ packages/server/src/server/session.ts | 49 ++++++++++++------- .../server/src/utils/tool-call-parsers.ts | 1 + 7 files changed, 154 insertions(+), 27 deletions(-) create mode 100644 packages/server/src/server/agent/working-directory-validation.ts diff --git a/packages/server/src/server/agent/activity-curator.test.ts b/packages/server/src/server/agent/activity-curator.test.ts index a0e10b615..dad730222 100644 --- a/packages/server/src/server/agent/activity-curator.test.ts +++ b/packages/server/src/server/agent/activity-curator.test.ts @@ -328,5 +328,37 @@ describe("curateAgentActivity", () => { expect(result).toBe("[Grep] TODO"); }); + + test("shows speak tool text input", () => { + const timeline: AgentTimelineItem[] = [ + { + type: "tool_call", + callId: "s1", + name: "speak", + input: { text: "hello from voice" }, + status: "completed", + }, + ]; + + const result = curateAgentActivity(timeline); + expect(result).toBe('[speak] {"text":"hello from voice"}'); + }); + + test("shows MCP tool input JSON", () => { + const timeline: AgentTimelineItem[] = [ + { + type: "tool_call", + callId: "m1", + name: "paseo__create_agent", + input: { cwd: "/tmp/repo", initialPrompt: "do the thing" }, + status: "completed", + }, + ]; + + const result = curateAgentActivity(timeline); + expect(result).toBe( + '[paseo__create_agent] {"cwd":"/tmp/repo","initialPrompt":"do the thing"}' + ); + }); }); }); diff --git a/packages/server/src/server/agent/activity-curator.ts b/packages/server/src/server/agent/activity-curator.ts index 806ad6e2e..5c8425c33 100644 --- a/packages/server/src/server/agent/activity-curator.ts +++ b/packages/server/src/server/agent/activity-curator.ts @@ -2,6 +2,7 @@ import type { AgentTimelineItem } from "./agent-sdk-types.js"; import { extractPrincipalParam } from "../../utils/tool-call-parsers.js"; const DEFAULT_MAX_ITEMS = 40; +const MAX_TOOL_INPUT_CHARS = 400; function appendText(buffer: string, text: string): string { const normalized = text.trim(); @@ -25,6 +26,34 @@ function flushBuffers(lines: string[], buffers: { message: string; thought: stri buffers.thought = ""; } +function isLikelyMcpToolCall(name: string): boolean { + const normalized = name.toLowerCase(); + return ( + normalized.includes("mcp") || + normalized.includes("paseo") || + normalized.includes("__") || + normalized === "speak" + ); +} + +function formatToolInputJson(input: unknown): string | null { + if (input === undefined) { + return null; + } + try { + const encoded = JSON.stringify(input); + if (!encoded) { + return null; + } + if (encoded.length <= MAX_TOOL_INPUT_CHARS) { + return encoded; + } + return `${encoded.slice(0, MAX_TOOL_INPUT_CHARS)}...`; + } catch { + return null; + } +} + /** * Collapse timeline items: * - Dedupe tool calls by callId (pending/completed -> single) @@ -127,6 +156,11 @@ export function curateAgentActivity( break; case "tool_call": { flushBuffers(lines, buffers); + const inputJson = formatToolInputJson(item.input); + if (isLikelyMcpToolCall(item.name) && inputJson) { + lines.push(`[${item.name}] ${inputJson}`); + break; + } const principal = extractPrincipalParam(item.input); if (principal) { lines.push(`[${item.name}] ${principal}`); diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index ce7186091..65a044174 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -1,4 +1,7 @@ import { describe, expect, it, vi } from "vitest"; +import { mkdtemp, mkdir, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; import { createTestLogger } from "../../test-utils/test-logger.js"; import { createAgentMcpServer } from "./mcp-server.js"; @@ -48,6 +51,7 @@ function createTestDeps(): TestDeps { describe("create_agent MCP tool", () => { const logger = createTestLogger(); + const existingCwd = process.cwd(); it("requires a concise title no longer than 60 characters", async () => { const { agentManager, agentStorage } = createTestDeps(); @@ -56,7 +60,7 @@ describe("create_agent MCP tool", () => { expect(tool).toBeDefined(); const missingTitle = await tool.inputSchema.safeParseAsync({ - cwd: "/tmp/repo", + cwd: existingCwd, initialMode: "default", initialPrompt: "test", }); @@ -64,7 +68,7 @@ describe("create_agent MCP tool", () => { expect(missingTitle.error.issues[0].path).toEqual(["title"]); const tooLong = await tool.inputSchema.safeParseAsync({ - cwd: "/tmp/repo", + cwd: existingCwd, initialMode: "default", title: "x".repeat(61), initialPrompt: "test", @@ -73,7 +77,7 @@ describe("create_agent MCP tool", () => { expect(tooLong.error.issues[0].path).toEqual(["title"]); const ok = await tool.inputSchema.safeParseAsync({ - cwd: "/tmp/repo", + cwd: existingCwd, initialMode: "default", title: "Short title", initialPrompt: "test", @@ -86,7 +90,7 @@ describe("create_agent MCP tool", () => { const server = await createAgentMcpServer({ agentManager, agentStorage, logger }); const tool = (server as any)._registeredTools["create_agent"]; const parsed = await tool.inputSchema.safeParseAsync({ - cwd: "/tmp/repo", + cwd: existingCwd, initialMode: "default", title: "Short title", }); @@ -94,6 +98,20 @@ describe("create_agent MCP tool", () => { expect(parsed.error.issues.some((issue: { path: string[] }) => issue.path[0] === "initialPrompt")).toBe(true); }); + it("fails immediately when cwd does not exist", async () => { + const { agentManager, agentStorage } = createTestDeps(); + const server = await createAgentMcpServer({ agentManager, agentStorage, logger }); + const tool = (server as any)._registeredTools["create_agent"]; + + await expect( + tool.callback({ + cwd: "/path/that/does/not/exist", + title: "Short title", + initialPrompt: "Do work", + }) + ).rejects.toThrow("Working directory does not exist"); + }); + it("passes caller-provided titles directly into createAgent", async () => { const { agentManager, agentStorage, spies } = createTestDeps(); spies.agentManager.createAgent.mockResolvedValue({ @@ -108,14 +126,14 @@ describe("create_agent MCP tool", () => { const server = await createAgentMcpServer({ agentManager, agentStorage, logger }); const tool = (server as any)._registeredTools["create_agent"]; await tool.callback({ - cwd: "/tmp/repo", + cwd: existingCwd, title: " Fix auth bug ", initialPrompt: "Do work", }); expect(spies.agentManager.createAgent).toHaveBeenCalledWith( expect.objectContaining({ - cwd: "/tmp/repo", + cwd: existingCwd, title: "Fix auth bug", }), undefined, @@ -137,7 +155,7 @@ describe("create_agent MCP tool", () => { const server = await createAgentMcpServer({ agentManager, agentStorage, logger }); const tool = (server as any)._registeredTools["create_agent"]; await tool.callback({ - cwd: "/tmp/repo", + cwd: existingCwd, title: " Fix auth ", initialPrompt: "Do work", }); @@ -153,15 +171,18 @@ describe("create_agent MCP tool", () => { it("allows caller agents to override cwd and applies caller context labels", async () => { const { agentManager, agentStorage, spies } = createTestDeps(); + const baseDir = await mkdtemp(join(tmpdir(), "paseo-mcp-test-")); + const subdir = join(baseDir, "subdir"); + await mkdir(subdir, { recursive: true }); spies.agentManager.getAgent.mockReturnValue({ id: "voice-agent", - cwd: "/tmp/voice", + cwd: baseDir, provider: "codex", currentModeId: "full-access", } as ManagedAgent); spies.agentManager.createAgent.mockResolvedValue({ id: "child-agent", - cwd: "/tmp/voice/subdir", + cwd: subdir, lifecycle: "idle", currentModeId: null, availableModes: [], @@ -189,11 +210,12 @@ describe("create_agent MCP tool", () => { expect(spies.agentManager.createAgent).toHaveBeenCalledWith( expect.objectContaining({ - cwd: "/tmp/voice/subdir", + cwd: subdir, }), undefined, { labels: { ui: "true" } } ); + await rm(baseDir, { recursive: true, force: true }); }); }); diff --git a/packages/server/src/server/agent/mcp-server.ts b/packages/server/src/server/agent/mcp-server.ts index 83d09a9ab..41c69f68f 100644 --- a/packages/server/src/server/agent/mcp-server.ts +++ b/packages/server/src/server/agent/mcp-server.ts @@ -28,6 +28,7 @@ import { AgentStorage } from "./agent-storage.js"; import { createWorktree } from "../../utils/worktree.js"; import { WaitForAgentTracker } from "./wait-for-agent-tracker.js"; import { scheduleAgentMetadataGeneration } from "./agent-metadata-generator.js"; +import { validateWorkingDirectoryExists } from "./working-directory-validation.js"; export interface AgentMcpServerOptions { agentManager: AgentManager; @@ -496,6 +497,7 @@ export async function createAgentMcpServer( } = topLevelArgs; resolvedCwd = expandPath(cwd); + await validateWorkingDirectoryExists(resolvedCwd); if (worktreeName) { if (!baseBranch) { @@ -513,6 +515,7 @@ export async function createAgentMcpServer( resolvedMode = initialMode; } + await validateWorkingDirectoryExists(resolvedCwd); const provider: AgentProvider = agentType ?? "claude"; const normalizedTitle = title?.trim() ?? null; diff --git a/packages/server/src/server/agent/working-directory-validation.ts b/packages/server/src/server/agent/working-directory-validation.ts new file mode 100644 index 000000000..ac6a567d6 --- /dev/null +++ b/packages/server/src/server/agent/working-directory-validation.ts @@ -0,0 +1,20 @@ +import { stat } from "node:fs/promises"; + +export async function validateWorkingDirectoryExists( + cwd: string +): Promise { + try { + const cwdStats = await stat(cwd); + if (!cwdStats.isDirectory()) { + throw new Error(`Working directory is not a directory: ${cwd}`); + } + } catch (error) { + if (error instanceof Error && "code" in error && (error as NodeJS.ErrnoException).code === "ENOENT") { + throw new Error(`Working directory does not exist: ${cwd}`); + } + if (error instanceof Error) { + throw error; + } + throw new Error(`Failed to access working directory: ${cwd}`); + } +} diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index c245db56a..1863b77fe 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -63,6 +63,7 @@ import { AgentManager } from "./agent/agent-manager.js"; import type { ManagedAgent } from "./agent/agent-manager.js"; import { scheduleAgentMetadataGeneration } from "./agent/agent-metadata-generator.js"; import { toAgentPayload } from "./agent/agent-projections.js"; +import { validateWorkingDirectoryExists } from "./agent/working-directory-validation.js"; import { StructuredAgentResponseError, generateStructuredAgentResponse, @@ -298,6 +299,7 @@ export class Session { // Voice mode state private isVoiceMode = false; private speechInProgress = false; + // OpenRouter voice-only conversation storage identifier. private voiceConversationId: string | null = null; private readonly dictationStreamManager: DictationStreamManager; @@ -1400,6 +1402,16 @@ export class Session { } this.isVoiceMode = true; + if (this.voiceLlmProvider !== "openrouter") { + this.voiceConversationId = null; + this.voiceAssistantAgentId = voiceConversationId; + this.sessionLogger.info( + { voiceAssistantAgentId: this.voiceAssistantAgentId }, + "Voice conversation enabled (agent-backed)" + ); + return; + } + this.voiceConversationId = voiceConversationId; const loaded = await this.voiceConversationStore.load( @@ -1416,6 +1428,14 @@ export class Session { } this.isVoiceMode = false; + if (this.voiceLlmProvider !== "openrouter") { + this.sessionLogger.info( + { voiceAssistantAgentId: this.voiceAssistantAgentId }, + "Voice conversation disabled (agent-backed)" + ); + return; + } + const idToPersist = this.voiceConversationId; if (idToPersist) { try { @@ -1548,21 +1568,7 @@ export class Session { try { // Validate that the working directory exists const resolvedCwd = expandTilde(config.cwd); - try { - const stats = await stat(resolvedCwd); - if (!stats.isDirectory()) { - throw new Error( - `Working directory is not a directory: ${config.cwd}` - ); - } - } catch (statError: any) { - if (statError.code === "ENOENT") { - throw new Error( - `Working directory does not exist: ${config.cwd}` - ); - } - throw statError; - } + await validateWorkingDirectoryExists(resolvedCwd); const { sessionConfig, worktreeConfig } = await this.buildAgentSessionConfig( config, @@ -4653,11 +4659,20 @@ export class Session { if (existing) { return existing.id; } - this.voiceAssistantAgentId = null; + try { + const hydrated = await this.ensureAgentLoaded(this.voiceAssistantAgentId); + this.voiceAssistantAgentId = hydrated.id; + return hydrated.id; + } catch (error) { + this.sessionLogger.debug( + { err: error, voiceAssistantAgentId: this.voiceAssistantAgentId }, + "Voice assistant agent not found in active/persisted state; creating new session" + ); + } } const provider = this.resolveVoiceAgentProvider(); - const voiceAgentId = `voice-${uuidv4()}`; + const voiceAgentId = this.voiceAssistantAgentId ?? uuidv4(); const cwd = join(this.paseoHome, "voice-agent-workspace"); await mkdir(cwd, { recursive: true }); diff --git a/packages/server/src/utils/tool-call-parsers.ts b/packages/server/src/utils/tool-call-parsers.ts index cc320b27b..a8cf81df0 100644 --- a/packages/server/src/utils/tool-call-parsers.ts +++ b/packages/server/src/utils/tool-call-parsers.ts @@ -28,6 +28,7 @@ const PrincipalParamSchema = z.union([ z.object({ pattern: z.string() }).transform((d) => ({ type: "text" as const, value: d.pattern })), z.object({ query: z.string() }).transform((d) => ({ type: "text" as const, value: d.query })), z.object({ url: z.string() }).transform((d) => ({ type: "text" as const, value: d.url })), + z.object({ text: z.string() }).transform((d) => ({ type: "text" as const, value: d.text })), // Files array (Codex apply_patch) z.object({ files: z.array(FileEntrySchema).nonempty() }).transform((d) => ({ type: "path" as const, value: d.files[0].path })), // TodoWrite - show in_progress item or count From bbad9b284a64f3423ae3b6402eb3f3b7a7e8e139 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 16:22:58 +0700 Subject: [PATCH 05/21] Disable openrouter voice conversation persistence --- packages/server/src/server/session.ts | 82 ++++++++++++++------------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 1863b77fe..4e038dfe3 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -1178,6 +1178,20 @@ export class Session { voiceConversationId: string, requestId: string ): Promise { + if (this.voiceLlmProvider === "openrouter") { + this.voiceConversationId = null; + this.messages = []; + this.emit({ + type: "voice_conversation_loaded", + payload: { + voiceConversationId, + messageCount: 0, + requestId, + }, + }); + return; + } + const loaded = await this.voiceConversationStore.load( this.sessionLogger, voiceConversationId @@ -1200,6 +1214,17 @@ export class Session { * List all voice conversations */ public async handleListVoiceConversations(requestId: string): Promise { + if (this.voiceLlmProvider === "openrouter") { + this.emit({ + type: "list_voice_conversations_response", + payload: { + conversations: [], + requestId, + }, + }); + return; + } + try { const conversations = await this.voiceConversationStore.list(this.sessionLogger); this.emit({ @@ -1237,6 +1262,18 @@ export class Session { voiceConversationId: string, requestId: string ): Promise { + if (this.voiceLlmProvider === "openrouter") { + this.emit({ + type: "delete_voice_conversation_response", + payload: { + voiceConversationId, + success: true, + requestId, + }, + }); + return; + } + try { await this.voiceConversationStore.delete(this.sessionLogger, voiceConversationId); this.emit({ @@ -1412,16 +1449,12 @@ export class Session { return; } - this.voiceConversationId = voiceConversationId; - - const loaded = await this.voiceConversationStore.load( - this.sessionLogger, - voiceConversationId - ); - this.messages = loaded ?? []; + // OpenRouter voice mode is always ephemeral: no out-of-band persistence. + this.voiceConversationId = null; + this.messages = []; this.sessionLogger.info( - { voiceConversationId, messageCount: this.messages.length }, + { messageCount: this.messages.length }, "Voice conversation enabled" ); return; @@ -1435,21 +1468,8 @@ export class Session { ); return; } - - const idToPersist = this.voiceConversationId; - if (idToPersist) { - try { - await this.voiceConversationStore.save( - this.sessionLogger, - idToPersist, - this.messages - ); - } catch (error) { - this.sessionLogger.warn({ err: error }, "Failed to persist voice conversation"); - } - } - - this.sessionLogger.info({ voiceConversationId: idToPersist }, "Voice conversation disabled"); + this.voiceConversationId = null; + this.sessionLogger.info("Voice conversation disabled"); } /** @@ -4941,22 +4961,6 @@ export class Session { `onFinish - saved message with ${newMessages.length} steps` ); } - - // Persist voice conversation to disk (best-effort; voice-only) - if (enableTTS && this.voiceConversationId) { - try { - await this.voiceConversationStore.save( - this.sessionLogger, - this.voiceConversationId, - this.messages - ); - } catch (error) { - this.sessionLogger.warn( - { err: error, voiceConversationId: this.voiceConversationId }, - "Failed to persist voice conversation" - ); - } - } }, onChunk: async ({ chunk }) => { if (chunk.type === "text-delta") { From c172e5c939854b343fc75fe841b962708472e010 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 16:31:41 +0700 Subject: [PATCH 06/21] Remove voice conversation store and centralize cwd validation --- .../src/server/agent/agent-manager.test.ts | 20 +++ .../server/src/server/agent/agent-manager.ts | 15 ++ .../src/server/agent/mcp-server.test.ts | 7 +- .../server/src/server/agent/mcp-server.ts | 3 - .../agent/working-directory-validation.ts | 20 --- packages/server/src/server/session.ts | 105 ++++++------- .../src/server/voice-conversation-store.ts | 140 ----------------- .../server/voice-conversations.e2e.test.ts | 147 ------------------ .../server/src/server/websocket-server.ts | 6 - 9 files changed, 88 insertions(+), 375 deletions(-) delete mode 100644 packages/server/src/server/agent/working-directory-validation.ts delete mode 100644 packages/server/src/server/voice-conversation-store.ts delete mode 100644 packages/server/src/server/voice-conversations.e2e.test.ts diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 067e3faf3..6e69d85fc 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -129,6 +129,26 @@ describe("AgentManager", () => { expect(snapshot.model).toBeUndefined(); }); + test("createAgent fails when cwd does not exist", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); + const storagePath = join(workdir, "agents"); + const storage = new AgentStorage(storagePath, logger); + const manager = new AgentManager({ + clients: { + codex: new TestAgentClient(), + }, + registry: storage, + logger, + }); + + await expect( + manager.createAgent({ + provider: "codex", + cwd: join(workdir, "does-not-exist"), + }) + ).rejects.toThrow("Working directory does not exist"); + }); + test("createAgent persists provided title before returning", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index f3578ba9b..76ec69fc4 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; import { resolve } from "node:path"; +import { stat } from "node:fs/promises"; import { AGENT_LIFECYCLE_STATUSES, type AgentLifecycleStatus, @@ -1319,6 +1320,20 @@ export class AgentManager { // Always resolve cwd to absolute path for consistent history file lookup if (normalized.cwd) { normalized.cwd = resolve(normalized.cwd); + try { + const cwdStats = await stat(normalized.cwd); + if (!cwdStats.isDirectory()) { + throw new Error(`Working directory is not a directory: ${normalized.cwd}`); + } + } catch (error) { + if (error instanceof Error && "code" in error && (error as NodeJS.ErrnoException).code === "ENOENT") { + throw new Error(`Working directory does not exist: ${normalized.cwd}`); + } + if (error instanceof Error) { + throw error; + } + throw new Error(`Failed to access working directory: ${normalized.cwd}`); + } } if (typeof normalized.model === "string") { diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index 65a044174..8deba1996 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -98,8 +98,11 @@ describe("create_agent MCP tool", () => { expect(parsed.error.issues.some((issue: { path: string[] }) => issue.path[0] === "initialPrompt")).toBe(true); }); - it("fails immediately when cwd does not exist", async () => { - const { agentManager, agentStorage } = createTestDeps(); + it("surfaces createAgent validation failures", async () => { + const { agentManager, agentStorage, spies } = createTestDeps(); + spies.agentManager.createAgent.mockRejectedValue( + new Error("Working directory does not exist: /path/that/does/not/exist") + ); const server = await createAgentMcpServer({ agentManager, agentStorage, logger }); const tool = (server as any)._registeredTools["create_agent"]; diff --git a/packages/server/src/server/agent/mcp-server.ts b/packages/server/src/server/agent/mcp-server.ts index 41c69f68f..83d09a9ab 100644 --- a/packages/server/src/server/agent/mcp-server.ts +++ b/packages/server/src/server/agent/mcp-server.ts @@ -28,7 +28,6 @@ import { AgentStorage } from "./agent-storage.js"; import { createWorktree } from "../../utils/worktree.js"; import { WaitForAgentTracker } from "./wait-for-agent-tracker.js"; import { scheduleAgentMetadataGeneration } from "./agent-metadata-generator.js"; -import { validateWorkingDirectoryExists } from "./working-directory-validation.js"; export interface AgentMcpServerOptions { agentManager: AgentManager; @@ -497,7 +496,6 @@ export async function createAgentMcpServer( } = topLevelArgs; resolvedCwd = expandPath(cwd); - await validateWorkingDirectoryExists(resolvedCwd); if (worktreeName) { if (!baseBranch) { @@ -515,7 +513,6 @@ export async function createAgentMcpServer( resolvedMode = initialMode; } - await validateWorkingDirectoryExists(resolvedCwd); const provider: AgentProvider = agentType ?? "claude"; const normalizedTitle = title?.trim() ?? null; diff --git a/packages/server/src/server/agent/working-directory-validation.ts b/packages/server/src/server/agent/working-directory-validation.ts deleted file mode 100644 index ac6a567d6..000000000 --- a/packages/server/src/server/agent/working-directory-validation.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { stat } from "node:fs/promises"; - -export async function validateWorkingDirectoryExists( - cwd: string -): Promise { - try { - const cwdStats = await stat(cwd); - if (!cwdStats.isDirectory()) { - throw new Error(`Working directory is not a directory: ${cwd}`); - } - } catch (error) { - if (error instanceof Error && "code" in error && (error as NodeJS.ErrnoException).code === "ENOENT") { - throw new Error(`Working directory does not exist: ${cwd}`); - } - if (error instanceof Error) { - throw error; - } - throw new Error(`Failed to access working directory: ${cwd}`); - } -} diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 4e038dfe3..c145a7e14 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -39,7 +39,6 @@ import { isPaseoDictationDebugEnabled } from "./agent/recordings-debug.js"; import { DictationStreamManager, } from "./dictation/dictation-stream-manager.js"; -import type { VoiceConversationStore } from "./voice-conversation-store.js"; import { buildConfigOverrides, buildSessionConfig, @@ -63,7 +62,6 @@ import { AgentManager } from "./agent/agent-manager.js"; import type { ManagedAgent } from "./agent/agent-manager.js"; import { scheduleAgentMetadataGeneration } from "./agent/agent-metadata-generator.js"; import { toAgentPayload } from "./agent/agent-projections.js"; -import { validateWorkingDirectoryExists } from "./agent/working-directory-validation.js"; import { StructuredAgentResponseError, generateStructuredAgentResponse, @@ -288,7 +286,6 @@ export class Session { private readonly sessionId: string; private readonly onMessage: (msg: SessionOutboundMessage) => void; private readonly sessionLogger: pino.Logger; - private readonly voiceConversationStore: VoiceConversationStore; private readonly paseoHome: string; // State machine @@ -299,8 +296,6 @@ export class Session { // Voice mode state private isVoiceMode = false; private speechInProgress = false; - // OpenRouter voice-only conversation storage identifier. - private voiceConversationId: string | null = null; private readonly dictationStreamManager: DictationStreamManager; @@ -381,7 +376,6 @@ export class Session { stt: SpeechToTextProvider | null, tts: TextToSpeechProvider | null, terminalManager: TerminalManager | null, - voiceConversationStore: VoiceConversationStore, voice?: { openrouterApiKey?: string | null; voiceLlmProvider?: VoiceLlmProvider | null; @@ -412,7 +406,6 @@ export class Session { this.agentStorage = agentStorage; this.createAgentMcpTransport = createAgentMcpTransport; this.terminalManager = terminalManager; - this.voiceConversationStore = voiceConversationStore; this.openrouterApiKey = voice?.openrouterApiKey ?? null; this.voiceLlmProvider = voice?.voiceLlmProvider ?? null; this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false; @@ -1179,7 +1172,7 @@ export class Session { requestId: string ): Promise { if (this.voiceLlmProvider === "openrouter") { - this.voiceConversationId = null; + this.voiceAssistantAgentId = null; this.messages = []; this.emit({ type: "voice_conversation_loaded", @@ -1191,20 +1184,13 @@ export class Session { }); return; } - - const loaded = await this.voiceConversationStore.load( - this.sessionLogger, - voiceConversationId - ); - - this.voiceConversationId = voiceConversationId; - this.messages = loaded ?? []; + this.voiceAssistantAgentId = voiceConversationId; this.emit({ type: "voice_conversation_loaded", payload: { voiceConversationId, - messageCount: this.messages.length, + messageCount: 0, requestId, }, }); @@ -1214,27 +1200,31 @@ export class Session { * List all voice conversations */ public async handleListVoiceConversations(requestId: string): Promise { - if (this.voiceLlmProvider === "openrouter") { - this.emit({ - type: "list_voice_conversations_response", - payload: { - conversations: [], - requestId, - }, - }); - return; - } - try { - const conversations = await this.voiceConversationStore.list(this.sessionLogger); + const agents = await this.agentStorage.list(); + const conversations = agents + .filter( + (agent) => + agent.labels?.surface === "voice" && + !agent.archivedAt && + !agent.internal + ) + .map((agent) => ({ + id: agent.id, + lastUpdated: new Date( + agent.lastActivityAt ?? agent.updatedAt + ).toISOString(), + messageCount: 0, + })) + .sort( + (a, b) => + new Date(b.lastUpdated).getTime() - + new Date(a.lastUpdated).getTime() + ); this.emit({ type: "list_voice_conversations_response", payload: { - conversations: conversations.map((conv) => ({ - id: conv.id, - lastUpdated: conv.lastUpdated.toISOString(), - messageCount: conv.messageCount, - })), + conversations, requestId, }, }); @@ -1262,20 +1252,26 @@ export class Session { voiceConversationId: string, requestId: string ): Promise { - if (this.voiceLlmProvider === "openrouter") { - this.emit({ - type: "delete_voice_conversation_response", - payload: { - voiceConversationId, - success: true, - requestId, - }, - }); - return; - } - try { - await this.voiceConversationStore.delete(this.sessionLogger, voiceConversationId); + const record = await this.agentStorage.get(voiceConversationId); + if (!record || record.labels?.surface !== "voice") { + this.emit({ + type: "delete_voice_conversation_response", + payload: { + voiceConversationId, + success: false, + error: "Voice conversation not found", + requestId, + }, + }); + return; + } + + const live = this.agentManager.getAgent(voiceConversationId); + if (live) { + await this.agentManager.closeAgent(voiceConversationId); + } + await this.agentStorage.remove(voiceConversationId); this.emit({ type: "delete_voice_conversation_response", payload: { @@ -1440,7 +1436,6 @@ export class Session { this.isVoiceMode = true; if (this.voiceLlmProvider !== "openrouter") { - this.voiceConversationId = null; this.voiceAssistantAgentId = voiceConversationId; this.sessionLogger.info( { voiceAssistantAgentId: this.voiceAssistantAgentId }, @@ -1449,8 +1444,8 @@ export class Session { return; } - // OpenRouter voice mode is always ephemeral: no out-of-band persistence. - this.voiceConversationId = null; + // OpenRouter voice mode is always ephemeral. + this.voiceAssistantAgentId = null; this.messages = []; this.sessionLogger.info( @@ -1468,7 +1463,7 @@ export class Session { ); return; } - this.voiceConversationId = null; + this.voiceAssistantAgentId = null; this.sessionLogger.info("Voice conversation disabled"); } @@ -1586,10 +1581,6 @@ export class Session { ); try { - // Validate that the working directory exists - const resolvedCwd = expandTilde(config.cwd); - await validateWorkingDirectoryExists(resolvedCwd); - const { sessionConfig, worktreeConfig } = await this.buildAgentSessionConfig( config, git, @@ -5436,11 +5427,11 @@ export class Session { const dumpDir = join(process.cwd(), ".debug.conversations"); await mkdir(dumpDir, { recursive: true }); - const filename = `${this.voiceConversationId ?? this.sessionId}-${this.turnIndex}.json`; + const filename = `${this.sessionId}-${this.turnIndex}.json`; const filepath = join(dumpDir, filename); const dump = { - voiceConversationId: this.voiceConversationId, + voiceAssistantAgentId: this.voiceAssistantAgentId, sessionId: this.sessionId, turnIndex: this.turnIndex, timestamp: new Date().toISOString(), diff --git a/packages/server/src/server/voice-conversation-store.ts b/packages/server/src/server/voice-conversation-store.ts deleted file mode 100644 index c18ad654a..000000000 --- a/packages/server/src/server/voice-conversation-store.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { readFile, writeFile, readdir, unlink, mkdir, stat } from "fs/promises"; -import { join } from "path"; -import type { ModelMessage } from "@ai-sdk/provider-utils"; -import { standardizePrompt } from "ai/internal"; - -type LoggerLike = { - child(bindings: Record): LoggerLike; - info(...args: any[]): void; - debug(...args: any[]): void; - warn(...args: any[]): void; - error(...args: any[]): void; -}; - -function getLogger(logger: LoggerLike): LoggerLike { - return logger.child({ module: "voice-conversation-store" }); -} - -export interface VoiceConversationMetadata { - id: string; - lastUpdated: Date; - messageCount: number; -} - -interface VoiceConversationData { - voiceConversationId: string; - lastUpdated: string; - messageCount: number; - messages: ModelMessage[]; -} - -export class VoiceConversationStore { - private readonly baseDir: string; - - constructor(baseDir: string) { - this.baseDir = baseDir; - } - - private async ensureBaseDir(): Promise { - await mkdir(this.baseDir, { recursive: true }); - } - - public async save( - logger: LoggerLike, - voiceConversationId: string, - messages: ModelMessage[] - ): Promise { - const log = getLogger(logger); - await this.ensureBaseDir(); - - const filepath = join(this.baseDir, `${voiceConversationId}.json`); - const data: VoiceConversationData = { - voiceConversationId, - lastUpdated: new Date().toISOString(), - messageCount: messages.length, - messages, - }; - - await writeFile(filepath, JSON.stringify(data, null, 2), "utf-8"); - log.debug({ voiceConversationId, messageCount: messages.length }, "Saved voice conversation"); - } - - /** - * Load voice conversation from disk. - * Returns null when missing or invalid (best-effort). - */ - public async load( - logger: LoggerLike, - voiceConversationId: string - ): Promise { - const log = getLogger(logger); - const filepath = join(this.baseDir, `${voiceConversationId}.json`); - - try { - await stat(filepath); - } catch { - log.debug({ voiceConversationId }, "Voice conversation not found"); - return null; - } - - try { - const fileContent = await readFile(filepath, "utf-8"); - const data: VoiceConversationData = JSON.parse(fileContent); - - const result = await standardizePrompt({ prompt: data.messages }); - - log.debug( - { voiceConversationId, messageCount: data.messageCount }, - "Loaded voice conversation" - ); - - return result.messages as ModelMessage[]; - } catch (error) { - log.warn({ err: error, voiceConversationId }, "Failed to load voice conversation"); - return null; - } - } - - public async list(logger: LoggerLike): Promise { - const log = getLogger(logger); - try { - await this.ensureBaseDir(); - - const files = await readdir(this.baseDir); - const jsonFiles = files.filter((f) => f.endsWith(".json")); - const conversations: VoiceConversationMetadata[] = []; - - for (const file of jsonFiles) { - try { - const filepath = join(this.baseDir, file); - const fileContent = await readFile(filepath, "utf-8"); - const data: VoiceConversationData = JSON.parse(fileContent); - - conversations.push({ - id: data.voiceConversationId, - lastUpdated: new Date(data.lastUpdated), - messageCount: data.messageCount, - }); - } catch (error) { - log.warn({ err: error, file }, "Failed to read voice conversation file"); - } - } - - conversations.sort( - (a, b) => b.lastUpdated.getTime() - a.lastUpdated.getTime() - ); - return conversations; - } catch (error) { - log.warn({ err: error }, "Failed to list voice conversations"); - return []; - } - } - - public async delete(logger: LoggerLike, voiceConversationId: string): Promise { - const log = getLogger(logger); - const filepath = join(this.baseDir, `${voiceConversationId}.json`); - await unlink(filepath); - log.debug({ voiceConversationId }, "Deleted voice conversation"); - } -} - diff --git a/packages/server/src/server/voice-conversations.e2e.test.ts b/packages/server/src/server/voice-conversations.e2e.test.ts deleted file mode 100644 index 8557315b7..000000000 --- a/packages/server/src/server/voice-conversations.e2e.test.ts +++ /dev/null @@ -1,147 +0,0 @@ -import { describe, test, expect } from "vitest"; -import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; -import { v4 as uuidv4 } from "uuid"; - -import { createTestPaseoDaemon } from "./test-utils/paseo-daemon.js"; -import { DaemonClient } from "./test-utils/daemon-client.js"; - -async function waitForFile(filepath: string, timeoutMs = 5000): Promise { - const start = Date.now(); - // eslint-disable-next-line no-constant-condition - while (true) { - if (existsSync(filepath)) { - return; - } - if (Date.now() - start > timeoutMs) { - throw new Error(`Timed out waiting for file: ${filepath}`); - } - await new Promise((r) => setTimeout(r, 50)); - } -} - -async function waitForJsonFile( - filepath: string, - timeoutMs = 5000 -): Promise { - const start = Date.now(); - // eslint-disable-next-line no-constant-condition - while (true) { - if (existsSync(filepath)) { - try { - const raw = readFileSync(filepath, "utf8"); - if (raw.trim().length > 0) { - return JSON.parse(raw) as T; - } - } catch { - // File may exist but still be mid-write; retry. - } - } - if (Date.now() - start > timeoutMs) { - throw new Error(`Timed out waiting for valid JSON: ${filepath}`); - } - await new Promise((r) => setTimeout(r, 50)); - } -} - -describe("voice conversations - daemon E2E", () => { - test( - "two concurrent clients persist independently under paseoHome/voice-conversations", - async () => { - const daemon = await createTestPaseoDaemon(); - const url = `ws://127.0.0.1:${daemon.port}/ws`; - - const clientA = new DaemonClient({ url }); - const clientB = new DaemonClient({ url }); - await clientA.connect(); - await clientB.connect(); - - try { - const voiceConversationIdA = uuidv4(); - const voiceConversationIdB = uuidv4(); - - await clientA.setVoiceConversation(true, voiceConversationIdA); - await clientB.setVoiceConversation(true, voiceConversationIdB); - - // Minimal traffic to cause a persist without requiring external APIs. - await clientA.setVoiceConversation(false); - await clientB.setVoiceConversation(false); - - const fileA = join( - daemon.paseoHome, - "voice-conversations", - `${voiceConversationIdA}.json` - ); - const fileB = join( - daemon.paseoHome, - "voice-conversations", - `${voiceConversationIdB}.json` - ); - - await waitForFile(fileA); - await waitForFile(fileB); - - const dataA = await waitForJsonFile<{ - voiceConversationId: string; - messageCount: number; - messages: unknown[]; - }>(fileA); - const dataB = await waitForJsonFile<{ - voiceConversationId: string; - messageCount: number; - messages: unknown[]; - }>(fileB); - - expect(dataA.voiceConversationId).toBe(voiceConversationIdA); - expect(dataB.voiceConversationId).toBe(voiceConversationIdB); - expect(dataA.messageCount).toBe(0); - expect(dataB.messageCount).toBe(0); - expect(Array.isArray(dataA.messages)).toBe(true); - expect(Array.isArray(dataB.messages)).toBe(true); - } finally { - await clientA.close().catch(() => undefined); - await clientB.close().catch(() => undefined); - await daemon.close(); - } - }, - 30000 - ); - - test( - "WS attach ignores URL conversationId param for voice conversation state", - async () => { - const daemon = await createTestPaseoDaemon(); - const urlConversationId = `url-${uuidv4()}`; - const url = `ws://127.0.0.1:${daemon.port}/ws?conversationId=${encodeURIComponent( - urlConversationId - )}`; - - const client = new DaemonClient({ url }); - await client.connect(); - - try { - const voiceConversationId = `client-${uuidv4()}`; - await client.setVoiceConversation(true, voiceConversationId); - await client.setVoiceConversation(false); - - const file = join( - daemon.paseoHome, - "voice-conversations", - `${voiceConversationId}.json` - ); - const urlFile = join( - daemon.paseoHome, - "voice-conversations", - `${urlConversationId}.json` - ); - - await waitForFile(file); - expect(existsSync(urlFile)).toBe(false); - } finally { - await client.close().catch(() => undefined); - await daemon.close(); - } - }, - 30000 - ); -}); diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 6ac67ff41..c38cf8f2e 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -19,7 +19,6 @@ import { Session } from "./session.js"; import type { AgentProvider } from "./agent/agent-sdk-types.js"; import { PushTokenStore } from "./push/token-store.js"; import { PushService } from "./push/push-service.js"; -import { VoiceConversationStore } from "./voice-conversation-store.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; export type AgentMcpTransportFactory = () => Promise; @@ -70,7 +69,6 @@ export class VoiceAssistantWebSocketServer { private readonly stt: SpeechToTextProvider | null; private readonly tts: TextToSpeechProvider | null; private readonly terminalManager: TerminalManager | null; - private readonly voiceConversationStore: VoiceConversationStore; private readonly dictation: { finalTimeoutMs?: number; stt?: SpeechToTextProvider | null; @@ -134,9 +132,6 @@ export class VoiceAssistantWebSocketServer { this.stt = speech?.stt ?? null; this.tts = speech?.tts ?? null; this.terminalManager = terminalManager ?? null; - this.voiceConversationStore = new VoiceConversationStore( - join(paseoHome, "voice-conversations") - ); this.voice = voice ?? null; this.dictation = dictation ?? null; @@ -247,7 +242,6 @@ export class VoiceAssistantWebSocketServer { this.stt, this.tts, this.terminalManager, - this.voiceConversationStore, this.voice ?? undefined, { registerVoiceSpeakHandler: (agentId, handler) => { From a5031e02e78b30873b1fbf978e2d57c02f233390 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 16:36:35 +0700 Subject: [PATCH 07/21] Prefer OpenAI speech in daemon-client e2e when available --- .../src/server/daemon-client.e2e.test.ts | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index d1119f402..06b0f7466 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -92,23 +92,31 @@ describe("daemon client E2E", () => { let ctx: DaemonTestContext; beforeAll(async () => { + const speechConfig = openaiApiKey + ? { + dictationSttProvider: "openai" as const, + voiceSttProvider: "openai" as const, + voiceTtsProvider: "openai" as const, + } + : { + dictationSttProvider: "local" as const, + voiceSttProvider: "local" as const, + voiceTtsProvider: "local" as const, + sherpaOnnx: { + modelsDir: sherpaModelsDir, + stt: { + preset: process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20", + }, + tts: { + preset: process.env.PASEO_SHERPA_TTS_PRESET ?? "kitten-nano-en-v0_1-fp16", + }, + }, + }; + ctx = await createDaemonTestContext({ dictationFinalTimeoutMs: 5000, ...(openaiApiKey ? { openai: { apiKey: openaiApiKey } } : {}), - speech: { - dictationSttProvider: "local", - voiceSttProvider: "local", - voiceTtsProvider: "local", - sherpaOnnx: { - modelsDir: sherpaModelsDir, - stt: { - preset: process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20", - }, - tts: { - preset: process.env.PASEO_SHERPA_TTS_PRESET ?? "kitten-nano-en-v0_1-fp16", - }, - }, - }, + speech: speechConfig, }); }, 60000); From 9f5b1ec087144da7bde537d152a985354dc5eae5 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 18:03:44 +0700 Subject: [PATCH 08/21] docs: document voice provider configuration and local defaults --- .../website/src/routes/docs/configuration.tsx | 71 ++++++++++++++++++- packages/website/src/routes/docs/index.tsx | 6 +- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/packages/website/src/routes/docs/configuration.tsx b/packages/website/src/routes/docs/configuration.tsx index fe7522fc9..96b671c60 100644 --- a/packages/website/src/routes/docs/configuration.tsx +++ b/packages/website/src/routes/docs/configuration.tsx @@ -79,6 +79,73 @@ function Configuration() { +
+

Voice

+

+ Voice is provider-based per feature: dictation STT, realtime voice STT, and voice TTS can each + use local or openai. + Defaults are local for all three. +

+ 
+{`{
+  "version": 1,
+  "features": {
+    "dictation": { "stt": { "provider": "local" } },
+    "voiceMode": {
+      "stt": { "provider": "local" },
+      "tts": { "provider": "local" }
+    }
+  },
+  "providers": {
+    "sherpaOnnx": {
+      "modelsDir": "~/.paseo/models/sherpa-onnx",
+      "autoDownload": true,
+      "stt": { "preset": "parakeet-tdt-0.6b-v3-int8" },
+      "tts": { "preset": "pocket-tts-onnx-int8" }
+    }
+  }
+}`}
+
+

+ Local voice uses ONNX models (sherpa-onnx + PocketTTS). Default presets are + parakeet-tdt-0.6b-v3-int8 for STT and + pocket-tts-onnx-int8 for TTS. +

+

+ With local provider enabled and auto-download on (default outside tests), missing model files are + downloaded at daemon startup into $PASEO_HOME/models/sherpa-onnx. + Downloads are only performed for missing files. +

+

+ Local ONNX execution is CPU-first by default. It works without GPU requirements, but performance + depends on your machine. +

+

+ To force OpenAI for voice/dictation, configure OpenAI providers explicitly and provide + OPENAI_API_KEY. If OpenAI is selected but credentials are + missing, daemon startup fails fast. +

+ 
+{`{
+  "version": 1,
+  "features": {
+    "dictation": { "stt": { "provider": "openai" } },
+    "voiceMode": {
+      "stt": { "provider": "openai" },
+      "tts": { "provider": "openai" }
+    }
+  },
+  "providers": {
+    "openai": { "apiKey": "..." }
+  }
+}`}
+
+

+ Realtime voice can create/manage coding agents. Use clear prompts and explicit paths, because + those agents can be launched in arbitrary working directories by request. +

+
+

Common env vars

    @@ -86,6 +153,9 @@ function Configuration() {
  • PASEO_LISTEN — override daemon.listen
  • PASEO_ALLOWED_HOSTS — override/extend daemon.allowedHosts
  • OPENAI_API_KEY and OPENROUTER_API_KEY — override provider keys
    • +
  • PASEO_DICTATION_STT_PROVIDER, PASEO_VOICE_STT_PROVIDER, PASEO_VOICE_TTS_PROVIDER — override voice provider selection (local or openai)
    • +
  • PASEO_SHERPA_ONNX_MODELS_DIR and PASEO_SHERPA_ONNX_AUTO_DOWNLOAD — control local model directory and download behavior
    • +
  • PASEO_SHERPA_STT_PRESET and PASEO_SHERPA_TTS_PRESET — override local STT/TTS model presets
@@ -101,4 +171,3 @@ function Configuration() { ) } - diff --git a/packages/website/src/routes/docs/index.tsx b/packages/website/src/routes/docs/index.tsx index 0ba254e3c..afd9cbbd9 100644 --- a/packages/website/src/routes/docs/index.tsx +++ b/packages/website/src/routes/docs/index.tsx @@ -87,14 +87,16 @@ function GettingStarted() {

Voice Setup

- Voice features currently require an OpenAI API key. Set it as an environment variable before running the server: + Voice supports two providers: local (default) and OpenAI. Local uses ONNX models and will + download missing model files automatically at daemon startup.

$ export OPENAI_API_KEY=your-key-here

- Local voice support is coming soon. + Set OPENAI_API_KEY if you want OpenAI speech providers. + For local defaults and model presets, see Configuration.

From e782fdd35d8b39756960f4b97d78f9d93b1dde64 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 19:05:45 +0700 Subject: [PATCH 09/21] voice: switch internal MCP path to stdio unix-socket bridge --- packages/cli/src/cli.ts | 9 + packages/cli/src/commands/voice-mcp-bridge.ts | 17 + packages/server/src/server/bootstrap.ts | 75 ++++- packages/server/src/server/exports.ts | 1 + packages/server/src/server/index.ts | 7 + packages/server/src/server/session.ts | 46 ++- .../server/session.voice-mcp-config.test.ts | 29 ++ .../server/src/server/voice-mcp-bridge.ts | 305 ++++++++++++++++++ .../server/src/server/websocket-server.ts | 9 +- 9 files changed, 473 insertions(+), 25 deletions(-) create mode 100644 packages/cli/src/commands/voice-mcp-bridge.ts create mode 100644 packages/server/src/server/session.voice-mcp-config.test.ts create mode 100644 packages/server/src/server/voice-mcp-bridge.ts diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index b3222c7c4..ee98ae87d 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -14,6 +14,7 @@ import { runInspectCommand } from './commands/agent/inspect.js' import { runWaitCommand } from './commands/agent/wait.js' import { runAttachCommand } from './commands/agent/attach.js' import { withOutput } from './output/index.js' +import { runVoiceMcpBridgeCommand } from './commands/voice-mcp-bridge.js' const VERSION = '0.1.0' @@ -141,5 +142,13 @@ export function createCli(): Command { // Worktree commands program.addCommand(createWorktreeCommand()) + // Internal voice MCP stdio bridge command (hidden). + program + .command('__paseo_voice_mcp_bridge') + .description('Internal voice MCP bridge command') + .argument('') + .requiredOption('--socket ') + .action(runVoiceMcpBridgeCommand) + return program } diff --git a/packages/cli/src/commands/voice-mcp-bridge.ts b/packages/cli/src/commands/voice-mcp-bridge.ts new file mode 100644 index 000000000..ddbb68bb8 --- /dev/null +++ b/packages/cli/src/commands/voice-mcp-bridge.ts @@ -0,0 +1,17 @@ +import { runVoiceMcpBridgeCli } from "@getpaseo/server"; + +type VoiceBridgeOptions = { + socket: string; +}; + +export async function runVoiceMcpBridgeCommand( + callerAgentId: string, + options: VoiceBridgeOptions +): Promise { + await runVoiceMcpBridgeCli([ + "--socket", + options.socket, + "--caller-agent-id", + callerAgentId, + ]); +} diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 081e5d8f7..d88b9c3b1 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -3,6 +3,7 @@ import { createServer as createHTTPServer } from "http"; import { createReadStream, unlinkSync, existsSync } from "fs"; import { stat } from "fs/promises"; import { randomUUID } from "node:crypto"; +import path from "node:path"; import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js"; @@ -75,11 +76,35 @@ import type { } from "./agent/agent-sdk-types.js"; import { acquirePidLock, releasePidLock } from "./pid-lock.js"; import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js"; +import { createVoiceMcpBridgeSocketServer, type VoiceMcpBridgeSocketServer } from "./voice-mcp-bridge.js"; type AgentMcpTransportMap = Map; type VoiceAgentProvider = "claude" | "codex" | "opencode"; const VOICE_AGENT_FALLBACK_ORDER: VoiceAgentProvider[] = ["claude", "codex", "opencode"]; +function resolveVoiceMcpBridgeCommand(logger: Logger): { command: string; baseArgs: string[] } { + const explicit = process.env.PASEO_BIN_PATH?.trim(); + if (explicit) { + return { command: explicit, baseArgs: ["__paseo_voice_mcp_bridge"] }; + } + + const argv1 = process.argv[1]?.trim(); + if (!argv1) { + logger.warn("Could not resolve argv[1] for voice MCP bridge; falling back to 'paseo'"); + return { command: "paseo", baseArgs: ["__paseo_voice_mcp_bridge"] }; + } + + const base = path.basename(argv1).toLowerCase(); + if (base.includes("tsx") && process.argv[2]) { + return { + command: process.execPath, + baseArgs: [argv1, process.argv[2], "__paseo_voice_mcp_bridge"], + }; + } + + return { command: argv1, baseArgs: ["__paseo_voice_mcp_bridge"] }; +} + export type PaseoOpenAIConfig = { apiKey?: string; stt?: Partial & { apiKey?: string }; @@ -350,14 +375,8 @@ export async function createPaseoDaemon( }, "Voice LLM provider reconciliation completed" ); - if (listenTarget.type !== "tcp" && requestedVoiceLlmProvider !== "openrouter") { - logger.error( - { listen: config.listen, requestedVoiceLlmProvider }, - "Local voice agent mode requires TCP listen target for HTTP MCP bridge" - ); - throw new Error("Local voice agent mode requires TCP listen target"); - } let wsServer: VoiceAssistantWebSocketServer | null = null; + let voiceMcpBridgeServer: VoiceMcpBridgeSocketServer | null = null; // Create in-memory transport for Session's Agent MCP client (voice assistant tools) const createInMemoryAgentMcpTransport = async (): Promise => { @@ -497,6 +516,25 @@ export async function createPaseoDaemon( logger.info("Agent MCP HTTP endpoint disabled"); } + const voiceMcpSocketPath = path.join(config.paseoHome, "runtime", "voice-mcp.sock"); + const voiceMcpBridgeCommand = resolveVoiceMcpBridgeCommand(logger); + voiceMcpBridgeServer = createVoiceMcpBridgeSocketServer({ + socketPath: voiceMcpSocketPath, + logger, + createAgentMcpServerForCaller: async (callerAgentId) => { + return createAgentMcpServer({ + agentManager, + agentStorage, + paseoHome: config.paseoHome, + callerAgentId, + enableVoiceTools: false, + resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null, + resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null, + logger, + }); + }, + }); + let sttService: SpeechToTextProvider | null = null; let ttsService: TextToSpeechProvider | null = null; @@ -875,11 +913,6 @@ export async function createPaseoDaemon( ); } - const voiceAgentMcpUrl = - listenTarget.type === "tcp" - ? `http://127.0.0.1:${listenTarget.port}/mcp/agents` - : null; - wsServer = new VoiceAssistantWebSocketServer( httpServer, logger, @@ -899,7 +932,17 @@ export async function createPaseoDaemon( voiceLlmDefaultProvider, voiceLlmModel: config.voiceLlmModel ?? null, voiceLlmAvailability, - voiceAgentMcpUrl, + voiceAgentMcpStdio: { + command: voiceMcpBridgeCommand.command, + baseArgs: [ + ...voiceMcpBridgeCommand.baseArgs, + "--socket", + voiceMcpSocketPath, + ], + env: { + PASEO_HOME: config.paseoHome, + }, + }, }, { finalTimeoutMs: config.dictationFinalTimeoutMs, @@ -994,6 +1037,9 @@ export async function createPaseoDaemon( httpServer.listen(listenTarget.path); } }); + if (voiceMcpBridgeServer) { + await voiceMcpBridgeServer.start(); + } }; const stop = async () => { @@ -1012,6 +1058,9 @@ export async function createPaseoDaemon( if (wsServer) { await wsServer.close(); } + if (voiceMcpBridgeServer) { + await voiceMcpBridgeServer.stop().catch(() => undefined); + } await new Promise((resolve) => { httpServer.close(() => resolve()); }); diff --git a/packages/server/src/server/exports.ts b/packages/server/src/server/exports.ts index 31262a6a0..c0779c865 100644 --- a/packages/server/src/server/exports.ts +++ b/packages/server/src/server/exports.ts @@ -5,6 +5,7 @@ export { resolvePaseoHome } from "./paseo-home.js"; export { createRootLogger, type LogLevel, type LogFormat } from "./logger.js"; export { loadPersistedConfig, type PersistedConfig } from "./persisted-config.js"; export { DaemonClient, type DaemonClientConfig, type ConnectionState, type DaemonEvent } from "../client/daemon-client.js"; +export { runVoiceMcpBridgeCli } from "./voice-mcp-bridge.js"; // Agent SDK types for CLI commands export type { diff --git a/packages/server/src/server/index.ts b/packages/server/src/server/index.ts index 7737855f8..60c91a602 100644 --- a/packages/server/src/server/index.ts +++ b/packages/server/src/server/index.ts @@ -10,8 +10,15 @@ import { resolvePaseoHome } from "./paseo-home.js"; import { createRootLogger } from "./logger.js"; import { loadPersistedConfig } from "./persisted-config.js"; import { PidLockError } from "./pid-lock.js"; +import { runVoiceMcpBridgeCli } from "./voice-mcp-bridge.js"; async function main() { + const bridgeArgIndex = process.argv.findIndex((arg) => arg === "__paseo_voice_mcp_bridge"); + if (bridgeArgIndex >= 0) { + await runVoiceMcpBridgeCli(process.argv.slice(bridgeArgIndex + 1)); + return; + } + let paseoHome: string; let logger: ReturnType; let config: ReturnType; diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index c145a7e14..d03196d6e 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -57,6 +57,11 @@ type VoiceCallerContext = { allowCustomCwd?: boolean; enableVoiceTools?: boolean; }; +type VoiceMcpStdioConfig = { + command: string; + baseArgs: string[]; + env?: Record; +}; import { buildProviderRegistry } from "./agent/provider-registry.js"; import { AgentManager } from "./agent/agent-manager.js"; import type { ManagedAgent } from "./agent/agent-manager.js"; @@ -158,6 +163,25 @@ const VOICE_AGENT_SYSTEM_INSTRUCTION = [ "Only use the paseo MCP tools.", ].join(" "); +export function buildVoiceAgentMcpServerConfig(params: { + callerAgentId: string; + command: string; + baseArgs: string[]; + env?: Record; +}): { + type: "stdio"; + command: string; + args: string[]; + env?: Record; +} { + return { + type: "stdio", + command: params.command, + args: [...params.baseArgs, "--caller-agent-id", params.callerAgentId], + ...(params.env ? { env: params.env } : {}), + }; +} + type ProcessingPhase = "idle" | "transcribing" | "llm"; type NormalizedGitOptions = { @@ -350,7 +374,7 @@ export class Session { private readonly voiceLlmDefaultProvider: VoiceAgentProvider | null; private readonly voiceLlmModel: string | null; private readonly voiceLlmAvailability: Record | null; - private readonly voiceAgentMcpUrl: string | null; + private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null; private readonly registerVoiceSpeakHandler?: ( agentId: string, handler: VoiceSpeakHandler @@ -383,7 +407,7 @@ export class Session { voiceLlmDefaultProvider?: VoiceAgentProvider | null; voiceLlmModel?: string | null; voiceLlmAvailability?: Record | null; - voiceAgentMcpUrl?: string | null; + voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; }, voiceBridge?: { registerVoiceSpeakHandler?: (agentId: string, handler: VoiceSpeakHandler) => void; @@ -412,7 +436,7 @@ export class Session { this.voiceLlmDefaultProvider = voice?.voiceLlmDefaultProvider ?? null; this.voiceLlmModel = voice?.voiceLlmModel ?? null; this.voiceLlmAvailability = voice?.voiceLlmAvailability ?? null; - this.voiceAgentMcpUrl = voice?.voiceAgentMcpUrl ?? null; + this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null; this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler; this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler; this.registerVoiceCallerContext = voiceBridge?.registerVoiceCallerContext; @@ -4687,9 +4711,9 @@ export class Session { const cwd = join(this.paseoHome, "voice-agent-workspace"); await mkdir(cwd, { recursive: true }); - const mcpUrl = this.voiceAgentMcpUrl; - if (!mcpUrl) { - throw new Error("Voice MCP URL is not configured"); + const mcpStdio = this.voiceAgentMcpStdio; + if (!mcpStdio) { + throw new Error("Voice MCP stdio bridge is not configured"); } const model = this.getVoiceAgentModel(provider); @@ -4699,10 +4723,12 @@ export class Session { modeId: VOICE_AGENT_DEFAULT_MODE[provider], ...(model ? { model } : {}), mcpServers: { - paseo: { - type: "http", - url: `${mcpUrl}?callerAgentId=${encodeURIComponent(voiceAgentId)}`, - }, + paseo: buildVoiceAgentMcpServerConfig({ + callerAgentId: voiceAgentId, + command: mcpStdio.command, + baseArgs: mcpStdio.baseArgs, + env: mcpStdio.env, + }), }, }; diff --git a/packages/server/src/server/session.voice-mcp-config.test.ts b/packages/server/src/server/session.voice-mcp-config.test.ts new file mode 100644 index 000000000..14f4c84e5 --- /dev/null +++ b/packages/server/src/server/session.voice-mcp-config.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, test } from "vitest"; + +import { buildVoiceAgentMcpServerConfig } from "./session.js"; + +describe("voice MCP stdio config", () => { + test("builds stdio MCP config for voice agent", () => { + const config = buildVoiceAgentMcpServerConfig({ + callerAgentId: "voice-agent-123", + command: "/usr/local/bin/paseo", + baseArgs: ["__paseo_voice_mcp_bridge", "--socket", "/tmp/paseo-voice.sock"], + env: { + PASEO_HOME: "/tmp/paseo-home", + }, + }); + + expect(config.type).toBe("stdio"); + expect(config.command).toBe("/usr/local/bin/paseo"); + expect(config.args).toEqual([ + "__paseo_voice_mcp_bridge", + "--socket", + "/tmp/paseo-voice.sock", + "--caller-agent-id", + "voice-agent-123", + ]); + expect(config.env).toEqual({ + PASEO_HOME: "/tmp/paseo-home", + }); + }); +}); diff --git a/packages/server/src/server/voice-mcp-bridge.ts b/packages/server/src/server/voice-mcp-bridge.ts new file mode 100644 index 000000000..e2f32b028 --- /dev/null +++ b/packages/server/src/server/voice-mcp-bridge.ts @@ -0,0 +1,305 @@ +import net from "node:net"; +import path from "node:path"; +import { mkdir, rm } from "node:fs/promises"; +import type { Logger } from "pino"; +import pino from "pino"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import type { JSONRPCMessage } from "@modelcontextprotocol/sdk/types.js"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; + + +type BridgeEnvelope = + | { type: "init"; callerAgentId: string } + | { type: "mcp"; message: JSONRPCMessage } + | { type: "ready" } + | { type: "error"; message: string }; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function parseEnvelope(raw: string): BridgeEnvelope { + const parsed = JSON.parse(raw); + if (!isRecord(parsed) || typeof parsed.type !== "string") { + throw new Error("Invalid bridge envelope"); + } + if (parsed.type === "init") { + const callerAgentId = typeof parsed.callerAgentId === "string" ? parsed.callerAgentId.trim() : ""; + if (!callerAgentId) { + throw new Error("Invalid init payload: callerAgentId is required"); + } + return { type: "init", callerAgentId }; + } + if (parsed.type === "mcp") { + if (!("message" in parsed)) { + throw new Error("Invalid mcp payload: message is required"); + } + return { type: "mcp", message: parsed.message as JSONRPCMessage }; + } + if (parsed.type === "ready") return { type: "ready" }; + if (parsed.type === "error") { + return { type: "error", message: typeof parsed.message === "string" ? parsed.message : "Unknown error" }; + } + throw new Error(`Unknown envelope type: ${parsed.type}`); +} + +function encodeEnvelope(envelope: BridgeEnvelope): string { + return `${JSON.stringify(envelope)}\n`; +} + +export type VoiceMcpBridgeSocketServer = { + socketPath: string; + start: () => Promise; + stop: () => Promise; +}; + +export function createVoiceMcpBridgeSocketServer(params: { + socketPath: string; + logger: Logger; + createAgentMcpServerForCaller: (callerAgentId: string) => Promise<{ connect: (transport: InMemoryTransport) => Promise; close?: () => Promise }>; +}): VoiceMcpBridgeSocketServer { + const logger = params.logger.child({ module: "voice-mcp-bridge" }); + const sockets = new Set(); + + const server = net.createServer((socket) => { + sockets.add(socket); + const connectionLogger = logger.child({ component: "connection" }); + let readBuffer = ""; + let initialized = false; + let clientTransport: InMemoryTransport | null = null; + let mcpServer: { close?: () => Promise } | null = null; + + const send = (payload: BridgeEnvelope) => { + socket.write(encodeEnvelope(payload)); + }; + + const fail = (message: string) => { + send({ type: "error", message }); + socket.end(); + }; + + const cleanup = async () => { + sockets.delete(socket); + const closeTasks: Promise[] = []; + if (clientTransport) { + closeTasks.push(clientTransport.close().catch(() => undefined)); + } + if (mcpServer?.close) { + closeTasks.push(mcpServer.close().catch(() => undefined)); + } + await Promise.all(closeTasks); + }; + + socket.on("data", (chunk) => { + readBuffer += chunk.toString("utf8"); + while (true) { + const newlineIndex = readBuffer.indexOf("\n"); + if (newlineIndex < 0) break; + const line = readBuffer.slice(0, newlineIndex).trim(); + readBuffer = readBuffer.slice(newlineIndex + 1); + if (!line) continue; + + let message: BridgeEnvelope; + try { + message = parseEnvelope(line); + } catch (error) { + fail(error instanceof Error ? error.message : String(error)); + return; + } + + if (message.type === "init") { + if (initialized) { + fail("Bridge already initialized"); + return; + } + initialized = true; + void (async () => { + try { + const [proxyClient, proxyServer] = InMemoryTransport.createLinkedPair(); + const serverInstance = await params.createAgentMcpServerForCaller(message.callerAgentId); + await serverInstance.connect(proxyServer); + await proxyClient.start(); + proxyClient.onmessage = (jsonrpcMessage) => { + send({ type: "mcp", message: jsonrpcMessage }); + }; + clientTransport = proxyClient; + mcpServer = serverInstance; + send({ type: "ready" }); + } catch (error) { + connectionLogger.error({ err: error }, "Failed to initialize voice MCP bridge connection"); + fail(error instanceof Error ? error.message : String(error)); + } + })(); + continue; + } + + if (message.type === "mcp") { + if (!clientTransport) { + fail("Bridge is not initialized"); + return; + } + void clientTransport.send(message.message).catch((error) => { + connectionLogger.error({ err: error }, "Failed to forward MCP message"); + fail(error instanceof Error ? error.message : String(error)); + }); + continue; + } + } + }); + + socket.on("error", (error) => { + connectionLogger.error({ err: error }, "Voice MCP bridge socket error"); + }); + socket.on("close", () => { + void cleanup(); + }); + }); + + return { + socketPath: params.socketPath, + async start() { + await mkdir(path.dirname(params.socketPath), { recursive: true }); + await rm(params.socketPath, { force: true }).catch(() => undefined); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(params.socketPath, () => { + server.off("error", reject); + resolve(); + }); + }); + logger.info({ socketPath: params.socketPath }, "Voice MCP bridge socket server listening"); + }, + async stop() { + for (const socket of sockets) { + socket.destroy(); + } + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) reject(error); + else resolve(); + }); + }); + await rm(params.socketPath, { force: true }).catch(() => undefined); + }, + }; +} + +function parseBridgeCliArgs(argv: string[]): { socketPath: string; callerAgentId: string } { + let socketPath: string | null = null; + let callerAgentId: string | null = null; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--socket") { + socketPath = argv[index + 1] ?? null; + index += 1; + continue; + } + if (arg === "--caller-agent-id") { + callerAgentId = argv[index + 1] ?? null; + index += 1; + continue; + } + } + + if (!socketPath?.trim()) { + throw new Error("Missing required --socket "); + } + if (!callerAgentId?.trim()) { + throw new Error("Missing required --caller-agent-id "); + } + + return { + socketPath: socketPath.trim(), + callerAgentId: callerAgentId.trim(), + }; +} + +export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Promise { + const bridgeLogger = logger ?? pino({ level: "error" }); + const parsed = parseBridgeCliArgs(argv); + + const socket = net.createConnection(parsed.socketPath); + const stdioTransport = new StdioServerTransport(process.stdin, process.stdout); + let ready = false; + let socketBuffer = ""; + + const sendEnvelope = (payload: BridgeEnvelope) => { + socket.write(encodeEnvelope(payload)); + }; + + const closeWithError = (message: string): never => { + throw new Error(message); + }; + + socket.on("data", (chunk) => { + socketBuffer += chunk.toString("utf8"); + while (true) { + const newlineIndex = socketBuffer.indexOf("\n"); + if (newlineIndex < 0) break; + const line = socketBuffer.slice(0, newlineIndex).trim(); + socketBuffer = socketBuffer.slice(newlineIndex + 1); + if (!line) continue; + + const envelope = parseEnvelope(line); + if (envelope.type === "ready") { + ready = true; + continue; + } + if (envelope.type === "error") { + socket.destroy(); + closeWithError(`Voice MCP bridge error: ${envelope.message}`); + } + if (envelope.type === "mcp") { + void stdioTransport.send(envelope.message); + } + } + }); + + socket.on("error", (error) => { + bridgeLogger.error({ err: error }, "Voice MCP bridge socket client error"); + }); + + await new Promise((resolve, reject) => { + socket.once("connect", () => resolve()); + socket.once("error", reject); + }); + + sendEnvelope({ type: "init", callerAgentId: parsed.callerAgentId }); + + // Wait for bridge readiness before forwarding stdio messages. + await new Promise((resolve, reject) => { + const deadline = setTimeout(() => { + reject(new Error("Timed out waiting for voice MCP bridge initialization")); + }, 10000); + const checkReady = () => { + if (ready) { + clearTimeout(deadline); + resolve(); + } else { + setTimeout(checkReady, 25); + } + }; + checkReady(); + }); + + stdioTransport.onmessage = (message) => { + sendEnvelope({ type: "mcp", message }); + }; + stdioTransport.onerror = (error) => { + bridgeLogger.error({ err: error }, "Voice MCP stdio transport error"); + socket.destroy(); + }; + stdioTransport.onclose = () => { + socket.end(); + }; + + await stdioTransport.start(); + + await new Promise((resolve) => { + socket.once("close", () => resolve()); + process.stdin.once("end", () => resolve()); + }); + + await stdioTransport.close().catch(() => undefined); +} diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index c38cf8f2e..11c500b5f 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -23,6 +23,11 @@ import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech export type AgentMcpTransportFactory = () => Promise; type VoiceAgentProvider = "claude" | "codex" | "opencode"; +type VoiceMcpStdioConfig = { + command: string; + baseArgs: string[]; + env?: Record; +}; type WebSocketServerConfig = { allowedOrigins: Set; @@ -80,7 +85,7 @@ export class VoiceAssistantWebSocketServer { voiceLlmDefaultProvider?: VoiceAgentProvider | null; voiceLlmModel?: string | null; voiceLlmAvailability?: Record | null; - voiceAgentMcpUrl?: string | null; + voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; } | null; private readonly voiceSpeakHandlers = new Map< string, @@ -115,7 +120,7 @@ export class VoiceAssistantWebSocketServer { voiceLlmDefaultProvider?: VoiceAgentProvider | null; voiceLlmModel?: string | null; voiceLlmAvailability?: Record | null; - voiceAgentMcpUrl?: string | null; + voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; }, dictation?: { finalTimeoutMs?: number; From 0726f4144f70c4503f17d445b11a40d0bc30136e Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 19:24:49 +0700 Subject: [PATCH 10/21] test: add stdio voice MCP unix-socket bridge integration test --- .../src/server/voice-mcp-bridge.test.ts | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 packages/server/src/server/voice-mcp-bridge.test.ts diff --git a/packages/server/src/server/voice-mcp-bridge.test.ts b/packages/server/src/server/voice-mcp-bridge.test.ts new file mode 100644 index 000000000..942d55925 --- /dev/null +++ b/packages/server/src/server/voice-mcp-bridge.test.ts @@ -0,0 +1,93 @@ +import os from "node:os"; +import path from "node:path"; +import { mkdtemp, rm } from "node:fs/promises"; +import { describe, expect, test } from "vitest"; +import { experimental_createMCPClient } from "ai"; +import { z } from "zod"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import pino from "pino"; + +import { createVoiceMcpBridgeSocketServer } from "./voice-mcp-bridge.js"; + +describe("voice MCP bridge", () => { + test("proxies stdio MCP messages through unix socket bridge", async () => { + const tmpRoot = await mkdtemp(path.join(os.tmpdir(), "paseo-voice-mcp-bridge-")); + const socketPath = path.join(tmpRoot, "voice-mcp.sock"); + const callerAgentId = "voice-agent-bridge-test"; + + const bridge = createVoiceMcpBridgeSocketServer({ + socketPath, + logger: pino({ level: "silent" }), + createAgentMcpServerForCaller: async (callerId) => { + const server = new McpServer({ + name: "bridge-test-server", + version: "1.0.0", + }); + + server.registerTool( + "echo_caller", + { + value: z.string().optional(), + }, + async (args) => { + return { + content: [ + { + type: "text", + text: JSON.stringify({ + callerAgentId: callerId, + value: args.value ?? null, + }), + }, + ], + structuredContent: { + callerAgentId: callerId, + value: args.value ?? null, + }, + }; + } + ); + + return server; + }, + }); + + await bridge.start(); + + const tsxBin = path.resolve(process.cwd(), "../../node_modules/.bin/tsx"); + const serverIndex = path.resolve(process.cwd(), "src/server/index.ts"); + + const transport = new StdioClientTransport({ + command: process.execPath, + args: [ + tsxBin, + serverIndex, + "__paseo_voice_mcp_bridge", + "--socket", + socketPath, + "--caller-agent-id", + callerAgentId, + ], + }); + + const client = await experimental_createMCPClient({ transport }); + + try { + const result = await client.callTool({ + name: "echo_caller", + args: { value: "ok" }, + }); + + const payload = + ((result as { structuredContent?: { callerAgentId?: string; value?: string | null } }) + .structuredContent) ?? null; + + expect(payload?.callerAgentId).toBe(callerAgentId); + } finally { + await client.close(); + await bridge.stop(); + await rm(tmpRoot, { recursive: true, force: true }); + } + }, 30_000); +}); From a44038a0481dd98c1b20a9e83931423ecd43ca89 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 20:12:55 +0700 Subject: [PATCH 11/21] Update files --- .../src/server/agent/agent-mcp.e2e.test.ts | 1 - .../src/server/agent/provider-manifest.ts | 19 + .../server/src/server/bootstrap.smoke.test.ts | 1 - packages/server/src/server/bootstrap.ts | 81 ++- packages/server/src/server/config.ts | 20 +- .../src/server/daemon-client.e2e.test.ts | 24 +- .../server/src/server/persisted-config.ts | 6 +- packages/server/src/server/session.ts | 582 +----------------- .../src/server/test-utils/paseo-daemon.ts | 2 - .../server/src/server/websocket-server.ts | 13 +- .../public/schemas/paseo.config.v1.json | 53 +- .../website/src/routes/docs/configuration.tsx | 14 +- 12 files changed, 175 insertions(+), 641 deletions(-) diff --git a/packages/server/src/server/agent/agent-mcp.e2e.test.ts b/packages/server/src/server/agent/agent-mcp.e2e.test.ts index 527725f9f..d03485365 100644 --- a/packages/server/src/server/agent/agent-mcp.e2e.test.ts +++ b/packages/server/src/server/agent/agent-mcp.e2e.test.ts @@ -90,7 +90,6 @@ describe("agent MCP end-to-end (offline)", () => { mcpDebug: false, agentClients: createTestAgentClients(), agentStoragePath: path.join(paseoHome, "agents"), - openrouterApiKey: null, }; const daemon = await createPaseoDaemon(daemonConfig, pino({ level: "silent" })); diff --git a/packages/server/src/server/agent/provider-manifest.ts b/packages/server/src/server/agent/provider-manifest.ts index 9f20d861f..189ac78f7 100644 --- a/packages/server/src/server/agent/provider-manifest.ts +++ b/packages/server/src/server/agent/provider-manifest.ts @@ -7,6 +7,11 @@ export interface AgentProviderDefinition { description: string; defaultModeId: string | null; modes: AgentMode[]; + voice?: { + enabled: boolean; + defaultModeId: string; + defaultModel?: string; + }; } const CLAUDE_MODES: AgentMode[] = [ @@ -68,6 +73,11 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [ "Anthropic's multi-tool assistant with MCP support, streaming, and deep reasoning", defaultModeId: "default", modes: CLAUDE_MODES, + voice: { + enabled: true, + defaultModeId: "default", + defaultModel: "haiku", + }, }, { id: "codex", @@ -76,6 +86,11 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [ "OpenAI's Codex workspace agent with sandbox controls and optional network access", defaultModeId: "auto", modes: CODEX_MODES, + voice: { + enabled: true, + defaultModeId: "read-only", + defaultModel: "gpt-5.2-mini", + }, }, { id: "opencode", @@ -84,6 +99,10 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [ "Open-source coding assistant with multi-provider model support", defaultModeId: "default", modes: OPENCODE_MODES, + voice: { + enabled: true, + defaultModeId: "default", + }, }, ]; diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 8ee4a3d14..82e82984a 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -60,7 +60,6 @@ describe("paseo daemon bootstrap", () => { voiceSttProvider: "openai", voiceTtsProvider: "openai", }, - openrouterApiKey: null, }; try { diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index d88b9c3b1..4e62e14c3 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -74,13 +74,12 @@ import type { AgentClient, AgentProvider, } from "./agent/agent-sdk-types.js"; +import { AGENT_PROVIDER_DEFINITIONS } from "./agent/provider-manifest.js"; import { acquirePidLock, releasePidLock } from "./pid-lock.js"; import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js"; import { createVoiceMcpBridgeSocketServer, type VoiceMcpBridgeSocketServer } from "./voice-mcp-bridge.js"; type AgentMcpTransportMap = Map; -type VoiceAgentProvider = "claude" | "codex" | "opencode"; -const VOICE_AGENT_FALLBACK_ORDER: VoiceAgentProvider[] = ["claude", "codex", "opencode"]; function resolveVoiceMcpBridgeCommand(logger: Logger): { command: string; baseArgs: string[] } { const explicit = process.env.PASEO_BIN_PATH?.trim(); @@ -147,8 +146,7 @@ export type PaseoDaemonConfig = { appBaseUrl?: string; openai?: PaseoOpenAIConfig; speech?: PaseoSpeechConfig; - openrouterApiKey?: string | null; - voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null; + voiceLlmProvider?: AgentProvider | null; voiceLlmProviderExplicit?: boolean; voiceLlmModel?: string | null; dictationFinalTimeoutMs?: number; @@ -313,21 +311,24 @@ export async function createPaseoDaemon( const requestedVoiceLlmProvider = config.voiceLlmProvider ?? null; const voiceLlmProviderExplicit = config.voiceLlmProviderExplicit ?? false; + const voiceEnabledProviders = AGENT_PROVIDER_DEFINITIONS + .filter((definition) => definition.voice?.enabled) + .map((definition) => definition.id as AgentProvider); logger.info( { requestedVoiceLlmProvider, voiceLlmProviderExplicit, + voiceEnabledProviders, }, "Voice LLM provider reconciliation started" ); const providerClients = createAllClients(logger); - const voiceLlmAvailability: Record = { - claude: false, - codex: false, - opencode: false, - }; - for (const provider of VOICE_AGENT_FALLBACK_ORDER) { + Object.assign(providerClients, config.agentClients); + const voiceLlmAvailability = Object.fromEntries( + voiceEnabledProviders.map((provider) => [provider, false]) + ) as Record; + for (const provider of voiceEnabledProviders) { try { voiceLlmAvailability[provider] = await providerClients[provider].isAvailable(); } catch (error) { @@ -336,21 +337,17 @@ export async function createPaseoDaemon( } } - const voiceLlmDefaultProvider = - VOICE_AGENT_FALLBACK_ORDER.find((provider) => voiceLlmAvailability[provider]) ?? null; - - if (requestedVoiceLlmProvider === "openrouter") { - const openrouterApiKey = - config.openrouterApiKey ?? process.env.OPENROUTER_API_KEY ?? null; - if (!openrouterApiKey) { - logger.error("voiceMode.llm.provider is openrouter but no OpenRouter API key is configured"); - throw new Error("Missing OpenRouter API key for voiceMode.llm.provider=openrouter"); + let resolvedVoiceLlmProvider: AgentProvider | null = null; + if (requestedVoiceLlmProvider) { + if (!voiceEnabledProviders.includes(requestedVoiceLlmProvider)) { + logger.error( + { provider: requestedVoiceLlmProvider, voiceEnabledProviders }, + "Configured voice LLM provider does not support voice mode" + ); + throw new Error( + `Configured voice LLM provider '${requestedVoiceLlmProvider}' does not support voice mode` + ); } - } else if ( - requestedVoiceLlmProvider === "claude" || - requestedVoiceLlmProvider === "codex" || - requestedVoiceLlmProvider === "opencode" - ) { if (!voiceLlmAvailability[requestedVoiceLlmProvider]) { logger.error( { provider: requestedVoiceLlmProvider, voiceLlmAvailability }, @@ -358,20 +355,40 @@ export async function createPaseoDaemon( ); throw new Error(`Configured voice LLM provider '${requestedVoiceLlmProvider}' is unavailable`); } - } else if (!voiceLlmDefaultProvider) { + resolvedVoiceLlmProvider = requestedVoiceLlmProvider; + } else { + resolvedVoiceLlmProvider = + voiceEnabledProviders.find((provider) => voiceLlmAvailability[provider]) ?? null; + } + + if (!resolvedVoiceLlmProvider) { logger.error( { requestedVoiceLlmProvider, voiceLlmAvailability }, - "No local voice LLM provider available for fallback" + "No voice LLM provider available" ); - throw new Error("No local voice LLM provider available (claude/codex/opencode)"); + throw new Error("No voice LLM provider available"); } + const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find( + (definition) => definition.id === resolvedVoiceLlmProvider + ); + if (!resolvedVoiceProviderDefinition?.voice?.enabled) { + throw new Error( + `Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry` + ); + } + const resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId; + const resolvedVoiceLlmModel = + config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null; + logger.info( { requestedVoiceLlmProvider, voiceLlmProviderExplicit, + resolvedVoiceLlmProvider, + resolvedVoiceLlmModeId, + resolvedVoiceLlmModel, voiceLlmAvailability, - voiceLlmDefaultProvider, }, "Voice LLM provider reconciliation completed" ); @@ -926,12 +943,10 @@ export async function createPaseoDaemon( { stt: sttService, tts: ttsService }, terminalManager, { - openrouterApiKey: config.openrouterApiKey ?? null, - voiceLlmProvider: config.voiceLlmProvider ?? null, + voiceLlmProvider: resolvedVoiceLlmProvider, + voiceLlmModeId: resolvedVoiceLlmModeId, voiceLlmProviderExplicit, - voiceLlmDefaultProvider, - voiceLlmModel: config.voiceLlmModel ?? null, - voiceLlmAvailability, + voiceLlmModel: resolvedVoiceLlmModel, voiceAgentMcpStdio: { command: voiceMcpBridgeCommand.command, baseArgs: [ diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index 47c1cd6f8..bbd455bdd 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -4,6 +4,8 @@ import type { PaseoDaemonConfig } from "./bootstrap.js"; import type { STTConfig } from "./speech/providers/openai/stt.js"; import type { TTSConfig } from "./speech/providers/openai/tts.js"; import { loadPersistedConfig } from "./persisted-config.js"; +import type { AgentProvider } from "./agent/agent-sdk-types.js"; +import { AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js"; import { mergeAllowedHosts, parseAllowedHostsEnv, @@ -13,15 +15,6 @@ import { const DEFAULT_PORT = 6767; const DEFAULT_RELAY_ENDPOINT = "relay.paseo.sh:443"; const DEFAULT_APP_BASE_URL = "https://app.paseo.sh"; -const VOICE_LLM_PROVIDER_IDS = [ - "openrouter", - "local-agent", - "claude", - "codex", - "opencode", -] as const; -type VoiceLlmProviderId = (typeof VOICE_LLM_PROVIDER_IDS)[number]; - function getDefaultListen(): string { // Main HTTP server defaults to TCP return `127.0.0.1:${DEFAULT_PORT}`; @@ -97,7 +90,7 @@ function parseSpeechProviderId(value: unknown): "openai" | "local" | null { return null; } -function parseVoiceLlmProviderId(value: unknown): VoiceLlmProviderId | null { +function parseVoiceLlmProviderId(value: unknown): AgentProvider | null { if (typeof value !== "string") { return null; } @@ -105,8 +98,8 @@ function parseVoiceLlmProviderId(value: unknown): VoiceLlmProviderId | null { if (!normalized) { return null; } - return (VOICE_LLM_PROVIDER_IDS as readonly string[]).includes(normalized) - ? (normalized as VoiceLlmProviderId) + return (AGENT_PROVIDER_IDS as readonly string[]).includes(normalized) + ? (normalized as AgentProvider) : null; } @@ -265,8 +258,6 @@ export function loadConfig( } : undefined; - const openrouterApiKey = - env.OPENROUTER_API_KEY ?? persisted.providers?.openrouter?.apiKey ?? null; const envVoiceLlmProvider = parseVoiceLlmProviderId(env.PASEO_VOICE_LLM_PROVIDER); const persistedVoiceLlmProvider = parseVoiceLlmProviderId( persisted.features?.voiceMode?.llm?.provider @@ -299,7 +290,6 @@ export function loadConfig( voiceTtsProvider, ...(sherpaOnnx ? { sherpaOnnx } : {}), }, - openrouterApiKey, voiceLlmProvider, voiceLlmProviderExplicit, voiceLlmModel, diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index 06b0f7466..b395c1191 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -574,16 +574,16 @@ describe("daemon client E2E", () => { 120000 ); - test.runIf(Boolean(process.env.OPENROUTER_API_KEY))( - "streams session activity logs and chunks", + test( + "does not process non-voice LLM turns via OpenRouter", async () => { await ctx.client.setVoiceConversation(false); - let sawAssistantChunk = false; let sawTranscriptLog = false; + let sawAssistantChunk = false; let sawAssistantLog = false; - const completion = waitForSignal(60000, (resolve) => { + const transcriptSeen = waitForSignal(60000, (resolve) => { const unsubscribeChunk = ctx.client.on("assistant_chunk", (message) => { if (message.type !== "assistant_chunk") { return; @@ -591,9 +591,6 @@ describe("daemon client E2E", () => { if (message.payload.chunk.length > 0) { sawAssistantChunk = true; } - if (sawAssistantChunk && sawTranscriptLog && sawAssistantLog) { - resolve(); - } }); const unsubscribeActivity = ctx.client.on("activity_log", (message) => { @@ -602,13 +599,11 @@ describe("daemon client E2E", () => { } if (message.payload.type === "transcript") { sawTranscriptLog = true; + resolve(); } if (message.payload.type === "assistant") { sawAssistantLog = true; } - if (sawAssistantChunk && sawTranscriptLog && sawAssistantLog) { - resolve(); - } }); return () => { @@ -618,9 +613,14 @@ describe("daemon client E2E", () => { }); await ctx.client.sendUserMessage("Say 'hello' and nothing else"); - await completion; + await transcriptSeen; + await new Promise((resolve) => setTimeout(resolve, 1500)); + + expect(sawTranscriptLog).toBe(true); + expect(sawAssistantChunk).toBe(false); + expect(sawAssistantLog).toBe(false); }, - 120000 + 90000 ); speechTest( diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index be1293241..2ee622536 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -1,6 +1,7 @@ import { existsSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; import { z } from "zod"; +import { AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js"; const LogConfigSchema = z .object({ @@ -41,7 +42,6 @@ const SherpaOnnxProviderSchema = z const ProvidersSchema = z .object({ openai: ProviderCredentialsSchema.optional(), - openrouter: ProviderCredentialsSchema.optional(), sherpaOnnx: SherpaOnnxProviderSchema.optional(), }) .strict(); @@ -74,9 +74,7 @@ const FeatureVoiceModeSchema = z .object({ llm: z .object({ - provider: z - .enum(["openrouter", "local-agent", "claude", "codex", "opencode"]) - .optional(), + provider: z.enum(AGENT_PROVIDER_IDS as [string, ...string[]]).optional(), model: z.string().min(1).optional(), }) .strict() diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index d03196d6e..7249e4d9e 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -1,17 +1,10 @@ import { v4 as uuidv4 } from "uuid"; -import { readFile, mkdir, writeFile, stat } from "fs/promises"; +import { mkdir, stat } from "fs/promises"; import { exec } from "child_process"; -import { promisify, inspect } from "util"; +import { promisify } from "util"; import { join, resolve, sep } from "path"; -import invariant from "tiny-invariant"; import { z } from "zod"; -import { streamText, stepCountIs } from "ai"; import type { ToolSet } from "ai"; -import type { ModelMessage } from "@ai-sdk/provider-utils"; -import { - createOpenRouter, - OpenRouterProviderOptions, -} from "@openrouter/ai-sdk-provider"; import { serializeAgentStreamEvent, type AgentSnapshotPayload, @@ -29,8 +22,6 @@ import { } from "./messages.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; import { parseAndHighlightDiff, type ParsedDiffFile } from "./utils/diff-highlighter.js"; -import { getSystemPrompt } from "./agent/system-prompt.js"; -import { getAllTools } from "./agent/llm-openai.js"; import { TTSManager } from "./agent/tts-manager.js"; import { STTManager } from "./agent/stt-manager.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; @@ -48,8 +39,6 @@ import { experimental_createMCPClient } from "ai"; import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js"; export type AgentMcpTransportFactory = () => Promise; -type VoiceLlmProvider = "openrouter" | "local-agent" | "claude" | "codex" | "opencode"; -type VoiceAgentProvider = Exclude; type VoiceSpeakHandler = (params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise; type VoiceCallerContext = { childAgentDefaultLabels?: Record; @@ -144,16 +133,6 @@ const RESTART_EXIT_DELAY_MS = 250; * Uses Claude Haiku for speed and cost efficiency. */ const AUTO_GEN_MODEL = "haiku"; -const VOICE_AGENT_FALLBACK_ORDER: VoiceAgentProvider[] = ["claude", "codex", "opencode"]; -const VOICE_AGENT_DEFAULT_MODE: Record = { - claude: "default", - codex: "read-only", - opencode: "default", -}; -const VOICE_AGENT_DEFAULT_MODEL: Partial> = { - claude: "haiku", - codex: "gpt-5.2-mini", -}; const VOICE_AGENT_SYSTEM_INSTRUCTION = [ "You are the Paseo voice assistant.", "The user cannot see your chat messages or tool calls.", @@ -210,18 +189,6 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round( ); const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/; -/** - * Type for present_artifact tool arguments - */ -interface PresentArtifactArgs { - type: "markdown" | "diff" | "image" | "code"; - source: - | { type: "file"; path: string } - | { type: "command_output"; command: string } - | { type: "text"; text: string }; - title: string; -} - interface AudioBufferState { chunks: Buffer[]; format: string; @@ -335,7 +302,6 @@ export class Session { >(); // Conversation history - private messages: ModelMessage[] = []; private turnIndex = 0; // Per-session managers @@ -368,12 +334,10 @@ export class Session { } | null = null; private readonly terminalManager: TerminalManager | null; private terminalSubscriptions: Map void> = new Map(); - private readonly openrouterApiKey: string | null; - private readonly voiceLlmProvider: VoiceLlmProvider | null; + private readonly voiceLlmProvider: AgentProvider | null; + private readonly voiceLlmModeId: string | null; private readonly voiceLlmProviderExplicit: boolean; - private readonly voiceLlmDefaultProvider: VoiceAgentProvider | null; private readonly voiceLlmModel: string | null; - private readonly voiceLlmAvailability: Record | null; private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null; private readonly registerVoiceSpeakHandler?: ( agentId: string, @@ -401,12 +365,10 @@ export class Session { tts: TextToSpeechProvider | null, terminalManager: TerminalManager | null, voice?: { - openrouterApiKey?: string | null; - voiceLlmProvider?: VoiceLlmProvider | null; + voiceLlmProvider?: AgentProvider | null; + voiceLlmModeId?: string | null; voiceLlmProviderExplicit?: boolean; - voiceLlmDefaultProvider?: VoiceAgentProvider | null; voiceLlmModel?: string | null; - voiceLlmAvailability?: Record | null; voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; }, voiceBridge?: { @@ -430,12 +392,10 @@ export class Session { this.agentStorage = agentStorage; this.createAgentMcpTransport = createAgentMcpTransport; this.terminalManager = terminalManager; - this.openrouterApiKey = voice?.openrouterApiKey ?? null; this.voiceLlmProvider = voice?.voiceLlmProvider ?? null; + this.voiceLlmModeId = voice?.voiceLlmModeId ?? null; this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false; - this.voiceLlmDefaultProvider = voice?.voiceLlmDefaultProvider ?? null; this.voiceLlmModel = voice?.voiceLlmModel ?? null; - this.voiceLlmAvailability = voice?.voiceLlmAvailability ?? null; this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null; this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler; this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler; @@ -467,28 +427,6 @@ export class Session { this.sessionLogger.info("Session created"); } - private escapeXmlText(value: string): string { - return value - .replace(/&/g, "&") - .replace(//g, ">"); - } - - private escapeXmlAttribute(value: string): string { - return this.escapeXmlText(value) - .replace(/"/g, """) - .replace(/'/g, "'"); - } - - private formatVoiceTranscriptionForLLM(text: string): string { - const trimmed = text.trim(); - const focusedAgentId = this.clientActivity?.focusedAgentId ?? null; - const focusedAttr = focusedAgentId - ? ` focused-agent-id="${this.escapeXmlAttribute(focusedAgentId)}"` - : ""; - return `${this.escapeXmlText(trimmed)}`; - } - /** * Get the client's current activity state */ @@ -1195,19 +1133,6 @@ export class Session { voiceConversationId: string, requestId: string ): Promise { - if (this.voiceLlmProvider === "openrouter") { - this.voiceAssistantAgentId = null; - this.messages = []; - this.emit({ - type: "voice_conversation_loaded", - payload: { - voiceConversationId, - messageCount: 0, - requestId, - }, - }); - return; - } this.voiceAssistantAgentId = voiceConversationId; this.emit({ @@ -1459,36 +1384,19 @@ export class Session { } this.isVoiceMode = true; - if (this.voiceLlmProvider !== "openrouter") { - this.voiceAssistantAgentId = voiceConversationId; - this.sessionLogger.info( - { voiceAssistantAgentId: this.voiceAssistantAgentId }, - "Voice conversation enabled (agent-backed)" - ); - return; - } - - // OpenRouter voice mode is always ephemeral. - this.voiceAssistantAgentId = null; - this.messages = []; - + this.voiceAssistantAgentId = voiceConversationId; this.sessionLogger.info( - { messageCount: this.messages.length }, - "Voice conversation enabled" + { voiceAssistantAgentId: this.voiceAssistantAgentId }, + "Voice conversation enabled (agent-backed)" ); return; } this.isVoiceMode = false; - if (this.voiceLlmProvider !== "openrouter") { - this.sessionLogger.info( - { voiceAssistantAgentId: this.voiceAssistantAgentId }, - "Voice conversation disabled (agent-backed)" - ); - return; - } - this.voiceAssistantAgentId = null; - this.sessionLogger.info("Voice conversation disabled"); + this.sessionLogger.info( + { voiceAssistantAgentId: this.voiceAssistantAgentId }, + "Voice conversation disabled (agent-backed)" + ); } /** @@ -4342,10 +4250,7 @@ export class Session { }, }); - // Add to conversation - this.messages.push({ role: "user", content: text }); - - // Process through LLM (TTS enabled in voice mode for voice conversations) + // Process through LLM (voice path is agent-backed only) this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, text); await this.currentStreamPromise; } @@ -4612,14 +4517,6 @@ export class Session { }, }); - // Add to conversation - this.messages.push({ - role: "user", - content: this.isVoiceMode - ? this.formatVoiceTranscriptionForLLM(result.text) - : result.text, - }); - // Set phase to LLM and process (TTS enabled in voice mode for voice conversations) this.clearSpeechInProgress("transcription complete"); this.setPhase("llm"); @@ -4642,52 +4539,6 @@ export class Session { } } - /** - * Resolve the effective voice LLM provider. - * - explicit provider => strict - * - local-agent / unset => fallback order - */ - private resolveVoiceAgentProvider(): VoiceAgentProvider { - const configured = this.voiceLlmProvider; - const availability = this.voiceLlmAvailability ?? { - claude: true, - codex: true, - opencode: true, - }; - - if (configured === "openrouter") { - throw new Error("voiceLlmProvider=openrouter cannot be used in local-agent flow"); - } - - if (configured === "claude" || configured === "codex" || configured === "opencode") { - if (!availability[configured]) { - throw new Error(`Configured voice LLM provider '${configured}' is unavailable`); - } - return configured; - } - - const fallbackOrder = - this.voiceLlmDefaultProvider && availability[this.voiceLlmDefaultProvider] - ? [this.voiceLlmDefaultProvider, ...VOICE_AGENT_FALLBACK_ORDER.filter((id) => id !== this.voiceLlmDefaultProvider)] - : VOICE_AGENT_FALLBACK_ORDER; - - for (const provider of fallbackOrder) { - if (availability[provider]) { - return provider; - } - } - - throw new Error("No local voice LLM provider is available (claude/codex/opencode)"); - } - - private getVoiceAgentModel(provider: VoiceAgentProvider): string | undefined { - const configured = this.voiceLlmModel?.trim(); - if (configured) { - return configured; - } - return VOICE_AGENT_DEFAULT_MODEL[provider]; - } - private async ensureVoiceAssistantAgent(): Promise { if (this.voiceAssistantAgentId) { const existing = this.agentManager.getAgent(this.voiceAssistantAgentId); @@ -4706,7 +4557,10 @@ export class Session { } } - const provider = this.resolveVoiceAgentProvider(); + const provider = this.voiceLlmProvider; + if (!provider) { + throw new Error("Voice LLM provider is not configured"); + } const voiceAgentId = this.voiceAssistantAgentId ?? uuidv4(); const cwd = join(this.paseoHome, "voice-agent-workspace"); await mkdir(cwd, { recursive: true }); @@ -4716,11 +4570,11 @@ export class Session { throw new Error("Voice MCP stdio bridge is not configured"); } - const model = this.getVoiceAgentModel(provider); + const model = this.voiceLlmModel?.trim() || undefined; const config: AgentSessionConfig = { provider, cwd, - modeId: VOICE_AGENT_DEFAULT_MODE[provider], + modeId: this.voiceLlmModeId ?? "default", ...(model ? { model } : {}), mcpServers: { paseo: buildVoiceAgentMcpServerConfig({ @@ -4872,372 +4726,22 @@ export class Session { * Process user message through LLM with streaming and tool execution */ private async processWithLLM(enableTTS: boolean, latestUserText?: string): Promise { - if (enableTTS && this.voiceLlmProvider !== "openrouter") { - const text = - typeof latestUserText === "string" && latestUserText.trim().length > 0 - ? latestUserText - : (() => { - const lastUser = [...this.messages] - .reverse() - .find((message) => message.role === "user"); - if (!lastUser) { - return ""; - } - return typeof lastUser.content === "string" - ? lastUser.content - : JSON.stringify(lastUser.content); - })(); - const normalized = text.trim(); + try { + if (!enableTTS) { + this.sessionLogger.warn("Ignoring non-voice processWithLLM call; voice is agent-only"); + return; + } + const normalized = (latestUserText ?? "").trim(); if (!normalized) { return; } await this.processWithVoiceAgent(normalized); - return; - } - - let assistantResponse = ""; - let pendingTTS: Promise | null = null; - let textBuffer = ""; - let sawTextDelta = false; - - const flushTextBuffer = () => { - if (textBuffer.length > 0) { - // TTS handling (capture mode at generation time for drift protection) - if (enableTTS && !this.speechInProgress) { - const modeAtGeneration = this.isVoiceMode; - pendingTTS = this.ttsManager.generateAndWaitForPlayback( - textBuffer, - (msg) => this.emit(msg), - this.abortController.signal, - modeAtGeneration - ); - } else if (enableTTS && this.speechInProgress) { - this.sessionLogger.debug("Skipping TTS chunk while speech in progress"); - } - - // Emit activity log - this.emit({ - type: "activity_log", - payload: { - id: uuidv4(), - timestamp: new Date(), - type: "assistant", - content: textBuffer, - }, - }); - } - textBuffer = ""; - }; - - try { - // Debug: dump conversation before LLM call - await this.dumpConversation(); - - const openrouterApiKey = - this.openrouterApiKey ?? process.env.OPENROUTER_API_KEY ?? null; - invariant( - openrouterApiKey, - "OpenRouter API key is required (set providers.openrouter.apiKey in config.json or OPENROUTER_API_KEY)" - ); - - const openrouter = createOpenRouter({ - apiKey: openrouterApiKey, - }); - - // Wait for agent MCP to initialize if needed - if (!this.agentTools) { - this.sessionLogger.debug("Waiting for agent MCP initialization..."); - const startTime = Date.now(); - while (!this.agentTools && Date.now() - startTime < 5000) { - await new Promise((resolve) => setTimeout(resolve, 100)); - } - if (!this.agentTools) { - this.sessionLogger.info("Agent MCP tools unavailable; continuing with default tool set"); - } - } - - const allTools = getAllTools(this.agentTools ?? undefined); - - const result = await streamText({ - model: openrouter(this.voiceLlmModel ?? "anthropic/claude-haiku-4.5"), - system: getSystemPrompt(), - providerOptions: { - openrouter: { - transforms: ["middle-out"], // Compress prompts that are > context size. - } as OpenRouterProviderOptions, - }, - messages: this.messages, - tools: allTools, - abortSignal: this.abortController.signal, - onFinish: async (event) => { - const newMessages = event.response.messages; - if (newMessages.length > 0) { - this.messages.push(...newMessages); - this.sessionLogger.debug( - { messageCount: newMessages.length }, - `onFinish - saved message with ${newMessages.length} steps` - ); - } - }, - onChunk: async ({ chunk }) => { - if (chunk.type === "text-delta") { - sawTextDelta = true; - // Accumulate text in buffer - textBuffer += chunk.text; - assistantResponse += chunk.text; - - // Emit chunk for UI streaming - this.emit({ - type: "assistant_chunk", - payload: { chunk: chunk.text }, - }); - } else if (chunk.type === "tool-call") { - // Flush accumulated text as a segment before tool call - flushTextBuffer(); - - // Wait for pending TTS before executing tool - if (pendingTTS) { - this.sessionLogger.debug( - { toolName: chunk.toolName }, - `Waiting for TTS before executing ${chunk.toolName}` - ); - await pendingTTS; - } - - // Handle present_artifact tool specially - if (chunk.toolName === "present_artifact") { - await this.handlePresentArtifact( - chunk.toolCallId, - chunk.input as PresentArtifactArgs - ); - } - - // Emit tool call activity log - this.emit({ - type: "activity_log", - payload: { - id: chunk.toolCallId, - timestamp: new Date(), - type: "tool_call", - content: `Calling ${chunk.toolName}`, - metadata: { - toolCallId: chunk.toolCallId, - toolName: chunk.toolName, - arguments: chunk.input, - }, - }, - }); - } else if (chunk.type === "tool-result") { - // Check if this is a create_agent result - if (chunk.toolName === "create_agent" && chunk.output) { - const result = chunk.output as any; - if (result.structuredContent?.agentId) { - const agentId = result.structuredContent.agentId; - this.emit({ - type: "status", - payload: { - status: "agent_created", - agentId, - }, - }); - } - } - - // Emit tool result event - this.emit({ - type: "activity_log", - payload: { - id: chunk.toolCallId, - timestamp: new Date(), - type: "tool_result", - content: `Tool ${chunk.toolName} completed`, - metadata: { - toolCallId: chunk.toolCallId, - toolName: chunk.toolName, - result: chunk.output, - }, - }, - }); - } - }, - onError: async (error) => { - this.sessionLogger.error({ err: error }, "Stream error"); - - this.emit({ - type: "activity_log", - payload: { - id: uuidv4(), - timestamp: new Date(), - type: "error", - content: `Stream error: ${ - error instanceof Error ? error.message : String(error) - }`, - }, - }); - }, - stopWhen: stepCountIs(10), - }); - - // Consume the fullStream to handle tool-error chunks - for await (const part of result.fullStream) { - if (part.type === "tool-error") { - this.emit({ - type: "activity_log", - payload: { - id: part.toolCallId, - timestamp: new Date(), - type: "error", - content: `Tool ${part.toolName} failed: ${ - part.error instanceof Error - ? part.error.message - : String(part.error) - }`, - metadata: { - toolCallId: part.toolCallId, - toolName: part.toolName, - error: part.error, - }, - }, - }); - } - } - - if (!sawTextDelta) { - let fallbackText = ""; - try { - fallbackText = (await result.text).trim(); - } catch { - fallbackText = ""; - } - if (fallbackText.length > 0) { - textBuffer += fallbackText; - assistantResponse += fallbackText; - this.emit({ - type: "assistant_chunk", - payload: { chunk: fallbackText }, - }); - } - } - - // Flush any remaining text at the end - flushTextBuffer(); - - // Note: Message is saved by onFinish callback with proper tool calls - - // Now wait for any pending TTS, but don't fail if it times out - if (pendingTTS) { - try { - await pendingTTS; - } catch (ttsError) { - this.sessionLogger.error( - { err: ttsError }, - "TTS playback failed (message already saved)" - ); - } - } - } catch (error) { - // Note: Partial messages are saved by onAbort callback with proper tool calls - - // Check if this is an abort error - const isAbortError = - error instanceof Error && error.name === "AbortError"; - - // Only emit error log for non-abort errors - if (!isAbortError) { - this.emit({ - type: "activity_log", - payload: { - id: uuidv4(), - timestamp: new Date(), - type: "error", - content: `Error: ${ - error instanceof Error ? error.message : String(error) - }`, - }, - }); - } - - // Don't re-throw abort errors (they're expected during interruptions) - if (isAbortError) { - this.sessionLogger.debug("Stream aborted (partial response saved)"); - return; - } - - // Re-throw unexpected errors - throw error; } finally { - // Increment turn index for next LLM call this.turnIndex++; - - // Clear the stream promise tracker this.currentStreamPromise = null; } } - /** - * Handle present_artifact tool execution - */ - private async handlePresentArtifact( - toolCallId: string, - args: PresentArtifactArgs - ): Promise { - let content: string; - let isBase64 = false; - - try { - if (args.source.type === "file") { - const fileBuffer = await readFile(args.source.path); - content = fileBuffer.toString("base64"); - isBase64 = true; - } else if (args.source.type === "command_output") { - const { stdout } = await execAsync(args.source.command, { - encoding: "buffer", - }); - content = stdout.toString("base64"); - isBase64 = true; - } else if (args.source.type === "text") { - content = args.source.text; - isBase64 = false; - } else { - content = "[Unknown source type]"; - isBase64 = false; - } - } catch (error) { - this.sessionLogger.error( - { err: error }, - "Failed to resolve artifact source" - ); - content = `[Error: ${ - error instanceof Error ? error.message : String(error) - }]`; - isBase64 = false; - } - - // Emit artifact message - this.emit({ - type: "artifact", - payload: { - type: args.type, - id: toolCallId, - title: args.title, - content, - isBase64, - }, - }); - - // Emit activity log for artifact - this.emit({ - type: "activity_log", - payload: { - id: toolCallId, - timestamp: new Date(), - type: "system", - content: `${args.type} artifact: ${args.title}`, - metadata: { artifactId: toolCallId, artifactType: args.type }, - }, - }); - } - /** * Handle abort request from client */ @@ -5445,38 +4949,6 @@ export class Session { this.onMessage(msg); } - /** - * Debug helper: dump conversation to disk - */ - private async dumpConversation(): Promise { - try { - const dumpDir = join(process.cwd(), ".debug.conversations"); - await mkdir(dumpDir, { recursive: true }); - - const filename = `${this.sessionId}-${this.turnIndex}.json`; - const filepath = join(dumpDir, filename); - - const dump = { - voiceAssistantAgentId: this.voiceAssistantAgentId, - sessionId: this.sessionId, - turnIndex: this.turnIndex, - timestamp: new Date().toISOString(), - messages: this.messages, - }; - - await writeFile(filepath, inspect(dump, { depth: null }), "utf-8"); - this.sessionLogger.debug( - { filepath }, - `Dumped conversation to ${filepath}` - ); - } catch (error) { - this.sessionLogger.error( - { err: error }, - "Failed to dump conversation" - ); - } - } - /** * Clean up session resources */ diff --git a/packages/server/src/server/test-utils/paseo-daemon.ts b/packages/server/src/server/test-utils/paseo-daemon.ts index 74da63fab..44c1a69b8 100644 --- a/packages/server/src/server/test-utils/paseo-daemon.ts +++ b/packages/server/src/server/test-utils/paseo-daemon.ts @@ -21,7 +21,6 @@ type TestPaseoDaemonOptions = { cleanup?: boolean; openai?: PaseoOpenAIConfig; speech?: PaseoSpeechConfig; - openrouterApiKey?: string | null; voiceLlmProvider?: PaseoDaemonConfig["voiceLlmProvider"]; voiceLlmProviderExplicit?: boolean; voiceLlmModel?: string | null; @@ -82,7 +81,6 @@ export async function createTestPaseoDaemon( appBaseUrl: "https://app.paseo.sh", openai: options.openai, speech: options.speech, - openrouterApiKey: options.openrouterApiKey ?? null, voiceLlmProvider: options.voiceLlmProvider ?? null, voiceLlmProviderExplicit: options.voiceLlmProviderExplicit ?? false, voiceLlmModel: options.voiceLlmModel ?? null, diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 11c500b5f..39b147368 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -22,7 +22,6 @@ import { PushService } from "./push/push-service.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; export type AgentMcpTransportFactory = () => Promise; -type VoiceAgentProvider = "claude" | "codex" | "opencode"; type VoiceMcpStdioConfig = { command: string; baseArgs: string[]; @@ -79,12 +78,10 @@ export class VoiceAssistantWebSocketServer { stt?: SpeechToTextProvider | null; } | null; private readonly voice: { - openrouterApiKey?: string | null; - voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null; + voiceLlmProvider?: AgentProvider | null; + voiceLlmModeId?: string | null; voiceLlmProviderExplicit?: boolean; - voiceLlmDefaultProvider?: VoiceAgentProvider | null; voiceLlmModel?: string | null; - voiceLlmAvailability?: Record | null; voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; } | null; private readonly voiceSpeakHandlers = new Map< @@ -114,12 +111,10 @@ export class VoiceAssistantWebSocketServer { speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null }, terminalManager?: TerminalManager | null, voice?: { - openrouterApiKey?: string | null; - voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null; + voiceLlmProvider?: AgentProvider | null; + voiceLlmModeId?: string | null; voiceLlmProviderExplicit?: boolean; - voiceLlmDefaultProvider?: VoiceAgentProvider | null; voiceLlmModel?: string | null; - voiceLlmAvailability?: Record | null; voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; }, dictation?: { diff --git a/packages/website/public/schemas/paseo.config.v1.json b/packages/website/public/schemas/paseo.config.v1.json index 14a45f390..116daee60 100644 --- a/packages/website/public/schemas/paseo.config.v1.json +++ b/packages/website/public/schemas/paseo.config.v1.json @@ -86,8 +86,44 @@ }, "additionalProperties": false }, - "openrouter": { - "$ref": "#/definitions/PaseoConfigV1/properties/providers/properties/openai" + "sherpaOnnx": { + "type": "object", + "properties": { + "modelsDir": { + "type": "string", + "minLength": 1 + }, + "autoDownload": { + "type": "boolean" + }, + "stt": { + "type": "object", + "properties": { + "preset": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "tts": { + "type": "object", + "properties": { + "preset": { + "type": "string", + "minLength": 1 + }, + "speakerId": { + "type": "number" + }, + "speed": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false } }, "additionalProperties": false @@ -104,7 +140,8 @@ "provider": { "type": "string", "enum": [ - "openai" + "openai", + "local" ] }, "model": { @@ -129,7 +166,9 @@ "provider": { "type": "string", "enum": [ - "openrouter" + "claude", + "codex", + "opencode" ] }, "model": { @@ -145,7 +184,8 @@ "provider": { "type": "string", "enum": [ - "openai" + "openai", + "local" ] }, "model": { @@ -161,7 +201,8 @@ "provider": { "type": "string", "enum": [ - "openai" + "openai", + "local" ] }, "model": { diff --git a/packages/website/src/routes/docs/configuration.tsx b/packages/website/src/routes/docs/configuration.tsx index 96b671c60..169287660 100644 --- a/packages/website/src/routes/docs/configuration.tsx +++ b/packages/website/src/routes/docs/configuration.tsx @@ -67,8 +67,7 @@ function Configuration() { "$schema": "https://paseo.sh/schemas/paseo.config.v1.json", "version": 1, "providers": { - "openai": { "apiKey": "..." }, - "openrouter": { "apiKey": "..." } + "openai": { "apiKey": "..." } }, "daemon": { "listen": "127.0.0.1:6767", @@ -92,6 +91,7 @@ function Configuration() { "features": { "dictation": { "stt": { "provider": "local" } }, "voiceMode": { + "llm": { "provider": "claude", "model": "haiku" }, "stt": { "provider": "local" }, "tts": { "provider": "local" } } @@ -106,6 +106,13 @@ function Configuration() { } }`} +

+ Voice LLM orchestration is agents-only. Set{' '} + features.voiceMode.llm.provider to one of{' '} + claude, codex, or{' '} + opencode. If set, startup is strict and fails if unavailable. + If omitted, Paseo picks the first available voice-enabled agent provider. +

Local voice uses ONNX models (sherpa-onnx + PocketTTS). Default presets are parakeet-tdt-0.6b-v3-int8 for STT and @@ -152,7 +159,8 @@ function Configuration() {

  • PASEO_HOME — set Paseo home directory
  • PASEO_LISTEN — override daemon.listen
  • PASEO_ALLOWED_HOSTS — override/extend daemon.allowedHosts
    • -
  • OPENAI_API_KEY and OPENROUTER_API_KEY — override provider keys
    • +
  • OPENAI_API_KEY — override OpenAI provider key
    • +
  • PASEO_VOICE_LLM_PROVIDER — override voice LLM provider (claude, codex, opencode)
  • PASEO_DICTATION_STT_PROVIDER, PASEO_VOICE_STT_PROVIDER, PASEO_VOICE_TTS_PROVIDER — override voice provider selection (local or openai)
  • PASEO_SHERPA_ONNX_MODELS_DIR and PASEO_SHERPA_ONNX_AUTO_DOWNLOAD — control local model directory and download behavior
  • PASEO_SHERPA_STT_PRESET and PASEO_SHERPA_TTS_PRESET — override local STT/TTS model presets
    • From a1008c36685581c2ef031f9cc89b3463e7200f23 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 6 Feb 2026 20:53:38 +0700 Subject: [PATCH 12/21] Update files --- packages/server/src/server/bootstrap.ts | 431 +------------- packages/server/src/server/config.ts | 178 +----- .../src/server/daemon-client.e2e.test.ts | 26 +- .../server/src/server/persisted-config.ts | 23 +- packages/server/src/server/session.ts | 28 +- .../providers/local/sherpa/model-catalog.ts | 46 ++ .../local/sherpa/model-downloader.ts | 2 +- .../local/sherpa/speech-download.e2e.test.ts | 16 +- .../speech/speech-config-resolver.test.ts | 94 ++++ .../server/speech/speech-config-resolver.ts | 196 +++++++ .../src/server/speech/speech-runtime.ts | 529 ++++++++++++++++++ .../server/src/server/websocket-server.ts | 9 + .../public/schemas/paseo.config.v1.json | 33 +- packages/website/src/routeTree.gen.ts | 21 + packages/website/src/routes/docs.tsx | 1 + .../website/src/routes/docs/configuration.tsx | 78 +-- packages/website/src/routes/docs/index.tsx | 19 +- packages/website/src/routes/docs/voice.tsx | 126 +++++ 18 files changed, 1122 insertions(+), 734 deletions(-) create mode 100644 packages/server/src/server/speech/speech-config-resolver.test.ts create mode 100644 packages/server/src/server/speech/speech-config-resolver.ts create mode 100644 packages/server/src/server/speech/speech-runtime.ts create mode 100644 packages/website/src/routes/docs/voice.tsx diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 4e62e14c3..04cfb27b6 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -41,20 +41,9 @@ function parseListenString(listen: string): ListenTarget { import { VoiceAssistantWebSocketServer } from "./websocket-server.js"; import { DownloadTokenStore } from "./file-download/token-store.js"; -import { OpenAISTT, type STTConfig } from "./speech/providers/openai/stt.js"; -import { OpenAITTS, type TTSConfig } from "./speech/providers/openai/tts.js"; -import { OpenAIRealtimeTranscriptionSession } from "./speech/providers/openai/realtime-transcription-session.js"; -import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; -import { SherpaOnlineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-online-recognizer.js"; -import { SherpaOfflineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-offline-recognizer.js"; -import { SherpaOnnxSTT } from "./speech/providers/local/sherpa/sherpa-stt.js"; -import { SherpaOnnxParakeetSTT } from "./speech/providers/local/sherpa/sherpa-parakeet-stt.js"; -import { SherpaOnnxTTS } from "./speech/providers/local/sherpa/sherpa-tts.js"; -import { SherpaRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-realtime-session.js"; -import { SherpaParakeetRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-parakeet-realtime-session.js"; -import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./speech/providers/local/sherpa/model-downloader.js"; -import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js"; -import { PocketTtsOnnxTTS } from "./speech/providers/local/pocket/pocket-tts-onnx.js"; +import type { STTConfig } from "./speech/providers/openai/stt.js"; +import type { TTSConfig } from "./speech/providers/openai/tts.js"; +import { initializeSpeechRuntime } from "./speech/speech-runtime.js"; import { AgentManager } from "./agent/agent-manager.js"; import { AgentStorage } from "./agent/agent-storage.js"; import { attachAgentStoragePersistence } from "./persistence-hooks.js"; @@ -108,26 +97,24 @@ export type PaseoOpenAIConfig = { apiKey?: string; stt?: Partial & { apiKey?: string }; tts?: Partial & { apiKey?: string }; + realtimeTranscriptionModel?: string; }; -export type PaseoSherpaOnnxConfig = { +export type PaseoLocalSpeechConfig = { modelsDir: string; autoDownload?: boolean; - stt?: { - preset?: string; - }; - tts?: { - preset?: string; - speakerId?: number; - speed?: number; - }; }; export type PaseoSpeechConfig = { dictationSttProvider?: "openai" | "local"; voiceSttProvider?: "openai" | "local"; voiceTtsProvider?: "openai" | "local"; - sherpaOnnx?: PaseoSherpaOnnxConfig; + local?: PaseoLocalSpeechConfig; + dictationLocalSttModel?: string; + voiceLocalSttModel?: string; + voiceLocalTtsModel?: string; + voiceLocalTtsSpeakerId?: number; + voiceLocalTtsSpeed?: number; }; export type PaseoDaemonConfig = { @@ -551,384 +538,17 @@ export async function createPaseoDaemon( }); }, }); - - - let sttService: SpeechToTextProvider | null = null; - let ttsService: TextToSpeechProvider | null = null; - let dictationSttService: SpeechToTextProvider | null = null; - - let sherpaOnline: SherpaOnlineRecognizerEngine | null = null; - let sherpaOffline: SherpaOfflineRecognizerEngine | null = null; - let sherpaTts: TextToSpeechProvider | null = null; - - const openaiApiKey = config.openai?.apiKey; - const speechConfig = config.speech ?? null; - const sherpaConfig = speechConfig?.sherpaOnnx ?? null; - - const voiceSttProvider = speechConfig?.voiceSttProvider ?? "local"; - const voiceTtsProvider = speechConfig?.voiceTtsProvider ?? "local"; - const dictationSttProvider = speechConfig?.dictationSttProvider ?? "local"; - - const wantsLocalDictation = dictationSttProvider === "local"; - const wantsLocalVoiceStt = voiceSttProvider === "local"; - const wantsLocalVoiceTts = voiceTtsProvider === "local"; - - const openaiSttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey; - const openaiTtsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey; - const openaiDictationApiKey = openaiApiKey; - - const missingOpenAiCredentialsFor: string[] = []; - if (voiceSttProvider === "openai" && !openaiSttApiKey) { - missingOpenAiCredentialsFor.push("voice.stt"); - } - if (voiceTtsProvider === "openai" && !openaiTtsApiKey) { - missingOpenAiCredentialsFor.push("voice.tts"); - } - if (dictationSttProvider === "openai" && !openaiDictationApiKey) { - missingOpenAiCredentialsFor.push("dictation.stt"); - } - - if (missingOpenAiCredentialsFor.length > 0) { - logger.error( - { - requestedProviders: { - dictationStt: dictationSttProvider, - voiceStt: voiceSttProvider, - voiceTts: voiceTtsProvider, - }, - missingOpenAiCredentialsFor, - }, - "Invalid speech configuration: OpenAI provider selected but credentials are missing" - ); - throw new Error( - `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}` - ); - } - - logger.info( - { - requestedProviders: { - dictationStt: dictationSttProvider, - voiceStt: voiceSttProvider, - voiceTts: voiceTtsProvider, - }, - availability: { - openai: { - stt: Boolean(openaiSttApiKey), - tts: Boolean(openaiTtsApiKey), - dictationStt: Boolean(openaiDictationApiKey), - }, - local: { - configured: Boolean(sherpaConfig), - modelsDir: sherpaConfig?.modelsDir ?? null, - autoDownload: sherpaConfig?.autoDownload ?? null, - }, - }, - }, - "Speech provider reconciliation started" - ); - - if ((wantsLocalDictation || wantsLocalVoiceStt || wantsLocalVoiceTts) && sherpaConfig) { - const autoDownload = sherpaConfig.autoDownload ?? (process.env.VITEST ? false : true); - let sttPreset = (sherpaConfig.stt?.preset ?? "parakeet-tdt-0.6b-v3-int8").trim(); - if ( - sttPreset !== "zipformer-bilingual-zh-en-2023-02-20" && - sttPreset !== "paraformer-bilingual-zh-en" && - sttPreset !== "parakeet-tdt-0.6b-v3-int8" - ) { - throw new Error(`Unknown local STT preset: ${sttPreset}`); - } - - let ttsPreset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim(); - if ( - ttsPreset !== "kitten-nano-en-v0_1-fp16" && - ttsPreset !== "kokoro-en-v0_19" && - ttsPreset !== "pocket-tts-onnx-int8" - ) { - throw new Error(`Unknown local TTS preset: ${ttsPreset}`); - } - - const modelIds: SherpaOnnxModelId[] = []; - if (wantsLocalDictation || wantsLocalVoiceStt) { - modelIds.push(sttPreset as SherpaOnnxModelId); - } - if (wantsLocalVoiceTts) { - modelIds.push(ttsPreset as SherpaOnnxModelId); - } - - try { - logger.info( - { - modelsDir: sherpaConfig.modelsDir, - modelIds, - autoDownload, - }, - "Ensuring local speech models" - ); - await ensureSherpaOnnxModels({ - modelsDir: sherpaConfig.modelsDir, - modelIds, - autoDownload, - logger, - }); - } catch (err) { - logger.error( - { - err, - modelsDir: sherpaConfig.modelsDir, - autoDownload, - hint: - "Run: npm run dev --workspace=@getpaseo/server, then run: " + - "`tsx packages/server/scripts/download-speech-models.ts --models-dir --model `", - }, - "Failed to ensure local speech models" - ); - } - } - - if ((wantsLocalDictation || wantsLocalVoiceStt) && sherpaConfig) { - let preset = (sherpaConfig.stt?.preset ?? "parakeet-tdt-0.6b-v3-int8").trim(); - if ( - preset !== "zipformer-bilingual-zh-en-2023-02-20" && - preset !== "paraformer-bilingual-zh-en" && - preset !== "parakeet-tdt-0.6b-v3-int8" - ) { - throw new Error(`Unknown local STT preset: ${preset}`); - } - const base = sherpaConfig.modelsDir; - - try { - if (preset === "parakeet-tdt-0.6b-v3-int8") { - const modelDir = getSherpaOnnxModelDir(base, "parakeet-tdt-0.6b-v3-int8"); - sherpaOffline = new SherpaOfflineRecognizerEngine( - { - model: { - kind: "nemo_transducer", - encoder: `${modelDir}/encoder.int8.onnx`, - decoder: `${modelDir}/decoder.int8.onnx`, - joiner: `${modelDir}/joiner.int8.onnx`, - tokens: `${modelDir}/tokens.txt`, - }, - numThreads: 2, - debug: 0, - }, - logger - ); - } else { - const model = - preset === "paraformer-bilingual-zh-en" - ? { - kind: "paraformer" as const, - encoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/encoder.int8.onnx`, - decoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/decoder.int8.onnx`, - tokens: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/tokens.txt`, - } - : { - kind: "transducer" as const, - encoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx`, - decoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx`, - joiner: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx`, - tokens: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt`, - modelType: "zipformer", - }; - - sherpaOnline = new SherpaOnlineRecognizerEngine( - { - model, - numThreads: 1, - debug: 0, - }, - logger - ); - } - } catch (err) { - logger.error( - { - err, - modelsDir: sherpaConfig.modelsDir, - preset, - hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`, - }, - "Failed to initialize Sherpa STT (models missing or invalid)" - ); - sherpaOnline = null; - sherpaOffline = null; - } - } else if (wantsLocalDictation || wantsLocalVoiceStt) { - logger.warn( - { configured: Boolean(sherpaConfig) }, - "Local STT selected but local provider config is missing; STT will be unavailable" - ); - } - - if (wantsLocalVoiceTts && sherpaConfig) { - let preset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim(); - if ( - preset !== "kitten-nano-en-v0_1-fp16" && - preset !== "kokoro-en-v0_19" && - preset !== "pocket-tts-onnx-int8" - ) { - throw new Error(`Unknown local TTS preset: ${preset}`); - } - try { - if (preset === "pocket-tts-onnx-int8") { - const modelDir = getSherpaOnnxModelDir(sherpaConfig.modelsDir, "pocket-tts-onnx-int8"); - sherpaTts = await PocketTtsOnnxTTS.create( - { - modelDir, - precision: "int8", - targetChunkMs: 50, - }, - logger - ); - } else { - const modelDir = `${sherpaConfig.modelsDir}/${preset}`; - sherpaTts = new SherpaOnnxTTS( - { - preset: preset as any, - modelDir, - speakerId: sherpaConfig.tts?.speakerId, - speed: sherpaConfig.tts?.speed, - }, - logger - ); - } - } catch (err) { - logger.error( - { - err, - preset, - hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`, - }, - "Failed to initialize Sherpa TTS (models missing or invalid)" - ); - sherpaTts = null; - } - } else if (wantsLocalVoiceTts) { - logger.warn( - { configured: Boolean(sherpaConfig) }, - "Local TTS selected but local provider config is missing; TTS will be unavailable" - ); - } - - if (wantsLocalVoiceStt && sherpaOffline) { - sttService = new SherpaOnnxParakeetSTT({ engine: sherpaOffline }, logger); - } else if (wantsLocalVoiceStt && sherpaOnline) { - sttService = new SherpaOnnxSTT({ engine: sherpaOnline }, logger); - } - - if (wantsLocalVoiceTts && sherpaTts) { - ttsService = sherpaTts; - } - - if (wantsLocalDictation && sherpaOnline) { - dictationSttService = { - id: "local", - createSession: () => new SherpaRealtimeTranscriptionSession({ engine: sherpaOnline! }), - }; - } else if (wantsLocalDictation && sherpaOffline) { - dictationSttService = { - id: "local", - createSession: () => - new SherpaParakeetRealtimeTranscriptionSession({ engine: sherpaOffline! }), - }; - } - - const needsOpenAiStt = !sttService && voiceSttProvider === "openai"; - const needsOpenAiTts = !ttsService && voiceTtsProvider === "openai"; - const needsOpenAiDictation = dictationSttProvider === "openai"; - - if ( - (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) && - (openaiSttApiKey || openaiTtsApiKey || openaiDictationApiKey) - ) { - logger.info("OpenAI speech provider initialized"); - - if (needsOpenAiStt) { - if (openaiSttApiKey) { - const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {}; - sttService = new OpenAISTT( - { - apiKey: openaiSttApiKey, - ...sttConfig, - }, - logger - ); - } - } - - if (needsOpenAiTts) { - if (openaiTtsApiKey) { - const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {}; - ttsService = new OpenAITTS( - { - apiKey: openaiTtsApiKey, - voice: "alloy", - model: "tts-1", - responseFormat: "pcm", - ...ttsConfig, - }, - logger - ); - } - } - - if (needsOpenAiDictation) { - const transcriptionModel = - process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe"; - - dictationSttService = { - id: "openai", - createSession: ({ logger: sessionLogger, language, prompt }) => - new OpenAIRealtimeTranscriptionSession({ - apiKey: openaiDictationApiKey!, - logger: sessionLogger, - transcriptionModel, - ...(language ? { language } : {}), - ...(prompt ? { prompt } : {}), - turnDetection: null, - }), - }; - } - } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) { - logger.warn("OPENAI_API_KEY not set - OpenAI STT/TTS/dictation is unavailable"); - } - - const effectiveProviders = { - dictationStt: dictationSttService?.id ?? "unavailable", - voiceStt: sttService?.id ?? "unavailable", - voiceTts: !ttsService ? "unavailable" : ttsService === sherpaTts ? "local" : "openai", - }; - const unavailableFeatures = [ - !dictationSttService ? "dictation.stt" : null, - !sttService ? "voice.stt" : null, - !ttsService ? "voice.tts" : null, - ].filter((feature): feature is string => feature !== null); - - if (unavailableFeatures.length > 0) { - logger.error( - { - requestedProviders: { - dictationStt: dictationSttProvider, - voiceStt: voiceSttProvider, - voiceTts: voiceTtsProvider, - }, - effectiveProviders, - unavailableFeatures, - }, - "Speech provider reconciliation failed: configured features are unavailable" - ); - throw new Error( - `Configured speech features unavailable: ${unavailableFeatures.join(", ")}` - ); - } else { - logger.info( - { - effectiveProviders, - }, - "Speech provider reconciliation completed" - ); - } + const { + sttService, + ttsService, + dictationSttService, + cleanup: cleanupSpeechRuntime, + localModelConfig, + } = await initializeSpeechRuntime({ + logger, + openaiConfig: config.openai, + speechConfig: config.speech, + }); wsServer = new VoiceAssistantWebSocketServer( httpServer, @@ -962,6 +582,7 @@ export async function createPaseoDaemon( { finalTimeoutMs: config.dictationFinalTimeoutMs, stt: dictationSttService, + localModels: localModelConfig ?? undefined, } ); @@ -1064,11 +685,7 @@ export async function createPaseoDaemon( await agentStorage.flush().catch(() => undefined); await shutdownProviders(logger); terminalManager.killAll(); - if (sherpaTts && typeof (sherpaTts as any).free === "function") { - (sherpaTts as any).free(); - } - sherpaOnline?.free(); - sherpaOffline?.free(); + cleanupSpeechRuntime(); await relayTransport?.stop().catch(() => undefined); if (wsServer) { await wsServer.close(); diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index bbd455bdd..1b3283c2d 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -1,11 +1,10 @@ import path from "node:path"; import type { PaseoDaemonConfig } from "./bootstrap.js"; -import type { STTConfig } from "./speech/providers/openai/stt.js"; -import type { TTSConfig } from "./speech/providers/openai/tts.js"; import { loadPersistedConfig } from "./persisted-config.js"; import type { AgentProvider } from "./agent/agent-sdk-types.js"; import { AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js"; +import { resolveSpeechConfig } from "./speech/speech-config-resolver.js"; import { mergeAllowedHosts, parseAllowedHostsEnv, @@ -27,69 +26,6 @@ export type CliConfigOverrides = Partial<{ allowedHosts: AllowedHostsConfig; }>; -function parseOpenAIConfig( - env: NodeJS.ProcessEnv, - configApiKey: string | undefined, - config: { - dictationSttModel?: string; - dictationSttConfidenceThreshold?: number; - voiceSttModel?: string; - voiceTtsVoice?: TTSConfig["voice"]; - voiceTtsModel?: TTSConfig["model"]; - } -) { - const apiKey = env.OPENAI_API_KEY ?? configApiKey; - if (!apiKey) return undefined; - - const sttConfidenceThreshold = env.STT_CONFIDENCE_THRESHOLD - ? parseFloat(env.STT_CONFIDENCE_THRESHOLD) - : config.dictationSttConfidenceThreshold; - const sttModel = ( - env.STT_MODEL ?? - config.voiceSttModel ?? - config.dictationSttModel - ) as STTConfig["model"] | undefined; - const ttsVoice = (env.TTS_VOICE || "alloy") as - | "alloy" - | "echo" - | "fable" - | "onyx" - | "nova" - | "shimmer"; - const ttsModel = (env.TTS_MODEL || config.voiceTtsModel || "tts-1") as - | "tts-1" - | "tts-1-hd"; - const configuredVoice = config.voiceTtsVoice; - - return { - apiKey, - stt: { - apiKey, - confidenceThreshold: sttConfidenceThreshold, - ...(sttModel ? { model: sttModel } : {}), - }, - tts: { - apiKey, - voice: configuredVoice ?? ttsVoice, - model: ttsModel, - responseFormat: "pcm" as TTSConfig["responseFormat"], - }, - }; -} - -function parseSpeechProviderId(value: unknown): "openai" | "local" | null { - if (typeof value !== "string") { - return null; - } - const normalized = value.trim().toLowerCase(); - if (!normalized) { - return null; - } - if (normalized === "openai") return "openai"; - if (normalized === "local") return "local"; - return null; -} - function parseVoiceLlmProviderId(value: unknown): AgentProvider | null { if (typeof value !== "string") { return null; @@ -103,36 +39,6 @@ function parseVoiceLlmProviderId(value: unknown): AgentProvider | null { : null; } -function normalizeSherpaSttPreset(value: string): string { - const raw = value.trim(); - const normalized = raw.toLowerCase(); - if (normalized === "zipformer" || normalized === "zipformer-bilingual") { - return "zipformer-bilingual-zh-en-2023-02-20"; - } - if (normalized === "paraformer") { - return "paraformer-bilingual-zh-en"; - } - if (normalized === "parakeet" || normalized === "parakeet-v3" || normalized === "parakeet-tdt") { - return "parakeet-tdt-0.6b-v3-int8"; - } - return raw; -} - -function normalizeSherpaTtsPreset(value: string): string { - const raw = value.trim(); - const normalized = raw.toLowerCase(); - if (normalized === "pocket" || normalized === "pocket-tts") { - return "pocket-tts-onnx-int8"; - } - if (normalized === "kitten") { - return "kitten-nano-en-v0_1-fp16"; - } - if (normalized === "kokoro") { - return "kokoro-en-v0_19"; - } - return raw; -} - export function loadConfig( paseoHome: string, options?: { @@ -185,79 +91,12 @@ export function loadConfig( const appBaseUrl = env.PASEO_APP_BASE_URL ?? persisted.app?.baseUrl ?? DEFAULT_APP_BASE_URL; - const openai = parseOpenAIConfig(env, persisted.providers?.openai?.apiKey, { - dictationSttModel: persisted.features?.dictation?.stt?.model, - dictationSttConfidenceThreshold: - persisted.features?.dictation?.stt?.confidenceThreshold, - voiceSttModel: persisted.features?.voiceMode?.stt?.model, - voiceTtsModel: persisted.features?.voiceMode?.tts?.model, - voiceTtsVoice: persisted.features?.voiceMode?.tts?.voice, + const { openai, speech } = resolveSpeechConfig({ + paseoHome, + env, + persisted, }); - const dictationSttProvider = - parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER) ?? - parseSpeechProviderId(persisted.features?.dictation?.stt?.provider) ?? - "local"; - - const voiceSttProvider = - parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER) ?? - parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider) ?? - "local"; - - const voiceTtsProvider = - parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER) ?? - parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider) ?? - "local"; - - const shouldConfigureSherpa = - dictationSttProvider === "local" || - voiceSttProvider === "local" || - voiceTtsProvider === "local" || - typeof env.PASEO_SHERPA_ONNX_MODELS_DIR === "string" || - Boolean(persisted.providers?.sherpaOnnx); - - const sherpaModelsDir = - (env.PASEO_SHERPA_ONNX_MODELS_DIR ?? persisted.providers?.sherpaOnnx?.modelsDir)?.trim() || - path.join(paseoHome, "models", "sherpa-onnx"); - - const sherpaOnnx = shouldConfigureSherpa - ? { - modelsDir: sherpaModelsDir, - autoDownload: - env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD !== undefined - ? env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD === "1" - : persisted.providers?.sherpaOnnx?.autoDownload ?? - // In tests we should never hit the network unexpectedly. - Boolean(env.VITEST) === false, - stt: { - preset: normalizeSherpaSttPreset( - (env.PASEO_SHERPA_STT_PRESET ?? persisted.providers?.sherpaOnnx?.stt?.preset)?.trim() || - (persisted.features?.voiceMode?.stt?.preset ?? - persisted.features?.dictation?.stt?.preset)?.trim() || - "parakeet-tdt-0.6b-v3-int8" - ), - }, - tts: { - preset: normalizeSherpaTtsPreset( - (env.PASEO_SHERPA_TTS_PRESET ?? - persisted.providers?.sherpaOnnx?.tts?.preset ?? - persisted.features?.voiceMode?.tts?.preset)?.trim() || - (env.VITEST ? "kitten-nano-en-v0_1-fp16" : "pocket-tts-onnx-int8") - ), - speakerId: - env.PASEO_SHERPA_TTS_SPEAKER_ID !== undefined - ? Number.parseInt(env.PASEO_SHERPA_TTS_SPEAKER_ID, 10) - : persisted.providers?.sherpaOnnx?.tts?.speakerId ?? - persisted.features?.voiceMode?.tts?.speakerId, - speed: - env.PASEO_SHERPA_TTS_SPEED !== undefined - ? Number.parseFloat(env.PASEO_SHERPA_TTS_SPEED) - : persisted.providers?.sherpaOnnx?.tts?.speed ?? - persisted.features?.voiceMode?.tts?.speed, - }, - } - : undefined; - const envVoiceLlmProvider = parseVoiceLlmProviderId(env.PASEO_VOICE_LLM_PROVIDER); const persistedVoiceLlmProvider = parseVoiceLlmProviderId( persisted.features?.voiceMode?.llm?.provider @@ -284,12 +123,7 @@ export function loadConfig( relayPublicEndpoint, appBaseUrl, openai, - speech: { - dictationSttProvider, - voiceSttProvider, - voiceTtsProvider, - ...(sherpaOnnx ? { sherpaOnnx } : {}), - }, + speech, voiceLlmProvider, voiceLlmProviderExplicit, voiceLlmModel, diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index b395c1191..3941ecb0f 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -18,9 +18,9 @@ import { const openaiApiKey = process.env.OPENAI_API_KEY ?? null; -const sherpaModelsDir = - process.env.PASEO_SHERPA_ONNX_MODELS_DIR ?? - path.join(homedir(), ".paseo", "models", "sherpa-onnx"); +const localModelsDir = + process.env.PASEO_LOCAL_MODELS_DIR ?? + path.join(homedir(), ".paseo", "models", "local-speech"); function hasSherpaZipformerModels(modelsDir: string): boolean { return ( @@ -49,7 +49,7 @@ function hasSherpaKittenModels(modelsDir: string): boolean { ); } -const hasLocalSpeech = hasSherpaZipformerModels(sherpaModelsDir) && hasSherpaKittenModels(sherpaModelsDir); +const hasLocalSpeech = hasSherpaZipformerModels(localModelsDir) && hasSherpaKittenModels(localModelsDir); const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey); const speechTest = hasAnySpeech ? test : test.skip; @@ -102,15 +102,17 @@ describe("daemon client E2E", () => { dictationSttProvider: "local" as const, voiceSttProvider: "local" as const, voiceTtsProvider: "local" as const, - sherpaOnnx: { - modelsDir: sherpaModelsDir, - stt: { - preset: process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20", - }, - tts: { - preset: process.env.PASEO_SHERPA_TTS_PRESET ?? "kitten-nano-en-v0_1-fp16", - }, + local: { + modelsDir: localModelsDir, }, + dictationLocalSttModel: + process.env.PASEO_DICTATION_LOCAL_STT_MODEL ?? + "zipformer-bilingual-zh-en-2023-02-20", + voiceLocalSttModel: + process.env.PASEO_VOICE_LOCAL_STT_MODEL ?? + "zipformer-bilingual-zh-en-2023-02-20", + voiceLocalTtsModel: + process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16", }; ctx = await createDaemonTestContext({ diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index 2ee622536..690e8bef3 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -18,31 +18,17 @@ const ProviderCredentialsSchema = z }) .strict(); -const SherpaOnnxProviderSchema = z +const LocalSpeechProviderSchema = z .object({ modelsDir: z.string().min(1).optional(), autoDownload: z.boolean().optional(), - stt: z - .object({ - preset: z.string().min(1).optional(), - }) - .strict() - .optional(), - tts: z - .object({ - preset: z.string().min(1).optional(), - speakerId: z.number().int().optional(), - speed: z.number().optional(), - }) - .strict() - .optional(), }) .strict(); const ProvidersSchema = z .object({ openai: ProviderCredentialsSchema.optional(), - sherpaOnnx: SherpaOnnxProviderSchema.optional(), + local: LocalSpeechProviderSchema.optional(), }) .strict(); @@ -62,7 +48,6 @@ const FeatureDictationSchema = z .object({ provider: SpeechProviderIdSchema.optional(), model: z.string().min(1).optional(), - preset: z.string().min(1).optional(), confidenceThreshold: z.number().optional(), }) .strict() @@ -83,16 +68,14 @@ const FeatureVoiceModeSchema = z .object({ provider: SpeechProviderIdSchema.optional(), model: z.string().min(1).optional(), - preset: z.string().min(1).optional(), }) .strict() .optional(), tts: z .object({ provider: SpeechProviderIdSchema.optional(), - model: z.enum(["tts-1", "tts-1-hd"]).optional(), + model: z.string().min(1).optional(), voice: z.enum(["alloy", "echo", "fable", "onyx", "nova", "shimmer"]).optional(), - preset: z.string().min(1).optional(), speakerId: z.number().int().optional(), speed: z.number().optional(), }) diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 7249e4d9e..ff4c87e1d 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -339,6 +339,8 @@ export class Session { private readonly voiceLlmProviderExplicit: boolean; private readonly voiceLlmModel: string | null; private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null; + private readonly localSpeechModelsDir: string; + private readonly defaultLocalSpeechModelIds: SherpaOnnxModelId[]; private readonly registerVoiceSpeakHandler?: ( agentId: string, handler: VoiceSpeakHandler @@ -380,6 +382,10 @@ export class Session { dictation?: { finalTimeoutMs?: number; stt?: SpeechToTextProvider | null; + localModels?: { + modelsDir: string; + defaultModelIds: SherpaOnnxModelId[]; + }; } ) { this.clientId = clientId; @@ -397,6 +403,15 @@ export class Session { this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false; this.voiceLlmModel = voice?.voiceLlmModel ?? null; this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null; + const configuredModelsDir = dictation?.localModels?.modelsDir?.trim(); + this.localSpeechModelsDir = + configuredModelsDir && configuredModelsDir.length > 0 + ? configuredModelsDir + : join(this.paseoHome, "models", "local-speech"); + this.defaultLocalSpeechModelIds = + dictation?.localModels?.defaultModelIds && dictation.localModels.defaultModelIds.length > 0 + ? [...new Set(dictation.localModels.defaultModelIds)] + : ["parakeet-tdt-0.6b-v3-int8", "pocket-tts-onnx-int8"]; this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler; this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler; this.registerVoiceCallerContext = voiceBridge?.registerVoiceCallerContext; @@ -1957,9 +1972,7 @@ export class Session { private async handleSpeechModelsListRequest( msg: Extract ): Promise { - const modelsDir = - process.env.PASEO_SHERPA_ONNX_MODELS_DIR?.trim() || - join(this.paseoHome, "models", "sherpa-onnx"); + const modelsDir = this.localSpeechModelsDir; const models = await Promise.all( listSherpaOnnxModels().map(async (model) => { @@ -2004,17 +2017,12 @@ export class Session { private async handleSpeechModelsDownloadRequest( msg: Extract ): Promise { - const modelsDir = - process.env.PASEO_SHERPA_ONNX_MODELS_DIR?.trim() || - join(this.paseoHome, "models", "sherpa-onnx"); + const modelsDir = this.localSpeechModelsDir; const modelIdsRaw = msg.modelIds && msg.modelIds.length > 0 ? msg.modelIds - : [ - process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20", - process.env.PASEO_SHERPA_TTS_PRESET ?? "pocket-tts-onnx-int8", - ]; + : this.defaultLocalSpeechModelIds; const allModelIds = new Set(listSherpaOnnxModels().map((m) => m.id)); const invalid = modelIdsRaw.filter((id) => !allModelIds.has(id as SherpaOnnxModelId)); diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts index 94c6d62bd..6e679d6db 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts @@ -1,3 +1,5 @@ +import { z } from "zod"; + export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts"; export type SherpaOnnxModelId = @@ -8,6 +10,50 @@ export type SherpaOnnxModelId = | "kokoro-en-v0_19" | "pocket-tts-onnx-int8"; +export const LOCAL_STT_MODEL_IDS = [ + "zipformer-bilingual-zh-en-2023-02-20", + "paraformer-bilingual-zh-en", + "parakeet-tdt-0.6b-v3-int8", +] as const; +export type LocalSttModelId = (typeof LOCAL_STT_MODEL_IDS)[number]; + +export const LOCAL_TTS_MODEL_IDS = [ + "kitten-nano-en-v0_1-fp16", + "kokoro-en-v0_19", + "pocket-tts-onnx-int8", +] as const; +export type LocalTtsModelId = (typeof LOCAL_TTS_MODEL_IDS)[number]; + +const STT_MODEL_ALIASES: Record = { + zipformer: "zipformer-bilingual-zh-en-2023-02-20", + "zipformer-bilingual": "zipformer-bilingual-zh-en-2023-02-20", + paraformer: "paraformer-bilingual-zh-en", + parakeet: "parakeet-tdt-0.6b-v3-int8", + "parakeet-v3": "parakeet-tdt-0.6b-v3-int8", + "parakeet-tdt": "parakeet-tdt-0.6b-v3-int8", +}; + +const TTS_MODEL_ALIASES: Record = { + pocket: "pocket-tts-onnx-int8", + "pocket-tts": "pocket-tts-onnx-int8", + kitten: "kitten-nano-en-v0_1-fp16", + kokoro: "kokoro-en-v0_19", +}; + +export const LocalSttModelIdSchema = z.preprocess((value) => { + if (typeof value !== "string") return value; + const normalized = value.trim().toLowerCase(); + if (!normalized) return value; + return STT_MODEL_ALIASES[normalized] ?? normalized; +}, z.enum(LOCAL_STT_MODEL_IDS)); + +export const LocalTtsModelIdSchema = z.preprocess((value) => { + if (typeof value !== "string") return value; + const normalized = value.trim().toLowerCase(); + if (!normalized) return value; + return TTS_MODEL_ALIASES[normalized] ?? normalized; +}, z.enum(LOCAL_TTS_MODEL_IDS)); + export type SherpaOnnxModelSpec = { id: SherpaOnnxModelId; kind: SherpaOnnxModelKind; diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts index 9c434c4be..68bbe83a6 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts @@ -110,7 +110,7 @@ export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOption if (!options.autoDownload) { throw new Error( `Missing sherpa-onnx model files for ${options.modelId} in ${modelDir}. ` + - `Set PASEO_SHERPA_ONNX_AUTO_DOWNLOAD=1 to auto-download.` + `Set PASEO_LOCAL_AUTO_DOWNLOAD=1 to auto-download.` ); } diff --git a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts index 8d66efdc5..01bf369ea 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts @@ -96,7 +96,7 @@ describe("speech models (download E2E)", () => { const set = getModelSet(); const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-")); - const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "sherpa-onnx"); + const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "local-speech"); const modelIds: SherpaOnnxModelId[] = set === "parakeet-pocket" @@ -117,12 +117,20 @@ describe("speech models (download E2E)", () => { dictationSttProvider: "local", voiceSttProvider: "local", voiceTtsProvider: "local", - sherpaOnnx: { + local: { modelsDir, autoDownload: false, - stt: { preset: set === "parakeet-pocket" ? "parakeet-tdt-0.6b-v3-int8" : "zipformer-bilingual-zh-en-2023-02-20" }, - tts: { preset: set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16" }, }, + dictationLocalSttModel: + set === "parakeet-pocket" + ? "parakeet-tdt-0.6b-v3-int8" + : "zipformer-bilingual-zh-en-2023-02-20", + voiceLocalSttModel: + set === "parakeet-pocket" + ? "parakeet-tdt-0.6b-v3-int8" + : "zipformer-bilingual-zh-en-2023-02-20", + voiceLocalTtsModel: + set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16", }, }); diff --git a/packages/server/src/server/speech/speech-config-resolver.test.ts b/packages/server/src/server/speech/speech-config-resolver.test.ts new file mode 100644 index 000000000..bb34fdb01 --- /dev/null +++ b/packages/server/src/server/speech/speech-config-resolver.test.ts @@ -0,0 +1,94 @@ +import path from "node:path"; + +import { describe, expect, test } from "vitest"; + +import { PersistedConfigSchema } from "../persisted-config.js"; +import { resolveSpeechConfig } from "./speech-config-resolver.js"; + +describe("resolveSpeechConfig", () => { + test("resolves local-first defaults without env overrides", () => { + const paseoHome = "/tmp/paseo-home"; + const persisted = PersistedConfigSchema.parse({}); + const env = {} as NodeJS.ProcessEnv; + + const result = resolveSpeechConfig({ + paseoHome, + env, + persisted, + }); + + expect(result.openai).toBeUndefined(); + expect(result.speech.dictationSttProvider).toBe("local"); + expect(result.speech.voiceSttProvider).toBe("local"); + expect(result.speech.voiceTtsProvider).toBe("local"); + expect(result.speech.local).toEqual({ + modelsDir: path.join(paseoHome, "models", "local-speech"), + autoDownload: true, + }); + expect(result.speech.dictationLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.voiceLocalTtsModel).toBe("pocket-tts-onnx-int8"); + }); + + test("resolves feature-scoped local model env vars", () => { + const persisted = PersistedConfigSchema.parse({ + features: { + voiceMode: { + stt: { provider: "openai", model: "gpt-4o-transcribe" }, + }, + }, + providers: { + openai: { apiKey: "persisted-key" }, + }, + }); + const env = { + PASEO_DICTATION_LOCAL_STT_MODEL: "zipformer", + PASEO_VOICE_LOCAL_STT_MODEL: "parakeet", + PASEO_VOICE_LOCAL_TTS_MODEL: "kitten", + PASEO_VOICE_LOCAL_TTS_SPEAKER_ID: "5", + PASEO_VOICE_LOCAL_TTS_SPEED: "1.35", + PASEO_LOCAL_MODELS_DIR: "/tmp/models", + PASEO_LOCAL_AUTO_DOWNLOAD: "0", + OPENAI_API_KEY: "env-key", + PASEO_VOICE_STT_PROVIDER: "openai", + PASEO_DICTATION_STT_PROVIDER: "local", + PASEO_VOICE_TTS_PROVIDER: "local", + } as NodeJS.ProcessEnv; + + const result = resolveSpeechConfig({ + paseoHome: "/tmp/paseo-home", + env, + persisted, + }); + + expect(result.speech.local).toEqual({ + modelsDir: "/tmp/models", + autoDownload: false, + }); + expect(result.speech.dictationLocalSttModel).toBe("zipformer-bilingual-zh-en-2023-02-20"); + expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.voiceLocalTtsModel).toBe("kitten-nano-en-v0_1-fp16"); + expect(result.speech.voiceLocalTtsSpeakerId).toBe(5); + expect(result.speech.voiceLocalTtsSpeed).toBe(1.35); + expect(result.openai?.apiKey).toBe("env-key"); + expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe"); + }); + + test("ignores deprecated shared local model env vars", () => { + const persisted = PersistedConfigSchema.parse({}); + const env = { + PASEO_LOCAL_STT_MODEL: "zipformer-bilingual-zh-en-2023-02-20", + PASEO_LOCAL_TTS_MODEL: "kitten-nano-en-v0_1-fp16", + } as NodeJS.ProcessEnv; + + const result = resolveSpeechConfig({ + paseoHome: "/tmp/paseo-home", + env, + persisted, + }); + + expect(result.speech.dictationLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.voiceLocalSttModel).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.voiceLocalTtsModel).toBe("pocket-tts-onnx-int8"); + }); +}); diff --git a/packages/server/src/server/speech/speech-config-resolver.ts b/packages/server/src/server/speech/speech-config-resolver.ts new file mode 100644 index 000000000..964f5b8df --- /dev/null +++ b/packages/server/src/server/speech/speech-config-resolver.ts @@ -0,0 +1,196 @@ +import path from "node:path"; + +import type { STTConfig } from "./providers/openai/stt.js"; +import type { TTSConfig } from "./providers/openai/tts.js"; +import type { PersistedConfig } from "../persisted-config.js"; +import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js"; +import { LocalSttModelIdSchema, LocalTtsModelIdSchema } from "./providers/local/sherpa/model-catalog.js"; + +const DEFAULT_LOCAL_STT_MODEL = "parakeet-tdt-0.6b-v3-int8"; +const DEFAULT_LOCAL_TTS_MODEL = "pocket-tts-onnx-int8"; +const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech"); +const DEFAULT_OPENAI_TTS_MODEL: TTSConfig["model"] = "tts-1"; +const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe"; + +type SpeechProviderId = "openai" | "local"; + +function parseSpeechProviderId(value: unknown): SpeechProviderId | null { + if (typeof value !== "string") return null; + const normalized = value.trim().toLowerCase(); + if (!normalized) return null; + if (normalized === "openai") return "openai"; + if (normalized === "local") return "local"; + return null; +} + +function parseBooleanFlag(value: string | undefined): boolean | null { + if (value === undefined) return null; + const normalized = value.trim().toLowerCase(); + if (normalized === "1" || normalized === "true" || normalized === "yes") return true; + if (normalized === "0" || normalized === "false" || normalized === "no") return false; + return null; +} + +function parseNumberOrUndefined(value: string | undefined): number | undefined { + if (value === undefined) return undefined; + const parsed = Number.parseFloat(value); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function parseIntOrUndefined(value: string | undefined): number | undefined { + if (value === undefined) return undefined; + const parsed = Number.parseInt(value, 10); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function parseOpenAiConfig( + env: NodeJS.ProcessEnv, + persisted: PersistedConfig, + providers: { + dictationSttProvider: SpeechProviderId; + voiceSttProvider: SpeechProviderId; + voiceTtsProvider: SpeechProviderId; + } +): PaseoOpenAIConfig | undefined { + const apiKey = env.OPENAI_API_KEY ?? persisted.providers?.openai?.apiKey; + if (!apiKey) return undefined; + + const sttConfidenceThreshold = parseNumberOrUndefined(env.STT_CONFIDENCE_THRESHOLD) + ?? persisted.features?.dictation?.stt?.confidenceThreshold; + + const sttModel = ( + env.STT_MODEL + ?? (providers.voiceSttProvider === "openai" ? persisted.features?.voiceMode?.stt?.model : undefined) + ?? (providers.dictationSttProvider === "openai" ? persisted.features?.dictation?.stt?.model : undefined) + ) as STTConfig["model"] | undefined; + + const ttsVoice = ( + env.TTS_VOICE + || (providers.voiceTtsProvider === "openai" ? persisted.features?.voiceMode?.tts?.voice : undefined) + || "alloy" + ) as TTSConfig["voice"]; + const ttsModelRaw = + env.TTS_MODEL + || (providers.voiceTtsProvider === "openai" ? persisted.features?.voiceMode?.tts?.model : undefined) + || DEFAULT_OPENAI_TTS_MODEL; + const ttsModel: TTSConfig["model"] = + ttsModelRaw === "tts-1" || ttsModelRaw === "tts-1-hd" ? ttsModelRaw : DEFAULT_OPENAI_TTS_MODEL; + + const realtimeTranscriptionModel = + env.OPENAI_REALTIME_TRANSCRIPTION_MODEL + || (providers.dictationSttProvider === "openai" + ? persisted.features?.dictation?.stt?.model + : undefined) + || DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL; + + return { + apiKey, + stt: { + apiKey, + ...(sttConfidenceThreshold !== undefined ? { confidenceThreshold: sttConfidenceThreshold } : {}), + ...(sttModel ? { model: sttModel } : {}), + }, + tts: { + apiKey, + voice: ttsVoice, + model: ttsModel, + responseFormat: "pcm", + }, + realtimeTranscriptionModel, + }; +} + +export function resolveSpeechConfig(params: { + paseoHome: string; + env: NodeJS.ProcessEnv; + persisted: PersistedConfig; +}): { + openai: PaseoOpenAIConfig | undefined; + speech: PaseoSpeechConfig; +} { + const { paseoHome, env, persisted } = params; + + const dictationSttProvider = + parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER) + ?? parseSpeechProviderId(persisted.features?.dictation?.stt?.provider) + ?? "local"; + + const voiceSttProvider = + parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER) + ?? parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider) + ?? "local"; + + const voiceTtsProvider = + parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER) + ?? parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider) + ?? "local"; + + const anyLocalRequested = + dictationSttProvider === "local" || + voiceSttProvider === "local" || + voiceTtsProvider === "local" || + env.PASEO_LOCAL_MODELS_DIR !== undefined || + persisted.providers?.local !== undefined; + + const localModelsDir = + env.PASEO_LOCAL_MODELS_DIR + ?? persisted.providers?.local?.modelsDir + ?? path.join(paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR); + + const localAutoDownload = + parseBooleanFlag(env.PASEO_LOCAL_AUTO_DOWNLOAD) + ?? persisted.providers?.local?.autoDownload + ?? true; + + const dictationLocalSttModel = LocalSttModelIdSchema.parse( + env.PASEO_DICTATION_LOCAL_STT_MODEL + ?? persisted.features?.dictation?.stt?.model + ?? DEFAULT_LOCAL_STT_MODEL + ); + + const voiceLocalSttModel = LocalSttModelIdSchema.parse( + env.PASEO_VOICE_LOCAL_STT_MODEL + ?? persisted.features?.voiceMode?.stt?.model + ?? DEFAULT_LOCAL_STT_MODEL + ); + + const voiceLocalTtsModel = LocalTtsModelIdSchema.parse( + env.PASEO_VOICE_LOCAL_TTS_MODEL + ?? persisted.features?.voiceMode?.tts?.model + ?? DEFAULT_LOCAL_TTS_MODEL + ); + + const voiceLocalTtsSpeakerId = + parseIntOrUndefined(env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID) + ?? persisted.features?.voiceMode?.tts?.speakerId; + + const voiceLocalTtsSpeed = + parseNumberOrUndefined(env.PASEO_VOICE_LOCAL_TTS_SPEED) + ?? persisted.features?.voiceMode?.tts?.speed; + + return { + openai: parseOpenAiConfig(env, persisted, { + dictationSttProvider, + voiceSttProvider, + voiceTtsProvider, + }), + speech: { + dictationSttProvider, + voiceSttProvider, + voiceTtsProvider, + ...(anyLocalRequested + ? { + local: { + modelsDir: localModelsDir.trim(), + autoDownload: localAutoDownload, + }, + } + : {}), + dictationLocalSttModel, + voiceLocalSttModel, + voiceLocalTtsModel, + ...(voiceLocalTtsSpeakerId !== undefined ? { voiceLocalTtsSpeakerId } : {}), + ...(voiceLocalTtsSpeed !== undefined ? { voiceLocalTtsSpeed } : {}), + }, + }; +} diff --git a/packages/server/src/server/speech/speech-runtime.ts b/packages/server/src/server/speech/speech-runtime.ts new file mode 100644 index 000000000..0349b8291 --- /dev/null +++ b/packages/server/src/server/speech/speech-runtime.ts @@ -0,0 +1,529 @@ +import type { Logger } from "pino"; + +import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js"; +import { + OpenAISTT, +} from "./providers/openai/stt.js"; +import { + OpenAITTS, +} from "./providers/openai/tts.js"; +import { OpenAIRealtimeTranscriptionSession } from "./providers/openai/realtime-transcription-session.js"; +import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js"; +import { SherpaOnlineRecognizerEngine } from "./providers/local/sherpa/sherpa-online-recognizer.js"; +import { SherpaOfflineRecognizerEngine } from "./providers/local/sherpa/sherpa-offline-recognizer.js"; +import { SherpaOnnxSTT } from "./providers/local/sherpa/sherpa-stt.js"; +import { SherpaOnnxParakeetSTT } from "./providers/local/sherpa/sherpa-parakeet-stt.js"; +import { SherpaOnnxTTS } from "./providers/local/sherpa/sherpa-tts.js"; +import { SherpaRealtimeTranscriptionSession } from "./providers/local/sherpa/sherpa-realtime-session.js"; +import { SherpaParakeetRealtimeTranscriptionSession } from "./providers/local/sherpa/sherpa-parakeet-realtime-session.js"; +import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./providers/local/sherpa/model-downloader.js"; +import { + LocalSttModelIdSchema, + LocalTtsModelIdSchema, + type LocalSttModelId, + type LocalTtsModelId, + type SherpaOnnxModelId, +} from "./providers/local/sherpa/model-catalog.js"; +import { PocketTtsOnnxTTS } from "./providers/local/pocket/pocket-tts-onnx.js"; + +type SpeechProviderId = "openai" | "local"; + +const DEFAULT_LOCAL_STT_MODEL = "parakeet-tdt-0.6b-v3-int8"; +const DEFAULT_LOCAL_TTS_MODEL = "pocket-tts-onnx-int8"; +const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe"; + +type LocalSttEngine = + | { kind: "offline"; engine: SherpaOfflineRecognizerEngine } + | { kind: "online"; engine: SherpaOnlineRecognizerEngine }; + +function buildModelDownloadHint(modelsDir: string, modelId: SherpaOnnxModelId): string { + return `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${modelsDir}' --model '${modelId}'`; +} + +function resolveSpeechProviders( + speechConfig: PaseoSpeechConfig | null +): { + dictationSttProvider: SpeechProviderId; + voiceSttProvider: SpeechProviderId; + voiceTtsProvider: SpeechProviderId; +} { + return { + dictationSttProvider: speechConfig?.dictationSttProvider ?? "local", + voiceSttProvider: speechConfig?.voiceSttProvider ?? "local", + voiceTtsProvider: speechConfig?.voiceTtsProvider ?? "local", + }; +} + +function resolveLocalModels( + speechConfig: PaseoSpeechConfig | null +): { + dictationLocalSttModel: LocalSttModelId; + voiceLocalSttModel: LocalSttModelId; + voiceLocalTtsModel: LocalTtsModelId; +} { + return { + dictationLocalSttModel: LocalSttModelIdSchema.parse( + speechConfig?.dictationLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL + ), + voiceLocalSttModel: LocalSttModelIdSchema.parse( + speechConfig?.voiceLocalSttModel ?? DEFAULT_LOCAL_STT_MODEL + ), + voiceLocalTtsModel: LocalTtsModelIdSchema.parse( + speechConfig?.voiceLocalTtsModel ?? DEFAULT_LOCAL_TTS_MODEL + ), + }; +} + +function computeDefaultLocalModelIds(params: { + providers: { + dictationSttProvider: SpeechProviderId; + voiceSttProvider: SpeechProviderId; + voiceTtsProvider: SpeechProviderId; + }; + models: { + dictationLocalSttModel: SherpaOnnxModelId; + voiceLocalSttModel: SherpaOnnxModelId; + voiceLocalTtsModel: SherpaOnnxModelId; + }; +}): SherpaOnnxModelId[] { + const ids = new Set(); + if (params.providers.dictationSttProvider === "local") { + ids.add(params.models.dictationLocalSttModel); + } + if (params.providers.voiceSttProvider === "local") { + ids.add(params.models.voiceLocalSttModel); + } + if (params.providers.voiceTtsProvider === "local") { + ids.add(params.models.voiceLocalTtsModel); + } + return Array.from(ids); +} + +async function createLocalSttEngine(params: { + modelId: LocalSttModelId; + modelsDir: string; + logger: Logger; +}): Promise { + const { modelId, modelsDir, logger } = params; + + if (modelId === "parakeet-tdt-0.6b-v3-int8") { + const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); + return { + kind: "offline", + engine: new SherpaOfflineRecognizerEngine( + { + model: { + kind: "nemo_transducer", + encoder: `${modelDir}/encoder.int8.onnx`, + decoder: `${modelDir}/decoder.int8.onnx`, + joiner: `${modelDir}/joiner.int8.onnx`, + tokens: `${modelDir}/tokens.txt`, + }, + numThreads: 2, + debug: 0, + }, + logger + ), + }; + } + + if (modelId === "paraformer-bilingual-zh-en") { + const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); + return { + kind: "online", + engine: new SherpaOnlineRecognizerEngine( + { + model: { + kind: "paraformer", + encoder: `${modelDir}/encoder.int8.onnx`, + decoder: `${modelDir}/decoder.int8.onnx`, + tokens: `${modelDir}/tokens.txt`, + }, + numThreads: 1, + debug: 0, + }, + logger + ), + }; + } + + if (modelId === "zipformer-bilingual-zh-en-2023-02-20") { + const modelDir = getSherpaOnnxModelDir(modelsDir, modelId); + return { + kind: "online", + engine: new SherpaOnlineRecognizerEngine( + { + model: { + kind: "transducer", + encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`, + decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`, + joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`, + tokens: `${modelDir}/tokens.txt`, + modelType: "zipformer", + }, + numThreads: 1, + debug: 0, + }, + logger + ), + }; + } + + throw new Error(`Unsupported local STT model '${modelId}'`); +} + +export type InitializedSpeechRuntime = { + sttService: SpeechToTextProvider | null; + ttsService: TextToSpeechProvider | null; + dictationSttService: SpeechToTextProvider | null; + cleanup: () => void; + localModelConfig: { + modelsDir: string; + defaultModelIds: SherpaOnnxModelId[]; + } | null; +}; + +export async function initializeSpeechRuntime(params: { + logger: Logger; + openaiConfig?: PaseoOpenAIConfig; + speechConfig?: PaseoSpeechConfig; +}): Promise { + const logger = params.logger; + const speechConfig = params.speechConfig ?? null; + const localConfig = speechConfig?.local ?? null; + const openaiConfig = params.openaiConfig; + + const providers = resolveSpeechProviders(speechConfig); + const localModels = resolveLocalModels(speechConfig); + + const wantsLocalDictation = providers.dictationSttProvider === "local"; + const wantsLocalVoiceStt = providers.voiceSttProvider === "local"; + const wantsLocalVoiceTts = providers.voiceTtsProvider === "local"; + + const openaiApiKey = openaiConfig?.apiKey; + const openaiSttApiKey = openaiConfig?.stt?.apiKey ?? openaiApiKey; + const openaiTtsApiKey = openaiConfig?.tts?.apiKey ?? openaiApiKey; + const openaiDictationApiKey = openaiApiKey; + + const missingOpenAiCredentialsFor: string[] = []; + if (providers.voiceSttProvider === "openai" && !openaiSttApiKey) { + missingOpenAiCredentialsFor.push("voice.stt"); + } + if (providers.voiceTtsProvider === "openai" && !openaiTtsApiKey) { + missingOpenAiCredentialsFor.push("voice.tts"); + } + if (providers.dictationSttProvider === "openai" && !openaiDictationApiKey) { + missingOpenAiCredentialsFor.push("dictation.stt"); + } + if (missingOpenAiCredentialsFor.length > 0) { + logger.error( + { + requestedProviders: { + dictationStt: providers.dictationSttProvider, + voiceStt: providers.voiceSttProvider, + voiceTts: providers.voiceTtsProvider, + }, + missingOpenAiCredentialsFor, + }, + "Invalid speech configuration: OpenAI provider selected but credentials are missing" + ); + throw new Error( + `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}` + ); + } + + logger.info( + { + requestedProviders: { + dictationStt: providers.dictationSttProvider, + voiceStt: providers.voiceSttProvider, + voiceTts: providers.voiceTtsProvider, + }, + availability: { + openai: { + stt: Boolean(openaiSttApiKey), + tts: Boolean(openaiTtsApiKey), + dictationStt: Boolean(openaiDictationApiKey), + }, + local: { + configured: Boolean(localConfig), + modelsDir: localConfig?.modelsDir ?? null, + autoDownload: localConfig?.autoDownload ?? null, + }, + }, + }, + "Speech provider reconciliation started" + ); + + let sttService: SpeechToTextProvider | null = null; + let ttsService: TextToSpeechProvider | null = null; + let dictationSttService: SpeechToTextProvider | null = null; + let localVoiceTtsProvider: TextToSpeechProvider | null = null; + + const requiredLocalModelIds = computeDefaultLocalModelIds({ + providers, + models: localModels, + }); + + if (requiredLocalModelIds.length > 0 && localConfig) { + try { + logger.info( + { + modelsDir: localConfig.modelsDir, + modelIds: requiredLocalModelIds, + autoDownload: localConfig.autoDownload ?? true, + }, + "Ensuring local speech models" + ); + await ensureSherpaOnnxModels({ + modelsDir: localConfig.modelsDir, + modelIds: requiredLocalModelIds, + autoDownload: localConfig.autoDownload ?? true, + logger, + }); + } catch (err) { + logger.error( + { + err, + modelsDir: localConfig.modelsDir, + modelIds: requiredLocalModelIds, + autoDownload: localConfig.autoDownload ?? true, + hint: + "Run: npm run dev --workspace=@getpaseo/server, then run: " + + "`tsx packages/server/scripts/download-speech-models.ts --models-dir --model `", + }, + "Failed to ensure local speech models" + ); + } + } + + const localSttEngines = new Map(); + const getLocalSttEngine = async ( + modelId: LocalSttModelId + ): Promise => { + const existing = localSttEngines.get(modelId); + if (existing) { + return existing; + } + if (!localConfig) { + return null; + } + try { + const created = await createLocalSttEngine({ + modelId, + modelsDir: localConfig.modelsDir, + logger, + }); + localSttEngines.set(modelId, created); + return created; + } catch (err) { + logger.error( + { + err, + modelsDir: localConfig.modelsDir, + modelId, + hint: buildModelDownloadHint(localConfig.modelsDir, modelId), + }, + "Failed to initialize local STT engine (models missing or invalid)" + ); + return null; + } + }; + + if (wantsLocalVoiceStt) { + if (!localConfig) { + logger.warn( + { configured: false }, + "Local STT selected for voice but local provider config is missing; STT will be unavailable" + ); + } else { + const voiceEngine = await getLocalSttEngine(localModels.voiceLocalSttModel); + if (voiceEngine?.kind === "offline") { + sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger); + } else if (voiceEngine?.kind === "online") { + sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger); + } + } + } + + if (wantsLocalDictation) { + if (!localConfig) { + logger.warn( + { configured: false }, + "Local STT selected for dictation but local provider config is missing; dictation STT will be unavailable" + ); + } else { + const dictationEngine = await getLocalSttEngine(localModels.dictationLocalSttModel); + if (dictationEngine?.kind === "offline") { + dictationSttService = { + id: "local", + createSession: () => + new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }), + }; + } else if (dictationEngine?.kind === "online") { + dictationSttService = { + id: "local", + createSession: () => new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }), + }; + } + } + } + + if (wantsLocalVoiceTts) { + if (!localConfig) { + logger.warn( + { configured: false }, + "Local TTS selected for voice but local provider config is missing; TTS will be unavailable" + ); + } else { + try { + if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") { + const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); + localVoiceTtsProvider = await PocketTtsOnnxTTS.create( + { + modelDir, + precision: "int8", + targetChunkMs: 50, + }, + logger + ); + } else { + const modelDir = getSherpaOnnxModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); + localVoiceTtsProvider = new SherpaOnnxTTS( + { + preset: localModels.voiceLocalTtsModel, + modelDir, + speakerId: speechConfig?.voiceLocalTtsSpeakerId, + speed: speechConfig?.voiceLocalTtsSpeed, + }, + logger + ); + } + ttsService = localVoiceTtsProvider; + } catch (err) { + logger.error( + { + err, + modelsDir: localConfig.modelsDir, + modelId: localModels.voiceLocalTtsModel, + hint: buildModelDownloadHint(localConfig.modelsDir, localModels.voiceLocalTtsModel), + }, + "Failed to initialize local TTS engine (models missing or invalid)" + ); + } + } + } + + const needsOpenAiStt = !sttService && providers.voiceSttProvider === "openai"; + const needsOpenAiTts = !ttsService && providers.voiceTtsProvider === "openai"; + const needsOpenAiDictation = !dictationSttService && providers.dictationSttProvider === "openai"; + + if ( + (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) && + (openaiSttApiKey || openaiTtsApiKey || openaiDictationApiKey) + ) { + logger.info("OpenAI speech provider initialized"); + + if (needsOpenAiStt && openaiSttApiKey) { + const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {}; + sttService = new OpenAISTT( + { + apiKey: openaiSttApiKey, + ...sttConfig, + }, + logger + ); + } + + if (needsOpenAiTts && openaiTtsApiKey) { + const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {}; + ttsService = new OpenAITTS( + { + apiKey: openaiTtsApiKey, + voice: "alloy", + model: "tts-1", + responseFormat: "pcm", + ...ttsConfig, + }, + logger + ); + } + + if (needsOpenAiDictation && openaiDictationApiKey) { + dictationSttService = { + id: "openai", + createSession: ({ logger: sessionLogger, language, prompt }) => + new OpenAIRealtimeTranscriptionSession({ + apiKey: openaiDictationApiKey, + logger: sessionLogger, + transcriptionModel: + openaiConfig?.realtimeTranscriptionModel + ?? DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, + ...(language ? { language } : {}), + ...(prompt ? { prompt } : {}), + turnDetection: null, + }), + }; + } + } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) { + logger.warn("OpenAI speech providers are configured but credentials are missing"); + } + + const effectiveProviders = { + dictationStt: dictationSttService?.id ?? "unavailable", + voiceStt: sttService?.id ?? "unavailable", + voiceTts: !ttsService ? "unavailable" : ttsService === localVoiceTtsProvider ? "local" : "openai", + }; + const unavailableFeatures = [ + !dictationSttService ? "dictation.stt" : null, + !sttService ? "voice.stt" : null, + !ttsService ? "voice.tts" : null, + ].filter((feature): feature is string => feature !== null); + + if (unavailableFeatures.length > 0) { + logger.error( + { + requestedProviders: { + dictationStt: providers.dictationSttProvider, + voiceStt: providers.voiceSttProvider, + voiceTts: providers.voiceTtsProvider, + }, + effectiveProviders, + unavailableFeatures, + }, + "Speech provider reconciliation failed: configured features are unavailable" + ); + throw new Error(`Configured speech features unavailable: ${unavailableFeatures.join(", ")}`); + } + + logger.info( + { + effectiveProviders, + }, + "Speech provider reconciliation completed" + ); + + const cleanup = () => { + const maybeFreeable = localVoiceTtsProvider as unknown as { free?: () => void } | null; + if (typeof maybeFreeable?.free === "function") { + maybeFreeable.free(); + } + for (const engine of localSttEngines.values()) { + engine.engine.free(); + } + }; + + return { + sttService, + ttsService, + dictationSttService, + cleanup, + localModelConfig: + localConfig + ? { + modelsDir: localConfig.modelsDir, + defaultModelIds: requiredLocalModelIds, + } + : null, + }; +} diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 39b147368..26d38a074 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -20,6 +20,7 @@ import type { AgentProvider } from "./agent/agent-sdk-types.js"; import { PushTokenStore } from "./push/token-store.js"; import { PushService } from "./push/push-service.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; +import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js"; export type AgentMcpTransportFactory = () => Promise; type VoiceMcpStdioConfig = { @@ -76,6 +77,10 @@ export class VoiceAssistantWebSocketServer { private readonly dictation: { finalTimeoutMs?: number; stt?: SpeechToTextProvider | null; + localModels?: { + modelsDir: string; + defaultModelIds: SherpaOnnxModelId[]; + }; } | null; private readonly voice: { voiceLlmProvider?: AgentProvider | null; @@ -120,6 +125,10 @@ export class VoiceAssistantWebSocketServer { dictation?: { finalTimeoutMs?: number; stt?: SpeechToTextProvider | null; + localModels?: { + modelsDir: string; + defaultModelIds: SherpaOnnxModelId[]; + }; } ) { this.logger = logger.child({ module: "websocket-server" }); diff --git a/packages/website/public/schemas/paseo.config.v1.json b/packages/website/public/schemas/paseo.config.v1.json index 116daee60..cb853f22c 100644 --- a/packages/website/public/schemas/paseo.config.v1.json +++ b/packages/website/public/schemas/paseo.config.v1.json @@ -86,7 +86,7 @@ }, "additionalProperties": false }, - "sherpaOnnx": { + "local": { "type": "object", "properties": { "modelsDir": { @@ -95,32 +95,6 @@ }, "autoDownload": { "type": "boolean" - }, - "stt": { - "type": "object", - "properties": { - "preset": { - "type": "string", - "minLength": 1 - } - }, - "additionalProperties": false - }, - "tts": { - "type": "object", - "properties": { - "preset": { - "type": "string", - "minLength": 1 - }, - "speakerId": { - "type": "number" - }, - "speed": { - "type": "number" - } - }, - "additionalProperties": false } }, "additionalProperties": false @@ -207,10 +181,7 @@ }, "model": { "type": "string", - "enum": [ - "tts-1", - "tts-1-hd" - ] + "minLength": 1 }, "voice": { "type": "string", diff --git a/packages/website/src/routeTree.gen.ts b/packages/website/src/routeTree.gen.ts index 6c8178098..f02250c52 100644 --- a/packages/website/src/routeTree.gen.ts +++ b/packages/website/src/routeTree.gen.ts @@ -13,6 +13,7 @@ import { Route as DocsRouteImport } from './routes/docs' import { Route as IndexRouteImport } from './routes/index' import { Route as DocsIndexRouteImport } from './routes/docs/index' import { Route as DocsWorktreesRouteImport } from './routes/docs/worktrees' +import { Route as DocsVoiceRouteImport } from './routes/docs/voice' import { Route as DocsSecurityRouteImport } from './routes/docs/security' import { Route as DocsConfigurationRouteImport } from './routes/docs/configuration' import { Route as DocsCliRouteImport } from './routes/docs/cli' @@ -38,6 +39,11 @@ const DocsWorktreesRoute = DocsWorktreesRouteImport.update({ path: '/worktrees', getParentRoute: () => DocsRoute, } as any) +const DocsVoiceRoute = DocsVoiceRouteImport.update({ + id: '/voice', + path: '/voice', + getParentRoute: () => DocsRoute, +} as any) const DocsSecurityRoute = DocsSecurityRouteImport.update({ id: '/security', path: '/security', @@ -66,6 +72,7 @@ export interface FileRoutesByFullPath { '/docs/cli': typeof DocsCliRoute '/docs/configuration': typeof DocsConfigurationRoute '/docs/security': typeof DocsSecurityRoute + '/docs/voice': typeof DocsVoiceRoute '/docs/worktrees': typeof DocsWorktreesRoute '/docs/': typeof DocsIndexRoute } @@ -75,6 +82,7 @@ export interface FileRoutesByTo { '/docs/cli': typeof DocsCliRoute '/docs/configuration': typeof DocsConfigurationRoute '/docs/security': typeof DocsSecurityRoute + '/docs/voice': typeof DocsVoiceRoute '/docs/worktrees': typeof DocsWorktreesRoute '/docs': typeof DocsIndexRoute } @@ -86,6 +94,7 @@ export interface FileRoutesById { '/docs/cli': typeof DocsCliRoute '/docs/configuration': typeof DocsConfigurationRoute '/docs/security': typeof DocsSecurityRoute + '/docs/voice': typeof DocsVoiceRoute '/docs/worktrees': typeof DocsWorktreesRoute '/docs/': typeof DocsIndexRoute } @@ -98,6 +107,7 @@ export interface FileRouteTypes { | '/docs/cli' | '/docs/configuration' | '/docs/security' + | '/docs/voice' | '/docs/worktrees' | '/docs/' fileRoutesByTo: FileRoutesByTo @@ -107,6 +117,7 @@ export interface FileRouteTypes { | '/docs/cli' | '/docs/configuration' | '/docs/security' + | '/docs/voice' | '/docs/worktrees' | '/docs' id: @@ -117,6 +128,7 @@ export interface FileRouteTypes { | '/docs/cli' | '/docs/configuration' | '/docs/security' + | '/docs/voice' | '/docs/worktrees' | '/docs/' fileRoutesById: FileRoutesById @@ -156,6 +168,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof DocsWorktreesRouteImport parentRoute: typeof DocsRoute } + '/docs/voice': { + id: '/docs/voice' + path: '/voice' + fullPath: '/docs/voice' + preLoaderRoute: typeof DocsVoiceRouteImport + parentRoute: typeof DocsRoute + } '/docs/security': { id: '/docs/security' path: '/security' @@ -192,6 +211,7 @@ interface DocsRouteChildren { DocsCliRoute: typeof DocsCliRoute DocsConfigurationRoute: typeof DocsConfigurationRoute DocsSecurityRoute: typeof DocsSecurityRoute + DocsVoiceRoute: typeof DocsVoiceRoute DocsWorktreesRoute: typeof DocsWorktreesRoute DocsIndexRoute: typeof DocsIndexRoute } @@ -201,6 +221,7 @@ const DocsRouteChildren: DocsRouteChildren = { DocsCliRoute: DocsCliRoute, DocsConfigurationRoute: DocsConfigurationRoute, DocsSecurityRoute: DocsSecurityRoute, + DocsVoiceRoute: DocsVoiceRoute, DocsWorktreesRoute: DocsWorktreesRoute, DocsIndexRoute: DocsIndexRoute, } diff --git a/packages/website/src/routes/docs.tsx b/packages/website/src/routes/docs.tsx index d393104b2..dfd820c29 100644 --- a/packages/website/src/routes/docs.tsx +++ b/packages/website/src/routes/docs.tsx @@ -7,6 +7,7 @@ export const Route = createFileRoute('/docs')({ const navigation = [ { name: 'Getting started', href: '/docs' }, + { name: 'Voice', href: '/docs/voice' }, { name: 'Git worktrees', href: '/docs/worktrees' }, { name: 'CLI', href: '/docs/cli' }, { name: 'Configuration', href: '/docs/configuration' }, diff --git a/packages/website/src/routes/docs/configuration.tsx b/packages/website/src/routes/docs/configuration.tsx index 169287660..d303acfbe 100644 --- a/packages/website/src/routes/docs/configuration.tsx +++ b/packages/website/src/routes/docs/configuration.tsx @@ -81,75 +81,13 @@ function Configuration() {

    Voice

    - Voice is provider-based per feature: dictation STT, realtime voice STT, and voice TTS can each - use local or openai. - Defaults are local for all three. -

    - 
    -{`{
-  "version": 1,
-  "features": {
-    "dictation": { "stt": { "provider": "local" } },
-    "voiceMode": {
-      "llm": { "provider": "claude", "model": "haiku" },
-      "stt": { "provider": "local" },
-      "tts": { "provider": "local" }
-    }
-  },
-  "providers": {
-    "sherpaOnnx": {
-      "modelsDir": "~/.paseo/models/sherpa-onnx",
-      "autoDownload": true,
-      "stt": { "preset": "parakeet-tdt-0.6b-v3-int8" },
-      "tts": { "preset": "pocket-tts-onnx-int8" }
-    }
-  }
-}`}
-
    -

    - Voice LLM orchestration is agents-only. Set{' '} - features.voiceMode.llm.provider to one of{' '} - claude, codex, or{' '} - opencode. If set, startup is strict and fails if unavailable. - If omitted, Paseo picks the first available voice-enabled agent provider. + Voice is configured through features.dictation and{' '} + features.voiceMode, with provider credentials under{' '} + providers.

    - Local voice uses ONNX models (sherpa-onnx + PocketTTS). Default presets are - parakeet-tdt-0.6b-v3-int8 for STT and - pocket-tts-onnx-int8 for TTS. -

    -

    - With local provider enabled and auto-download on (default outside tests), missing model files are - downloaded at daemon startup into $PASEO_HOME/models/sherpa-onnx. - Downloads are only performed for missing files. -

    -

    - Local ONNX execution is CPU-first by default. It works without GPU requirements, but performance - depends on your machine. -

    -

    - To force OpenAI for voice/dictation, configure OpenAI providers explicitly and provide - OPENAI_API_KEY. If OpenAI is selected but credentials are - missing, daemon startup fails fast. -

    - 
    -{`{
-  "version": 1,
-  "features": {
-    "dictation": { "stt": { "provider": "openai" } },
-    "voiceMode": {
-      "stt": { "provider": "openai" },
-      "tts": { "provider": "openai" }
-    }
-  },
-  "providers": {
-    "openai": { "apiKey": "..." }
-  }
-}`}
-
    -

    - Realtime voice can create/manage coding agents. Use clear prompts and explicit paths, because - those agents can be launched in arbitrary working directories by request. + For voice philosophy, architecture, and complete local/OpenAI setup examples, see{' '} + Voice docs.

    @@ -162,8 +100,10 @@ function Configuration() {
  • OPENAI_API_KEY — override OpenAI provider key
  • PASEO_VOICE_LLM_PROVIDER — override voice LLM provider (claude, codex, opencode)
  • PASEO_DICTATION_STT_PROVIDER, PASEO_VOICE_STT_PROVIDER, PASEO_VOICE_TTS_PROVIDER — override voice provider selection (local or openai)
    • -
  • PASEO_SHERPA_ONNX_MODELS_DIR and PASEO_SHERPA_ONNX_AUTO_DOWNLOAD — control local model directory and download behavior
    • -
  • PASEO_SHERPA_STT_PRESET and PASEO_SHERPA_TTS_PRESET — override local STT/TTS model presets
    • +
  • PASEO_LOCAL_MODELS_DIR and PASEO_LOCAL_AUTO_DOWNLOAD — control local model directory and download behavior
    • +
  • PASEO_DICTATION_LOCAL_STT_MODEL — override local dictation STT model
    • +
  • PASEO_VOICE_LOCAL_STT_MODEL, PASEO_VOICE_LOCAL_TTS_MODEL — override local voice STT/TTS models
    • +
  • PASEO_VOICE_LOCAL_TTS_SPEAKER_ID, PASEO_VOICE_LOCAL_TTS_SPEED — optional local voice TTS tuning
    • diff --git a/packages/website/src/routes/docs/index.tsx b/packages/website/src/routes/docs/index.tsx index afd9cbbd9..69dcfe5ec 100644 --- a/packages/website/src/routes/docs/index.tsx +++ b/packages/website/src/routes/docs/index.tsx @@ -87,22 +87,25 @@ function GettingStarted() {

    Voice Setup

    - Voice supports two providers: local (default) and OpenAI. Local uses ONNX models and will - download missing model files automatically at daemon startup. + Paseo includes first-class voice support with a local-first architecture and configurable speech + providers.

    -
    - $ - export OPENAI_API_KEY=your-key-here -

    - Set OPENAI_API_KEY if you want OpenAI speech providers. - For local defaults and model presets, see Configuration. + For architecture, local model behavior, and provider configuration, see the Voice docs page.

    + + Voice docs +

    Next