diff --git a/package-lock.json b/package-lock.json index 4ec5f45a0..3c6ed89a7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4688,17 +4688,6 @@ "version": "1.1.12", "license": "MIT" }, - "node_modules/@openrouter/ai-sdk-provider": { - "version": "1.2.0", - "license": "Apache-2.0", - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "ai": "^5.0.0", - "zod": "^3.24.1 || ^v4" - } - }, "node_modules/@opentelemetry/api": { "version": "1.9.0", "license": "Apache-2.0", @@ -20286,7 +20275,6 @@ "@lezer/python": "^1.1.18", "@modelcontextprotocol/sdk": "^1.20.1", "@opencode-ai/sdk": "^1.1.12", - "@openrouter/ai-sdk-provider": "^1.2.0", "@sctg/sentencepiece-js": "^1.1.0", "@xterm/headless": "^6.0.0", "ai": "^5.0.76", diff --git a/packages/app/src/contexts/voice-context.tsx b/packages/app/src/contexts/voice-context.tsx index 83fa5deee..9fcf233e7 100644 --- a/packages/app/src/contexts/voice-context.tsx +++ b/packages/app/src/contexts/voice-context.tsx @@ -2,12 +2,13 @@ import { createContext, useContext, useState, ReactNode, useCallback, useEffect, import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio"; import type { SessionState } from "@/stores/session-store"; import { useSessionStore } from "@/stores/session-store"; -import AsyncStorage from "@react-native-async-storage/async-storage"; -import { randomUUID } from "expo-crypto"; import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; -const VOICE_CONVERSATION_ID_STORAGE_KEY = "@paseo:voice-conversation-id"; const KEEP_AWAKE_TAG = "paseo:voice"; +const VOICE_VAD_VOLUME_THRESHOLD = 0.18; +const VOICE_VAD_SILENCE_DURATION_MS = 1400; +const VOICE_VAD_SPEECH_CONFIRMATION_MS = 120; +const VOICE_VAD_DETECTION_GRACE_PERIOD_MS = 700; interface VoiceContextValue { isVoiceMode: boolean; @@ -123,10 +124,10 @@ export function VoiceProvider({ children }: VoiceProviderProps) { console.error("[Voice] Cannot handle error - setMessages not available from SessionState"); } }, - volumeThreshold: 0.3, - silenceDuration: 2000, - speechConfirmationDuration: 300, - detectionGracePeriod: 200, + volumeThreshold: VOICE_VAD_VOLUME_THRESHOLD, + silenceDuration: VOICE_VAD_SILENCE_DURATION_MS, + speechConfirmationDuration: VOICE_VAD_SPEECH_CONFIRMATION_MS, + detectionGracePeriod: VOICE_VAD_DETECTION_GRACE_PERIOD_MS, }); useEffect(() => { @@ -166,18 +167,9 @@ export function VoiceProvider({ children }: VoiceProviderProps) { console.log("[Voice] Mode enabled"); if (session?.client) { - let voiceConversationId = - (await AsyncStorage.getItem(VOICE_CONVERSATION_ID_STORAGE_KEY)) ?? null; - if (!voiceConversationId) { - voiceConversationId = randomUUID(); - await AsyncStorage.setItem( - VOICE_CONVERSATION_ID_STORAGE_KEY, - voiceConversationId - ); - } - await session.client.setVoiceConversation(true, voiceConversationId); + await session.client.setVoiceMode(true); } else { - console.warn("[Voice] setVoiceConversation skipped: daemon unavailable"); + console.warn("[Voice] setVoiceMode skipped: daemon unavailable"); } } catch (error: any) { console.error("[Voice] Failed to start:", error); @@ -200,9 +192,9 @@ export function VoiceProvider({ children }: VoiceProviderProps) { console.log("[Voice] Mode disabled"); if (session?.client) { - await session.client.setVoiceConversation(false); + await session.client.setVoiceMode(false); } else { - console.warn("[Voice] setVoiceConversation skipped: daemon unavailable"); + console.warn("[Voice] setVoiceMode skipped: daemon unavailable"); } } catch (error: any) { console.error("[Voice] Failed to stop:", error); diff --git a/packages/app/src/voice/speech-segmenter.test.ts b/packages/app/src/voice/speech-segmenter.test.ts index c9b0278a5..a02428a26 100644 --- a/packages/app/src/voice/speech-segmenter.test.ts +++ b/packages/app/src/voice/speech-segmenter.test.ts @@ -88,4 +88,37 @@ describe("SpeechSegmenter", () => { expect(lastCall.isLast).toBe(true); expect(lastCall.audioData.length).toBeGreaterThan(0); }); + + it("keeps detection alive across a brief pause and confirms short utterances", () => { + const onSpeechStart = vi.fn(); + const detectingChanges: boolean[] = []; + + const segmenter = new SpeechSegmenter( + { + enableContinuousStreaming: false, + volumeThreshold: 0.18, + silenceDurationMs: 1400, + speechConfirmationMs: 120, + detectionGracePeriodMs: 700, + minChunkDurationMs: 100, + pcmSampleRate: 1000, + }, + { + onSpeechStart, + onDetectingChange: (v) => detectingChanges.push(v), + } + ); + + const t0 = 20_000; + + segmenter.pushVolumeLevel(0.4, t0); + segmenter.pushPcmChunk(mkPcmBytes(20)); + segmenter.pushVolumeLevel(0.0, t0 + 80); + segmenter.pushPcmChunk(mkPcmBytes(20)); + segmenter.pushVolumeLevel(0.4, t0 + 140); + segmenter.pushPcmChunk(mkPcmBytes(20)); + + expect(detectingChanges).toContain(true); + expect(onSpeechStart).toHaveBeenCalledTimes(1); + }); }); diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index b3222c7c4..b5f2d6c03 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -14,6 +14,7 @@ import { runInspectCommand } from './commands/agent/inspect.js' import { runWaitCommand } from './commands/agent/wait.js' import { runAttachCommand } from './commands/agent/attach.js' import { withOutput } from './output/index.js' +import { runVoiceMcpBridgeCommand } from './commands/voice-mcp-bridge.js' const VERSION = '0.1.0' @@ -141,5 +142,13 @@ export function createCli(): Command { // Worktree commands program.addCommand(createWorktreeCommand()) + // Internal voice MCP stdio bridge command (hidden). + program + .command('__paseo_voice_mcp_bridge', { hidden: true }) + .description('Internal voice MCP bridge command') + .requiredOption('--socket ') + .requiredOption('--caller-agent-id ') + .action(runVoiceMcpBridgeCommand) + return program } diff --git a/packages/cli/src/commands/voice-mcp-bridge.ts b/packages/cli/src/commands/voice-mcp-bridge.ts new file mode 100644 index 000000000..e50887fa7 --- /dev/null +++ b/packages/cli/src/commands/voice-mcp-bridge.ts @@ -0,0 +1,17 @@ +import { runVoiceMcpBridgeCli } from "@getpaseo/server"; + +type VoiceBridgeOptions = { + socket: string; + callerAgentId: string; +}; + +export async function runVoiceMcpBridgeCommand( + options: VoiceBridgeOptions +): Promise { + await runVoiceMcpBridgeCli([ + "--socket", + options.socket, + "--caller-agent-id", + options.callerAgentId, + ]); +} diff --git a/packages/server/package.json b/packages/server/package.json index 88293bf86..dc935208e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -36,7 +36,7 @@ "@lezer/python": "^1.1.18", "@modelcontextprotocol/sdk": "^1.20.1", "@opencode-ai/sdk": "^1.1.12", - "@openrouter/ai-sdk-provider": "^1.2.0", + "@sctg/sentencepiece-js": "^1.1.0", "@xterm/headless": "^6.0.0", "ai": "^5.0.76", "ajv": "^8.17.1", @@ -45,13 +45,12 @@ "express-basic-auth": "^1.2.1", "mnemonic-id": "^3.2.7", "node-pty": "^1.0.0", + "onnxruntime-node": "^1.23.0", "openai": "^4.20.0", "pino": "^10.2.0", "pino-pretty": "^13.1.3", "playwright": "^1.56.1", "qrcode": "^1.5.4", - "@sctg/sentencepiece-js": "^1.1.0", - "onnxruntime-node": "^1.23.0", "sherpa-onnx": "^1.12.23", "sherpa-onnx-node": "^1.12.23", "strip-ansi": "^7.1.2", diff --git a/packages/server/scripts/download-speech-models.ts b/packages/server/scripts/download-speech-models.ts index a531c5942..fe2b91667 100644 --- a/packages/server/scripts/download-speech-models.ts +++ b/packages/server/scripts/download-speech-models.ts @@ -1,12 +1,16 @@ import { resolvePaseoHome } from "../src/server/paseo-home.js"; import { createRootLogger } from "../src/server/logger.js"; -import { ensureSherpaOnnxModels } from "../src/server/speech/providers/local/sherpa/model-downloader.js"; -import type { SherpaOnnxModelId } from "../src/server/speech/providers/local/sherpa/model-catalog.js"; +import { + DEFAULT_LOCAL_STT_MODEL, + DEFAULT_LOCAL_TTS_MODEL, + ensureLocalSpeechModels, + type LocalSpeechModelId, +} from "../src/server/speech/providers/local/models.js"; -function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxModelId[] } { +function parseArgs(argv: string[]): { modelsDir: string; modelIds: LocalSpeechModelId[] } { const home = resolvePaseoHome(); - let modelsDir = process.env.PASEO_SHERPA_ONNX_MODELS_DIR || `${home}/models/sherpa-onnx`; - const modelIds: SherpaOnnxModelId[] = []; + let modelsDir = process.env.PASEO_LOCAL_MODELS_DIR || `${home}/models/local-speech`; + const modelIds: LocalSpeechModelId[] = []; for (let i = 0; i < argv.length; i++) { const arg = argv[i]; @@ -16,7 +20,7 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxMod continue; } if (arg === "--model") { - const id = argv[i + 1] as SherpaOnnxModelId | undefined; + const id = argv[i + 1] as LocalSpeechModelId | undefined; if (!id) { throw new Error("--model requires a value"); } @@ -27,8 +31,8 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxMod } if (modelIds.length === 0) { - const stt = (process.env.PASEO_SHERPA_STT_PRESET || "zipformer-bilingual-zh-en-2023-02-20") as SherpaOnnxModelId; - const tts = (process.env.PASEO_SHERPA_TTS_PRESET || "pocket-tts-onnx-int8") as SherpaOnnxModelId; + const stt = (process.env.PASEO_LOCAL_STT_MODEL || DEFAULT_LOCAL_STT_MODEL) as LocalSpeechModelId; + const tts = (process.env.PASEO_LOCAL_TTS_MODEL || DEFAULT_LOCAL_TTS_MODEL) as LocalSpeechModelId; modelIds.push(stt, tts); } @@ -38,5 +42,5 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxMod const logger = createRootLogger({ level: "info", format: "pretty" }); const { modelsDir, modelIds } = parseArgs(process.argv.slice(2)); -await ensureSherpaOnnxModels({ modelsDir, modelIds, autoDownload: true, logger }); +await ensureLocalSpeechModels({ modelsDir, modelIds, autoDownload: true, logger }); logger.info({ modelsDir, modelIds }, "Done downloading speech models"); diff --git a/packages/server/scripts/list-speech-models.ts b/packages/server/scripts/list-speech-models.ts index 8462dfc27..dc014bb5e 100644 --- a/packages/server/scripts/list-speech-models.ts +++ b/packages/server/scripts/list-speech-models.ts @@ -1,6 +1,6 @@ -import { listSherpaOnnxModels } from "../src/server/speech/providers/local/sherpa/model-catalog.js"; +import { listLocalSpeechModels } from "../src/server/speech/providers/local/models.js"; -const models = listSherpaOnnxModels() +const models = listLocalSpeechModels() .slice() .sort((a, b) => a.kind.localeCompare(b.kind) || a.id.localeCompare(b.id)); diff --git a/packages/server/src/client/daemon-client.ts b/packages/server/src/client/daemon-client.ts index 8994e9407..43387c76f 100644 --- a/packages/server/src/client/daemon-client.ts +++ b/packages/server/src/client/daemon-client.ts @@ -12,9 +12,7 @@ import type { AgentStreamEventPayload, AgentSnapshotPayload, AgentPermissionResolvedMessage, - VoiceConversationLoadedMessage, CreateAgentRequestMessage, - DeleteVoiceConversationResponseMessage, FileDownloadTokenResponse, FileExplorerResponse, GitDiffResponse, @@ -34,7 +32,6 @@ import type { ProjectIconResponse, ListCommandsResponse, ExecuteCommandResponse, - ListVoiceConversationsResponseMessage, ListProviderModelsResponseMessage, SpeechModelsListResponse, SpeechModelsDownloadResponse, @@ -184,9 +181,6 @@ export type CreateAgentRequestOptions = { labels?: Record; } & AgentConfigOverrides; -type VoiceConversationLoadedPayload = VoiceConversationLoadedMessage["payload"]; -type ListVoiceConversationsPayload = ListVoiceConversationsResponseMessage["payload"]; -type DeleteVoiceConversationPayload = DeleteVoiceConversationResponseMessage["payload"]; type GitDiffPayload = GitDiffResponse["payload"]; type HighlightedDiffPayload = HighlightedDiffResponse["payload"]; type CheckoutStatusPayload = CheckoutStatusResponse["payload"]; @@ -765,10 +759,6 @@ export class DaemonClient { } } - sendUserMessage(text: string): void { - this.sendSessionMessage({ type: "user_text", text }); - } - clearAgentAttention(agentId: string | string[]): void { this.sendSessionMessage({ type: "clear_agent_attention", agentId }); } @@ -926,87 +916,6 @@ export class DaemonClient { } } - // ============================================================================ - // Voice Conversation RPC - // ============================================================================ - - async loadVoiceConversation( - voiceConversationId: string, - requestId?: string - ): Promise { - const resolvedRequestId = this.createRequestId(requestId); - const message = SessionInboundMessageSchema.parse({ - type: "load_voice_conversation_request", - voiceConversationId, - requestId: resolvedRequestId, - }); - return this.sendRequest({ - requestId: resolvedRequestId, - message, - timeout: 10000, - options: { skipQueue: true }, - select: (msg) => { - if (msg.type !== "voice_conversation_loaded") { - return null; - } - if (msg.payload.requestId !== resolvedRequestId) { - return null; - } - return msg.payload; - }, - }); - } - - async listVoiceConversations(requestId?: string): Promise { - const resolvedRequestId = this.createRequestId(requestId); - const message = SessionInboundMessageSchema.parse({ - type: "list_voice_conversations_request", - requestId: resolvedRequestId, - }); - return this.sendRequest({ - requestId: resolvedRequestId, - message, - timeout: 10000, - options: { skipQueue: true }, - select: (msg) => { - if (msg.type !== "list_voice_conversations_response") { - return null; - } - if (msg.payload.requestId !== resolvedRequestId) { - return null; - } - return msg.payload; - }, - }); - } - - async deleteVoiceConversation( - voiceConversationId: string, - requestId?: string - ): Promise { - const resolvedRequestId = this.createRequestId(requestId); - const message = SessionInboundMessageSchema.parse({ - type: "delete_voice_conversation_request", - voiceConversationId, - requestId: resolvedRequestId, - }); - return this.sendRequest({ - requestId: resolvedRequestId, - message, - timeout: 10000, - options: { skipQueue: true }, - select: (msg) => { - if (msg.type !== "delete_voice_conversation_response") { - return null; - } - if (msg.payload.requestId !== resolvedRequestId) { - return null; - } - return msg.payload; - }, - }); - } - // ============================================================================ // Agent Lifecycle // ============================================================================ @@ -1376,8 +1285,8 @@ export class DaemonClient { // Audio / Voice // ============================================================================ - async setVoiceConversation(enabled: boolean, voiceConversationId?: string): Promise { - this.sendSessionMessage({ type: "set_voice_conversation", enabled, voiceConversationId }); + async setVoiceMode(enabled: boolean, voiceAgentId?: string): Promise { + this.sendSessionMessage({ type: "set_voice_mode", enabled, voiceAgentId }); } async sendVoiceAudioChunk( diff --git a/packages/server/src/server/agent/activity-curator.test.ts b/packages/server/src/server/agent/activity-curator.test.ts index a0e10b615..781d1342b 100644 --- a/packages/server/src/server/agent/activity-curator.test.ts +++ b/packages/server/src/server/agent/activity-curator.test.ts @@ -66,6 +66,22 @@ describe("curateAgentActivity", () => { expect(result).toBe("[ListFiles]"); }); + test("does not treat generic double-underscore tool names as MCP calls", () => { + const timeline: AgentTimelineItem[] = [ + { + type: "tool_call", + callId: "call-1", + name: "custom__tool", + input: {}, + status: "completed", + }, + ]; + + const result = curateAgentActivity(timeline); + + expect(result).toBe("[custom__tool]"); + }); + test("serializes todo items as [Tasks]", () => { const timeline: AgentTimelineItem[] = [ { @@ -328,5 +344,37 @@ describe("curateAgentActivity", () => { expect(result).toBe("[Grep] TODO"); }); + + test("shows speak tool text input", () => { + const timeline: AgentTimelineItem[] = [ + { + type: "tool_call", + callId: "s1", + name: "speak", + input: { text: "hello from voice" }, + status: "completed", + }, + ]; + + const result = curateAgentActivity(timeline); + expect(result).toBe('[speak] {"text":"hello from voice"}'); + }); + + test("shows MCP tool input JSON", () => { + const timeline: AgentTimelineItem[] = [ + { + type: "tool_call", + callId: "m1", + name: "paseo__create_agent", + input: { cwd: "/tmp/repo", initialPrompt: "do the thing" }, + status: "completed", + }, + ]; + + const result = curateAgentActivity(timeline); + expect(result).toBe( + '[paseo__create_agent] {"cwd":"/tmp/repo","initialPrompt":"do the thing"}' + ); + }); }); }); diff --git a/packages/server/src/server/agent/activity-curator.ts b/packages/server/src/server/agent/activity-curator.ts index 806ad6e2e..8370b0d6a 100644 --- a/packages/server/src/server/agent/activity-curator.ts +++ b/packages/server/src/server/agent/activity-curator.ts @@ -2,6 +2,7 @@ import type { AgentTimelineItem } from "./agent-sdk-types.js"; import { extractPrincipalParam } from "../../utils/tool-call-parsers.js"; const DEFAULT_MAX_ITEMS = 40; +const MAX_TOOL_INPUT_CHARS = 400; function appendText(buffer: string, text: string): string { const normalized = text.trim(); @@ -25,6 +26,35 @@ function flushBuffers(lines: string[], buffers: { message: string; thought: stri buffers.thought = ""; } +function isLikelyMcpToolCall(name: string): boolean { + const normalized = name.toLowerCase(); + return ( + normalized === "speak" || + normalized.startsWith("mcp") || + normalized.includes("mcp__") || + normalized.startsWith("paseo") || + normalized.includes("paseo__") + ); +} + +function formatToolInputJson(input: unknown): string | null { + if (input === undefined) { + return null; + } + try { + const encoded = JSON.stringify(input); + if (!encoded) { + return null; + } + if (encoded.length <= MAX_TOOL_INPUT_CHARS) { + return encoded; + } + return `${encoded.slice(0, MAX_TOOL_INPUT_CHARS)}...`; + } catch { + return null; + } +} + /** * Collapse timeline items: * - Dedupe tool calls by callId (pending/completed -> single) @@ -127,6 +157,11 @@ export function curateAgentActivity( break; case "tool_call": { flushBuffers(lines, buffers); + const inputJson = formatToolInputJson(item.input); + if (isLikelyMcpToolCall(item.name) && inputJson) { + lines.push(`[${item.name}] ${inputJson}`); + break; + } const principal = extractPrincipalParam(item.input); if (principal) { lines.push(`[${item.name}] ${principal}`); diff --git a/packages/server/src/server/agent/agent-management-mcp.ts b/packages/server/src/server/agent/agent-management-mcp.ts index e2e26d0b6..4f46e0882 100644 --- a/packages/server/src/server/agent/agent-management-mcp.ts +++ b/packages/server/src/server/agent/agent-management-mcp.ts @@ -23,8 +23,6 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; -import { homedir } from "node:os"; -import { resolve } from "node:path"; import { ensureValidJson } from "../json-utils.js"; import type { Logger } from "pino"; @@ -51,6 +49,7 @@ import { AgentStorage } from "./agent-storage.js"; import { createWorktree } from "../../utils/worktree.js"; import { WaitForAgentTracker } from "./wait-for-agent-tracker.js"; import { scheduleAgentMetadataGeneration } from "./agent-metadata-generator.js"; +import { expandUserPath } from "../path-utils.js"; export interface AgentManagementMcpOptions { agentManager: AgentManager; @@ -77,13 +76,6 @@ const AgentStatusEnum = z.enum([ // 50 seconds - surface friendly message before SDK tool timeout (~60s) const AGENT_WAIT_TIMEOUT_MS = 50000; -function expandPath(path: string): string { - if (path.startsWith("~/") || path === "~") { - return resolve(homedir(), path.slice(2)); - } - return resolve(path); -} - async function waitForAgentWithTimeout( agentManager: AgentManager, agentId: string, @@ -320,7 +312,7 @@ export async function createAgentManagementMcpServer( title: string; }; - let resolvedCwd = expandPath(cwd); + let resolvedCwd = expandUserPath(cwd); if (worktreeName) { if (!baseBranch) { diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 067e3faf3..d7302ef77 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -118,7 +118,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "agent-without-model", + idFactory: () => "00000000-0000-4000-8000-000000000101", }); const snapshot = await manager.createAgent({ @@ -129,7 +129,7 @@ describe("AgentManager", () => { expect(snapshot.model).toBeUndefined(); }); - test("createAgent persists provided title before returning", async () => { + test("createAgent fails when cwd does not exist", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); const storage = new AgentStorage(storagePath, logger); @@ -139,7 +139,72 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "agent-with-title", + }); + + await expect( + manager.createAgent({ + provider: "codex", + cwd: join(workdir, "does-not-exist"), + }) + ).rejects.toThrow("Working directory does not exist"); + }); + + test("createAgent fails when generated agent ID is not a UUID", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); + const storagePath = join(workdir, "agents"); + const storage = new AgentStorage(storagePath, logger); + const manager = new AgentManager({ + clients: { + codex: new TestAgentClient(), + }, + registry: storage, + logger, + idFactory: () => "not-a-uuid", + }); + + await expect( + manager.createAgent({ + provider: "codex", + cwd: workdir, + }) + ).rejects.toThrow("createAgent: agentId must be a UUID"); + }); + + test("createAgent fails when explicit agent ID is not a UUID", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); + const storagePath = join(workdir, "agents"); + const storage = new AgentStorage(storagePath, logger); + const manager = new AgentManager({ + clients: { + codex: new TestAgentClient(), + }, + registry: storage, + logger, + }); + + await expect( + manager.createAgent( + { + provider: "codex", + cwd: workdir, + }, + "not-a-uuid" + ) + ).rejects.toThrow("createAgent: agentId must be a UUID"); + }); + + test("createAgent persists provided title before returning", async () => { + const agentId = "00000000-0000-4000-8000-000000000102"; + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); + const storagePath = join(workdir, "agents"); + const storage = new AgentStorage(storagePath, logger); + const manager = new AgentManager({ + clients: { + codex: new TestAgentClient(), + }, + registry: storage, + logger, + idFactory: () => agentId, }); const snapshot = await manager.createAgent({ @@ -148,12 +213,12 @@ describe("AgentManager", () => { title: "Fix Login Bug", }); - expect(snapshot.id).toBe("agent-with-title"); + expect(snapshot.id).toBe(agentId); expect(snapshot.lifecycle).toBe("idle"); - const persisted = await storage.get("agent-with-title"); + const persisted = await storage.get(agentId); expect(persisted?.title).toBe("Fix Login Bug"); - expect(persisted?.id).toBe("agent-with-title"); + expect(persisted?.id).toBe(agentId); }); test("createAgent populates runtimeInfo after session creation", async () => { @@ -166,7 +231,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "agent-with-runtime-info", + idFactory: () => "00000000-0000-4000-8000-000000000103", }); const snapshot = await manager.createAgent({ @@ -191,7 +256,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "agent-with-run-runtime", + idFactory: () => "00000000-0000-4000-8000-000000000104", }); const snapshot = await manager.createAgent({ @@ -211,6 +276,10 @@ describe("AgentManager", () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); const storage = new AgentStorage(storagePath, logger); + const generatedAgentIds = [ + "00000000-0000-4000-8000-000000000105", + "00000000-0000-4000-8000-000000000106", + ]; let agentCounter = 0; const manager = new AgentManager({ clients: { @@ -218,7 +287,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => `agent-${agentCounter++}`, + idFactory: () => generatedAgentIds[agentCounter++] ?? randomUUID(), }); // Create a normal agent @@ -242,6 +311,7 @@ describe("AgentManager", () => { }); test("getAgent returns internal agents by ID", async () => { + const internalAgentId = "00000000-0000-4000-8000-000000000107"; const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); const storage = new AgentStorage(storagePath, logger); @@ -251,7 +321,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "internal-agent", + idFactory: () => internalAgentId, }); await manager.createAgent({ @@ -261,7 +331,7 @@ describe("AgentManager", () => { internal: true, }); - const agent = manager.getAgent("internal-agent"); + const agent = manager.getAgent(internalAgentId); expect(agent).not.toBeNull(); expect(agent?.internal).toBe(true); }); @@ -270,6 +340,10 @@ describe("AgentManager", () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); const storage = new AgentStorage(storagePath, logger); + const generatedAgentIds = [ + "00000000-0000-4000-8000-000000000108", + "00000000-0000-4000-8000-000000000109", + ]; let agentCounter = 0; const manager = new AgentManager({ clients: { @@ -277,7 +351,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => `agent-${agentCounter++}`, + idFactory: () => generatedAgentIds[agentCounter++] ?? randomUUID(), }); const receivedEvents: string[] = []; @@ -303,11 +377,12 @@ describe("AgentManager", () => { }); // Should only have events from the normal agent - expect(receivedEvents.filter((id) => id === "agent-0").length).toBeGreaterThan(0); - expect(receivedEvents.filter((id) => id === "agent-1").length).toBe(0); + expect(receivedEvents.filter((id) => id === generatedAgentIds[0]).length).toBeGreaterThan(0); + expect(receivedEvents.filter((id) => id === generatedAgentIds[1]).length).toBe(0); }); test("subscribe emits state events for internal agents when subscribed by agentId", async () => { + const internalAgentId = "00000000-0000-4000-8000-000000000110"; const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); const storage = new AgentStorage(storagePath, logger); @@ -317,7 +392,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "internal-agent", + idFactory: () => internalAgentId, }); const receivedEvents: string[] = []; @@ -328,7 +403,7 @@ describe("AgentManager", () => { receivedEvents.push(event.agent.id); } }, - { agentId: "internal-agent", replayState: false } + { agentId: internalAgentId, replayState: false } ); await manager.createAgent({ @@ -339,10 +414,26 @@ describe("AgentManager", () => { }); // Should receive events when subscribed by specific agentId - expect(receivedEvents.filter((id) => id === "internal-agent").length).toBeGreaterThan(0); + expect(receivedEvents.filter((id) => id === internalAgentId).length).toBeGreaterThan(0); + }); + + test("subscribe fails when filter agentId is not a UUID", () => { + const manager = new AgentManager({ + clients: { + codex: new TestAgentClient(), + }, + logger, + }); + + expect(() => + manager.subscribe(() => {}, { + agentId: "invalid-agent-id", + }) + ).toThrow("subscribe: agentId must be a UUID"); }); test("onAgentAttention is not called for internal agents", async () => { + const internalAgentId = "00000000-0000-4000-8000-000000000111"; const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-")); const storagePath = join(workdir, "agents"); const storage = new AgentStorage(storagePath, logger); @@ -353,7 +444,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "internal-agent", + idFactory: () => internalAgentId, onAgentAttention: ({ agentId }) => { attentionCalls.push(agentId); }, @@ -461,7 +552,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "plan-mode-agent", + idFactory: () => "00000000-0000-4000-8000-000000000112", }); // Create agent in plan mode @@ -588,7 +679,7 @@ describe("AgentManager", () => { }, registry: storage, logger, - idFactory: () => "close-race-agent", + idFactory: () => "00000000-0000-4000-8000-000000000113", }); const snapshot = await manager.createAgent({ diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index f3578ba9b..efb939c15 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -1,10 +1,12 @@ import { randomUUID } from "node:crypto"; import { resolve } from "node:path"; +import { stat } from "node:fs/promises"; import { AGENT_LIFECYCLE_STATUSES, type AgentLifecycleStatus, } from "../../shared/agent-lifecycle.js"; import type { Logger } from "pino"; +import { z } from "zod"; import type { AgentCapabilityFlags, @@ -182,6 +184,7 @@ const BUSY_STATUSES: AgentLifecycleStatus[] = [ "initializing", "running", ]; +const AgentIdSchema = z.string().uuid(); function isAgentBusy(status: AgentLifecycleStatus): boolean { return BUSY_STATUSES.includes(status); @@ -201,6 +204,14 @@ function createAbortError( return Object.assign(new Error(message), { name: "AbortError" }); } +function validateAgentId(agentId: string, source: string): string { + const result = AgentIdSchema.safeParse(agentId); + if (!result.success) { + throw new Error(`${source}: agentId must be a UUID`); + } + return result.data; +} + export class AgentManager { private readonly clients = new Map(); private readonly agents = new Map(); @@ -238,9 +249,13 @@ export class AgentManager { } subscribe(callback: AgentSubscriber, options?: SubscribeOptions): () => void { + const targetAgentId = + options?.agentId == null + ? null + : validateAgentId(options.agentId, "subscribe"); const record: SubscriptionRecord = { callback, - agentId: options?.agentId ?? null, + agentId: targetAgentId, }; this.subscribers.add(record); @@ -334,7 +349,10 @@ export class AgentManager { options?: { labels?: Record } ): Promise { // Generate agent ID early so we can use it in MCP config - const resolvedAgentId = agentId ?? this.idFactory(); + const resolvedAgentId = validateAgentId( + agentId ?? this.idFactory(), + "createAgent" + ); const normalizedConfig = await this.normalizeConfig(config, { labels: options?.labels, agentId: resolvedAgentId, @@ -364,6 +382,10 @@ export class AgentManager { labels?: Record; } ): Promise { + const resolvedAgentId = validateAgentId( + agentId ?? this.idFactory(), + "resumeAgent" + ); const metadata = (handle.metadata ?? {}) as Partial; const mergedConfig = { ...metadata, @@ -380,7 +402,7 @@ export class AgentManager { return this.registerSession( session, normalizedConfig, - agentId ?? this.idFactory(), + resolvedAgentId, options ); } @@ -981,13 +1003,14 @@ export class AgentManager { labels?: Record; } ): Promise { - if (this.agents.has(agentId)) { - throw new Error(`Agent with id ${agentId} already exists`); + const resolvedAgentId = validateAgentId(agentId, "registerSession"); + if (this.agents.has(resolvedAgentId)) { + throw new Error(`Agent with id ${resolvedAgentId} already exists`); } const now = new Date(); const managed = { - id: agentId, + id: resolvedAgentId, provider: config.provider, cwd: config.cwd, session, @@ -1010,9 +1033,9 @@ export class AgentManager { labels: options?.labels ?? {}, } as ActiveManagedAgent; - this.agents.set(agentId, managed); + this.agents.set(resolvedAgentId, managed); // Initialize previousStatus to track transitions - this.previousStatuses.set(agentId, managed.lifecycle); + this.previousStatuses.set(resolvedAgentId, managed.lifecycle); await this.refreshRuntimeInfo(managed); await this.persistSnapshot(managed, { title: config.title ?? null, @@ -1319,6 +1342,20 @@ export class AgentManager { // Always resolve cwd to absolute path for consistent history file lookup if (normalized.cwd) { normalized.cwd = resolve(normalized.cwd); + try { + const cwdStats = await stat(normalized.cwd); + if (!cwdStats.isDirectory()) { + throw new Error(`Working directory is not a directory: ${normalized.cwd}`); + } + } catch (error) { + if (error instanceof Error && "code" in error && (error as NodeJS.ErrnoException).code === "ENOENT") { + throw new Error(`Working directory does not exist: ${normalized.cwd}`); + } + if (error instanceof Error) { + throw error; + } + throw new Error(`Failed to access working directory: ${normalized.cwd}`); + } } if (typeof normalized.model === "string") { @@ -1338,9 +1375,10 @@ export class AgentManager { } private requireAgent(id: string): ActiveManagedAgent { - const agent = this.agents.get(id); + const normalizedId = validateAgentId(id, "requireAgent"); + const agent = this.agents.get(normalizedId); if (!agent) { - throw new Error(`Unknown agent '${id}'`); + throw new Error(`Unknown agent '${normalizedId}'`); } return agent; } diff --git a/packages/server/src/server/agent/agent-mcp.e2e.test.ts b/packages/server/src/server/agent/agent-mcp.e2e.test.ts index 527725f9f..d03485365 100644 --- a/packages/server/src/server/agent/agent-mcp.e2e.test.ts +++ b/packages/server/src/server/agent/agent-mcp.e2e.test.ts @@ -90,7 +90,6 @@ describe("agent MCP end-to-end (offline)", () => { mcpDebug: false, agentClients: createTestAgentClients(), agentStoragePath: path.join(paseoHome, "agents"), - openrouterApiKey: null, }; const daemon = await createPaseoDaemon(daemonConfig, pino({ level: "silent" })); diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index afbfbbf64..8deba1996 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -1,4 +1,7 @@ import { describe, expect, it, vi } from "vitest"; +import { mkdtemp, mkdir, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; import { createTestLogger } from "../../test-utils/test-logger.js"; import { createAgentMcpServer } from "./mcp-server.js"; @@ -48,6 +51,7 @@ function createTestDeps(): TestDeps { describe("create_agent MCP tool", () => { const logger = createTestLogger(); + const existingCwd = process.cwd(); it("requires a concise title no longer than 60 characters", async () => { const { agentManager, agentStorage } = createTestDeps(); @@ -56,26 +60,59 @@ describe("create_agent MCP tool", () => { expect(tool).toBeDefined(); const missingTitle = await tool.inputSchema.safeParseAsync({ - cwd: "/tmp/repo", + cwd: existingCwd, initialMode: "default", + initialPrompt: "test", }); expect(missingTitle.success).toBe(false); expect(missingTitle.error.issues[0].path).toEqual(["title"]); const tooLong = await tool.inputSchema.safeParseAsync({ - cwd: "/tmp/repo", + cwd: existingCwd, initialMode: "default", title: "x".repeat(61), + initialPrompt: "test", }); expect(tooLong.success).toBe(false); expect(tooLong.error.issues[0].path).toEqual(["title"]); const ok = await tool.inputSchema.safeParseAsync({ - cwd: "/tmp/repo", + cwd: existingCwd, + initialMode: "default", + title: "Short title", + initialPrompt: "test", + }); + expect(ok.success).toBe(true); + }); + + it("requires initialPrompt", async () => { + const { agentManager, agentStorage } = createTestDeps(); + const server = await createAgentMcpServer({ agentManager, agentStorage, logger }); + const tool = (server as any)._registeredTools["create_agent"]; + const parsed = await tool.inputSchema.safeParseAsync({ + cwd: existingCwd, initialMode: "default", title: "Short title", }); - expect(ok.success).toBe(true); + expect(parsed.success).toBe(false); + expect(parsed.error.issues.some((issue: { path: string[] }) => issue.path[0] === "initialPrompt")).toBe(true); + }); + + it("surfaces createAgent validation failures", async () => { + const { agentManager, agentStorage, spies } = createTestDeps(); + spies.agentManager.createAgent.mockRejectedValue( + new Error("Working directory does not exist: /path/that/does/not/exist") + ); + const server = await createAgentMcpServer({ agentManager, agentStorage, logger }); + const tool = (server as any)._registeredTools["create_agent"]; + + await expect( + tool.callback({ + cwd: "/path/that/does/not/exist", + title: "Short title", + initialPrompt: "Do work", + }) + ).rejects.toThrow("Working directory does not exist"); }); it("passes caller-provided titles directly into createAgent", async () => { @@ -86,20 +123,24 @@ describe("create_agent MCP tool", () => { lifecycle: "idle", currentModeId: null, availableModes: [], + config: { title: "Fix auth bug" }, } as ManagedAgent); const server = await createAgentMcpServer({ agentManager, agentStorage, logger }); const tool = (server as any)._registeredTools["create_agent"]; await tool.callback({ - cwd: "/tmp/repo", + cwd: existingCwd, title: " Fix auth bug ", + initialPrompt: "Do work", }); expect(spies.agentManager.createAgent).toHaveBeenCalledWith( expect.objectContaining({ - cwd: "/tmp/repo", + cwd: existingCwd, title: "Fix auth bug", - }) + }), + undefined, + undefined ); }); @@ -111,19 +152,127 @@ describe("create_agent MCP tool", () => { lifecycle: "idle", currentModeId: null, availableModes: [], + config: { title: "Fix auth" }, } as ManagedAgent); const server = await createAgentMcpServer({ agentManager, agentStorage, logger }); const tool = (server as any)._registeredTools["create_agent"]; await tool.callback({ - cwd: "/tmp/repo", + cwd: existingCwd, title: " Fix auth ", + initialPrompt: "Do work", }); expect(spies.agentManager.createAgent).toHaveBeenCalledWith( expect.objectContaining({ title: "Fix auth", + }), + undefined, + undefined + ); + }); + + it("allows caller agents to override cwd and applies caller context labels", async () => { + const { agentManager, agentStorage, spies } = createTestDeps(); + const baseDir = await mkdtemp(join(tmpdir(), "paseo-mcp-test-")); + const subdir = join(baseDir, "subdir"); + await mkdir(subdir, { recursive: true }); + spies.agentManager.getAgent.mockReturnValue({ + id: "voice-agent", + cwd: baseDir, + provider: "codex", + currentModeId: "full-access", + } as ManagedAgent); + spies.agentManager.createAgent.mockResolvedValue({ + id: "child-agent", + cwd: subdir, + lifecycle: "idle", + currentModeId: null, + availableModes: [], + config: { title: "Child" }, + } as ManagedAgent); + + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + callerAgentId: "voice-agent", + resolveCallerContext: () => ({ + childAgentDefaultLabels: { ui: "true" }, + allowCustomCwd: true, + }), + logger, + }); + + const tool = (server as any)._registeredTools["create_agent"]; + await tool.callback({ + cwd: "subdir", + title: "Child", + agentType: "codex", + initialPrompt: "Do work", + }); + + expect(spies.agentManager.createAgent).toHaveBeenCalledWith( + expect.objectContaining({ + cwd: subdir, + }), + undefined, + { labels: { ui: "true" } } + ); + await rm(baseDir, { recursive: true, force: true }); + }); +}); + +describe("speak MCP tool", () => { + const logger = createTestLogger(); + + it("invokes registered speak handler for caller agent", async () => { + const { agentManager, agentStorage } = createTestDeps(); + const speak = vi.fn().mockResolvedValue(undefined); + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + callerAgentId: "voice-agent-1", + enableVoiceTools: true, + resolveSpeakHandler: () => speak, + logger, + }); + const tool = (server as any)._registeredTools["speak"]; + expect(tool).toBeDefined(); + + await tool.callback({ text: "Hello from voice agent." }); + expect(speak).toHaveBeenCalledWith( + expect.objectContaining({ + text: "Hello from voice agent.", + callerAgentId: "voice-agent-1", }) ); }); + + it("fails when no speak handler exists", async () => { + const { agentManager, agentStorage } = createTestDeps(); + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + callerAgentId: "voice-agent-2", + enableVoiceTools: true, + resolveSpeakHandler: () => null, + logger, + }); + const tool = (server as any)._registeredTools["speak"]; + await expect(tool.callback({ text: "Hello." })).rejects.toThrow( + "No speak handler registered for caller agent" + ); + }); + + it("does not register speak tool unless voice tools are enabled", async () => { + const { agentManager, agentStorage } = createTestDeps(); + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + callerAgentId: "agent-no-voice", + logger, + }); + const tool = (server as any)._registeredTools["speak"]; + expect(tool).toBeUndefined(); + }); }); diff --git a/packages/server/src/server/agent/mcp-server.ts b/packages/server/src/server/agent/mcp-server.ts index ee7d4d70a..b8c62667e 100644 --- a/packages/server/src/server/agent/mcp-server.ts +++ b/packages/server/src/server/agent/mcp-server.ts @@ -1,9 +1,12 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; -import { homedir } from "node:os"; -import { resolve } from "node:path"; import { ensureValidJson } from "../json-utils.js"; import type { Logger } from "pino"; +import type { RequestHandlerExtra } from "@modelcontextprotocol/sdk/shared/protocol.js"; +import type { + ServerNotification, + ServerRequest, +} from "@modelcontextprotocol/sdk/types.js"; import type { AgentPromptInput, @@ -28,6 +31,11 @@ import { AgentStorage } from "./agent-storage.js"; import { createWorktree } from "../../utils/worktree.js"; import { WaitForAgentTracker } from "./wait-for-agent-tracker.js"; import { scheduleAgentMetadataGeneration } from "./agent-metadata-generator.js"; +import type { + VoiceCallerContext, + VoiceSpeakHandler, +} from "../voice-types.js"; +import { expandUserPath, resolvePathFromBase } from "../path-utils.js"; export interface AgentMcpServerOptions { agentManager: AgentManager; @@ -38,6 +46,17 @@ export interface AgentMcpServerOptions { * Used for cwd/mode inheritance when agents spawn child agents. */ callerAgentId?: string; + /** + * Optional resolver for session-bound speak handlers. + * Used by hidden voice agents to narrate through daemon-managed TTS. + */ + resolveSpeakHandler?: ( + callerAgentId: string + ) => VoiceSpeakHandler | null; + resolveCallerContext?: ( + callerAgentId: string + ) => VoiceCallerContext | null; + enableVoiceTools?: boolean; logger: Logger; } @@ -100,11 +119,25 @@ const AgentStatusEnum = z.enum([ // 50 seconds - surface friendly message before SDK tool timeout (~60s) const AGENT_WAIT_TIMEOUT_MS = 50000; -function expandPath(path: string): string { - if (path.startsWith("~/") || path === "~") { - return resolve(homedir(), path.slice(2)); +type McpToolContext = RequestHandlerExtra; + +function resolveChildAgentCwd(params: { + parentCwd: string; + requestedCwd?: string; + lockedCwd?: string; + allowCustomCwd: boolean; +}): string { + const lockedCwd = params.lockedCwd?.trim(); + if (lockedCwd) { + return expandUserPath(lockedCwd); } - return resolve(path); + + const requestedCwd = params.requestedCwd?.trim(); + if (!requestedCwd || !params.allowCustomCwd) { + return params.parentCwd; + } + + return resolvePathFromBase(params.parentCwd, requestedCwd); } /** @@ -252,9 +285,17 @@ async function serializeSnapshotWithMetadata( export async function createAgentMcpServer( options: AgentMcpServerOptions ): Promise { - const { agentManager, agentStorage, callerAgentId, logger } = options; + const { + agentManager, + agentStorage, + callerAgentId, + resolveSpeakHandler, + resolveCallerContext, + logger, + } = options; const childLogger = logger.child({ module: "agent", component: "mcp-server" }); const waitTracker = new WaitForAgentTracker(logger); + const callerContext = callerAgentId ? resolveCallerContext?.(callerAgentId) ?? null : null; const server = new McpServer({ name: "agent-mcp", @@ -262,6 +303,12 @@ export async function createAgentMcpServer( }); const agentToAgentInputSchema = { + cwd: z + .string() + .optional() + .describe( + "Optional working directory. Defaults to the caller agent working directory." + ), title: z .string() .trim() @@ -275,9 +322,10 @@ export async function createAgentMcpServer( ), initialPrompt: z .string() - .optional() + .trim() + .min(1, "initialPrompt is required") .describe( - "Optional task to start immediately after creation (non-blocking)." + "Required first task to run immediately after creation." ), background: z .boolean() @@ -307,9 +355,10 @@ export async function createAgentMcpServer( ), initialPrompt: z .string() - .optional() + .trim() + .min(1, "initialPrompt is required") .describe( - "Optional task to start immediately after creation (non-blocking)." + "Required first task to run immediately after creation." ), initialMode: z .string() @@ -338,6 +387,50 @@ export async function createAgentMcpServer( const createAgentInputSchema = callerAgentId ? agentToAgentInputSchema : topLevelInputSchema; + const agentToAgentCreateAgentArgsSchema = z.object(agentToAgentInputSchema); + const topLevelCreateAgentArgsSchema = z.object({ + ...topLevelInputSchema, + initialMode: topLevelInputSchema.initialMode.optional(), + }); + + if (options.enableVoiceTools || callerContext?.enableVoiceTools) { + server.registerTool( + "speak", + { + title: "Speak", + description: + "Speak text to the user via daemon-managed voice output. Blocks until playback completes.", + inputSchema: { + text: z + .string() + .trim() + .min(1, "text is required") + .max(4000, "text must be 4000 characters or fewer"), + }, + outputSchema: { + ok: z.boolean(), + }, + }, + async (args, context?: McpToolContext) => { + if (!callerAgentId) { + throw new Error("speak is only available to agent-scoped MCP sessions"); + } + const handler = resolveSpeakHandler?.(callerAgentId) ?? null; + if (!handler) { + throw new Error(`No speak handler registered for caller agent '${callerAgentId}'`); + } + await handler({ + text: args.text, + callerAgentId, + signal: context?.signal, + }); + return { + content: [], + structuredContent: ensureValidJson({ ok: true }), + }; + } + ); + } server.registerTool( "create_agent", @@ -363,33 +456,32 @@ export async function createAgentMcpServer( permission: AgentPermissionRequestPayloadSchema.nullable().optional(), }, }, - async (args) => { - const { - agentType, - initialPrompt, - background = false, - title, - } = args as { - cwd?: string; - agentType?: AgentProvider; - initialPrompt?: string; - initialMode?: string; - worktreeName?: string; - background?: boolean; - title: string; - }; + async (args: unknown) => { + let provider: AgentProvider; + let initialPrompt: string; + let background = false; + let normalizedTitle: string | null; let resolvedCwd: string; let resolvedMode: string | undefined; if (callerAgentId) { + const callerArgs = agentToAgentCreateAgentArgsSchema.parse(args); + provider = callerArgs.agentType ?? "claude"; + initialPrompt = callerArgs.initialPrompt; + background = callerArgs.background ?? false; + normalizedTitle = callerArgs.title.trim(); + const parentAgent = agentManager.getAgent(callerAgentId); if (!parentAgent) { throw new Error(`Parent agent ${callerAgentId} not found`); } - resolvedCwd = parentAgent.cwd; - - const provider: AgentProvider = agentType ?? "claude"; + resolvedCwd = resolveChildAgentCwd({ + parentCwd: parentAgent.cwd, + requestedCwd: callerArgs.cwd, + lockedCwd: callerContext?.lockedCwd, + allowCustomCwd: callerContext?.allowCustomCwd ?? true, + }); const parentMode = parentAgent.currentModeId; if (parentMode) { resolvedMode = mapModeAcrossProviders( @@ -399,12 +491,11 @@ export async function createAgentMcpServer( ); } } else { - const topLevelArgs = args as unknown as { - cwd: string; - initialMode: string; - worktreeName?: string; - baseBranch?: string; - }; + const topLevelArgs = topLevelCreateAgentArgsSchema.parse(args); + provider = topLevelArgs.agentType ?? "claude"; + initialPrompt = topLevelArgs.initialPrompt; + background = topLevelArgs.background ?? false; + normalizedTitle = topLevelArgs.title.trim(); const { cwd, initialMode, @@ -412,7 +503,7 @@ export async function createAgentMcpServer( baseBranch, } = topLevelArgs; - resolvedCwd = expandPath(cwd); + resolvedCwd = expandUserPath(cwd); if (worktreeName) { if (!baseBranch) { @@ -431,75 +522,78 @@ export async function createAgentMcpServer( resolvedMode = initialMode; } - const provider: AgentProvider = agentType ?? "claude"; - const normalizedTitle = title?.trim() ?? null; - const snapshot = await agentManager.createAgent({ - provider, - cwd: resolvedCwd, - modeId: resolvedMode, - title: normalizedTitle ?? undefined, + const childAgentDefaultLabels = + callerAgentId && callerContext?.childAgentDefaultLabels + ? callerContext.childAgentDefaultLabels + : undefined; + const snapshot = await agentManager.createAgent( + { + provider, + cwd: resolvedCwd, + modeId: resolvedMode, + title: normalizedTitle ?? undefined, + }, + undefined, + childAgentDefaultLabels ? { labels: childAgentDefaultLabels } : undefined + ); + + const trimmedPrompt = initialPrompt.trim(); + scheduleAgentMetadataGeneration({ + agentManager, + agentId: snapshot.id, + cwd: snapshot.cwd, + initialPrompt: trimmedPrompt, + explicitTitle: snapshot.config.title, + paseoHome: options.paseoHome, + logger: childLogger, }); - const trimmedPrompt = initialPrompt?.trim(); - if (trimmedPrompt) { - scheduleAgentMetadataGeneration({ - agentManager, - agentId: snapshot.id, - cwd: snapshot.cwd, - initialPrompt: trimmedPrompt, - explicitTitle: snapshot.config.title, - paseoHome: options.paseoHome, - logger: childLogger, - }); - - try { - agentManager.recordUserMessage(snapshot.id, trimmedPrompt); - } catch (error) { - childLogger.error( - { err: error, agentId: snapshot.id }, - "Failed to record initial prompt" - ); - } - - try { - startAgentRun(agentManager, snapshot.id, trimmedPrompt, childLogger); - - // If not running in background, wait for completion - if (!background) { - const result = await waitForAgentWithTimeout( - agentManager, - snapshot.id, - { waitForActive: true } - ); - - const responseData = { - agentId: snapshot.id, - type: provider, - status: result.status, - cwd: snapshot.cwd, - currentModeId: snapshot.currentModeId, - availableModes: snapshot.availableModes, - lastMessage: result.lastMessage, - permission: sanitizePermissionRequest(result.permission), - }; - const validJson = ensureValidJson(responseData); - - const response = { - content: [], - structuredContent: validJson, - }; - return response; - } - } catch (error) { - childLogger.error( - { err: error, agentId: snapshot.id }, - "Failed to run initial prompt" - ); - } - } else { + try { + agentManager.recordUserMessage(snapshot.id, trimmedPrompt); + } catch (error) { + childLogger.error( + { err: error, agentId: snapshot.id }, + "Failed to record initial prompt" + ); } - // Return immediately if background=true or no initialPrompt + try { + startAgentRun(agentManager, snapshot.id, trimmedPrompt, childLogger); + + // If not running in background, wait for completion + if (!background) { + const result = await waitForAgentWithTimeout( + agentManager, + snapshot.id, + { waitForActive: true } + ); + + const responseData = { + agentId: snapshot.id, + type: provider, + status: result.status, + cwd: snapshot.cwd, + currentModeId: snapshot.currentModeId, + availableModes: snapshot.availableModes, + lastMessage: result.lastMessage, + permission: sanitizePermissionRequest(result.permission), + }; + const validJson = ensureValidJson(responseData); + + const response = { + content: [], + structuredContent: validJson, + }; + return response; + } + } catch (error) { + childLogger.error( + { err: error, agentId: snapshot.id }, + "Failed to run initial prompt" + ); + } + + // Return immediately if background=true const response = { content: [], structuredContent: ensureValidJson({ diff --git a/packages/server/src/server/agent/provider-manifest.ts b/packages/server/src/server/agent/provider-manifest.ts index 9f20d861f..189ac78f7 100644 --- a/packages/server/src/server/agent/provider-manifest.ts +++ b/packages/server/src/server/agent/provider-manifest.ts @@ -7,6 +7,11 @@ export interface AgentProviderDefinition { description: string; defaultModeId: string | null; modes: AgentMode[]; + voice?: { + enabled: boolean; + defaultModeId: string; + defaultModel?: string; + }; } const CLAUDE_MODES: AgentMode[] = [ @@ -68,6 +73,11 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [ "Anthropic's multi-tool assistant with MCP support, streaming, and deep reasoning", defaultModeId: "default", modes: CLAUDE_MODES, + voice: { + enabled: true, + defaultModeId: "default", + defaultModel: "haiku", + }, }, { id: "codex", @@ -76,6 +86,11 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [ "OpenAI's Codex workspace agent with sandbox controls and optional network access", defaultModeId: "auto", modes: CODEX_MODES, + voice: { + enabled: true, + defaultModeId: "read-only", + defaultModel: "gpt-5.2-mini", + }, }, { id: "opencode", @@ -84,6 +99,10 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [ "Open-source coding assistant with multi-provider model support", defaultModeId: "default", modes: OPENCODE_MODES, + voice: { + enabled: true, + defaultModeId: "default", + }, }, ]; diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 65834a8b6..404f01780 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -10,7 +10,16 @@ import { createTestAgentClients } from "./test-utils/fake-agent-client.js"; describe("paseo daemon bootstrap", () => { test("starts and serves health endpoint", async () => { - const daemonHandle = await createTestPaseoDaemon(); + const daemonHandle = await createTestPaseoDaemon({ + openai: { apiKey: "test-openai-api-key" }, + speech: { + providers: { + dictationStt: { provider: "openai", explicit: true }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "openai", explicit: true }, + }, + }, + }); try { const response = await fetch( `http://127.0.0.1:${daemonHandle.port}/api/health`, @@ -49,11 +58,12 @@ describe("paseo daemon bootstrap", () => { appBaseUrl: "https://app.paseo.sh", openai: undefined, speech: { - dictationSttProvider: "openai", - voiceSttProvider: "openai", - voiceTtsProvider: "openai", + providers: { + dictationStt: { provider: "openai", explicit: true }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "openai", explicit: true }, + }, }, - openrouterApiKey: null, }; try { diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index df688172f..11b5f2705 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -3,6 +3,7 @@ import { createServer as createHTTPServer } from "http"; import { createReadStream, unlinkSync, existsSync } from "fs"; import { stat } from "fs/promises"; import { randomUUID } from "node:crypto"; +import path from "node:path"; import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js"; @@ -40,20 +41,10 @@ function parseListenString(listen: string): ListenTarget { import { VoiceAssistantWebSocketServer } from "./websocket-server.js"; import { DownloadTokenStore } from "./file-download/token-store.js"; -import { OpenAISTT, type STTConfig } from "./speech/providers/openai/stt.js"; -import { OpenAITTS, type TTSConfig } from "./speech/providers/openai/tts.js"; -import { OpenAIRealtimeTranscriptionSession } from "./speech/providers/openai/realtime-transcription-session.js"; -import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; -import { SherpaOnlineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-online-recognizer.js"; -import { SherpaOfflineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-offline-recognizer.js"; -import { SherpaOnnxSTT } from "./speech/providers/local/sherpa/sherpa-stt.js"; -import { SherpaOnnxParakeetSTT } from "./speech/providers/local/sherpa/sherpa-parakeet-stt.js"; -import { SherpaOnnxTTS } from "./speech/providers/local/sherpa/sherpa-tts.js"; -import { SherpaRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-realtime-session.js"; -import { SherpaParakeetRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-parakeet-realtime-session.js"; -import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./speech/providers/local/sherpa/model-downloader.js"; -import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js"; -import { PocketTtsOnnxTTS } from "./speech/providers/local/pocket/pocket-tts-onnx.js"; +import type { OpenAiSpeechProviderConfig } from "./speech/providers/openai/config.js"; +import type { LocalSpeechProviderConfig } from "./speech/providers/local/config.js"; +import type { RequestedSpeechProviders } from "./speech/speech-types.js"; +import { initializeSpeechRuntime } from "./speech/speech-runtime.js"; import { AgentManager } from "./agent/agent-manager.js"; import { AgentStorage } from "./agent/agent-storage.js"; import { attachAgentStoragePersistence } from "./persistence-hooks.js"; @@ -73,35 +64,50 @@ import type { AgentClient, AgentProvider, } from "./agent/agent-sdk-types.js"; +import { AGENT_PROVIDER_DEFINITIONS } from "./agent/provider-manifest.js"; import { acquirePidLock, releasePidLock } from "./pid-lock.js"; import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js"; +import { createVoiceMcpBridgeSocketServer, type VoiceMcpBridgeSocketServer } from "./voice-mcp-bridge.js"; type AgentMcpTransportMap = Map; -export type PaseoOpenAIConfig = { - apiKey?: string; - stt?: Partial & { apiKey?: string }; - tts?: Partial & { apiKey?: string }; -}; +function resolveVoiceMcpBridgeCommand(logger: Logger): { command: string; baseArgs: string[] } { + const explicit = process.env.PASEO_BIN_PATH?.trim(); + if (explicit) { + const resolved = { command: explicit, baseArgs: ["__paseo_voice_mcp_bridge"] }; + logger.info({ source: "PASEO_BIN_PATH", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command"); + return resolved; + } -export type PaseoSherpaOnnxConfig = { - modelsDir: string; - autoDownload?: boolean; - stt?: { - preset?: string; - }; - tts?: { - preset?: string; - speakerId?: number; - speed?: number; - }; -}; + const argv1 = process.argv[1]?.trim(); + if (!argv1) { + logger.warn("Could not resolve argv[1] for voice MCP bridge; falling back to 'paseo'"); + const resolved = { command: "paseo", baseArgs: ["__paseo_voice_mcp_bridge"] }; + logger.info({ source: "fallback", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command"); + return resolved; + } + + const base = path.basename(argv1).toLowerCase(); + if (base.includes("tsx") && process.argv[2]) { + const resolved = { + command: process.execPath, + baseArgs: [argv1, process.argv[2], "__paseo_voice_mcp_bridge"], + }; + logger.info({ source: "tsx", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command"); + return resolved; + } + + const resolved = { command: argv1, baseArgs: ["__paseo_voice_mcp_bridge"] }; + logger.info({ source: "argv", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command"); + return resolved; +} + +export type PaseoOpenAIConfig = OpenAiSpeechProviderConfig; +export type PaseoLocalSpeechConfig = LocalSpeechProviderConfig; export type PaseoSpeechConfig = { - dictationSttProvider?: "openai" | "local"; - voiceSttProvider?: "openai" | "local"; - voiceTtsProvider?: "openai" | "local"; - sherpaOnnx?: PaseoSherpaOnnxConfig; + providers: RequestedSpeechProviders; + local?: PaseoLocalSpeechConfig; }; export type PaseoDaemonConfig = { @@ -120,7 +126,8 @@ export type PaseoDaemonConfig = { appBaseUrl?: string; openai?: PaseoOpenAIConfig; speech?: PaseoSpeechConfig; - openrouterApiKey?: string | null; + voiceLlmProvider?: AgentProvider | null; + voiceLlmProviderExplicit?: boolean; voiceLlmModel?: string | null; dictationFinalTimeoutMs?: number; downloadTokenTtlMs?: number; @@ -282,12 +289,110 @@ export async function createPaseoDaemon( `Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand` ); + const requestedVoiceLlmProvider = config.voiceLlmProvider ?? null; + const voiceLlmProviderExplicit = config.voiceLlmProviderExplicit ?? false; + const voiceEnabledProviders = AGENT_PROVIDER_DEFINITIONS + .filter((definition) => definition.voice?.enabled) + .map((definition) => definition.id as AgentProvider); + logger.info( + { + requestedVoiceLlmProvider, + voiceLlmProviderExplicit, + voiceEnabledProviders, + }, + "Voice LLM provider reconciliation started" + ); + + const providerClients = createAllClients(logger); + Object.assign(providerClients, config.agentClients); + const voiceLlmAvailability = Object.fromEntries( + voiceEnabledProviders.map((provider) => [provider, false]) + ) as Record; + for (const provider of voiceEnabledProviders) { + try { + voiceLlmAvailability[provider] = await providerClients[provider].isAvailable(); + } catch (error) { + logger.warn({ err: error, provider }, "Voice LLM provider availability check failed"); + voiceLlmAvailability[provider] = false; + } + } + + let resolvedVoiceLlmProvider: AgentProvider | null = null; + if (requestedVoiceLlmProvider) { + if (!voiceEnabledProviders.includes(requestedVoiceLlmProvider)) { + logger.error( + { provider: requestedVoiceLlmProvider, voiceEnabledProviders }, + "Configured voice LLM provider does not support voice mode" + ); + throw new Error( + `Configured voice LLM provider '${requestedVoiceLlmProvider}' does not support voice mode` + ); + } + if (!voiceLlmAvailability[requestedVoiceLlmProvider]) { + logger.error( + { provider: requestedVoiceLlmProvider, voiceLlmAvailability }, + "Configured voice LLM provider is unavailable" + ); + throw new Error(`Configured voice LLM provider '${requestedVoiceLlmProvider}' is unavailable`); + } + resolvedVoiceLlmProvider = requestedVoiceLlmProvider; + } else { + resolvedVoiceLlmProvider = + voiceEnabledProviders.find((provider) => voiceLlmAvailability[provider]) ?? null; + } + + let resolvedVoiceLlmModeId: string | null = null; + let resolvedVoiceLlmModel: string | null = null; + + if (!resolvedVoiceLlmProvider) { + if (voiceLlmProviderExplicit) { + logger.error( + { requestedVoiceLlmProvider, voiceLlmAvailability }, + "No voice LLM provider available" + ); + throw new Error("No voice LLM provider available"); + } + logger.warn( + { requestedVoiceLlmProvider, voiceLlmAvailability }, + "No default voice LLM provider available; voice mode will be disabled until a provider is configured" + ); + } else { + const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find( + (definition) => definition.id === resolvedVoiceLlmProvider + ); + if (!resolvedVoiceProviderDefinition?.voice?.enabled) { + throw new Error( + `Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry` + ); + } + resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId; + resolvedVoiceLlmModel = + config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null; + } + + logger.info( + { + requestedVoiceLlmProvider, + voiceLlmProviderExplicit, + resolvedVoiceLlmProvider, + resolvedVoiceLlmModeId, + resolvedVoiceLlmModel, + voiceLlmAvailability, + }, + "Voice LLM provider reconciliation completed" + ); + let wsServer: VoiceAssistantWebSocketServer | null = null; + let voiceMcpBridgeServer: VoiceMcpBridgeSocketServer | null = null; + // Create in-memory transport for Session's Agent MCP client (voice assistant tools) const createInMemoryAgentMcpTransport = async (): Promise => { const agentMcpServer = await createAgentMcpServer({ agentManager, agentStorage, paseoHome: config.paseoHome, + enableVoiceTools: false, + resolveSpeakHandler: (callerAgentId) => wsServer?.resolveVoiceSpeakHandler(callerAgentId) ?? null, + resolveCallerContext: (callerAgentId) => wsServer?.resolveVoiceCallerContext(callerAgentId) ?? null, logger, }); @@ -309,6 +414,9 @@ export async function createPaseoDaemon( agentStorage, paseoHome: config.paseoHome, callerAgentId, + enableVoiceTools: false, + resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null, + resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null, logger, }); @@ -414,385 +522,37 @@ export async function createPaseoDaemon( logger.info("Agent MCP HTTP endpoint disabled"); } - - let sttService: SpeechToTextProvider | null = null; - let ttsService: TextToSpeechProvider | null = null; - let dictationSttService: SpeechToTextProvider | null = null; - - let sherpaOnline: SherpaOnlineRecognizerEngine | null = null; - let sherpaOffline: SherpaOfflineRecognizerEngine | null = null; - let sherpaTts: TextToSpeechProvider | null = null; - - const openaiApiKey = config.openai?.apiKey; - const speechConfig = config.speech ?? null; - const sherpaConfig = speechConfig?.sherpaOnnx ?? null; - - const voiceSttProvider = speechConfig?.voiceSttProvider ?? "local"; - const voiceTtsProvider = speechConfig?.voiceTtsProvider ?? "local"; - const dictationSttProvider = speechConfig?.dictationSttProvider ?? "local"; - - const wantsLocalDictation = dictationSttProvider === "local"; - const wantsLocalVoiceStt = voiceSttProvider === "local"; - const wantsLocalVoiceTts = voiceTtsProvider === "local"; - - const openaiSttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey; - const openaiTtsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey; - const openaiDictationApiKey = openaiApiKey; - - const missingOpenAiCredentialsFor: string[] = []; - if (voiceSttProvider === "openai" && !openaiSttApiKey) { - missingOpenAiCredentialsFor.push("voice.stt"); - } - if (voiceTtsProvider === "openai" && !openaiTtsApiKey) { - missingOpenAiCredentialsFor.push("voice.tts"); - } - if (dictationSttProvider === "openai" && !openaiDictationApiKey) { - missingOpenAiCredentialsFor.push("dictation.stt"); - } - - if (missingOpenAiCredentialsFor.length > 0) { - logger.error( - { - requestedProviders: { - dictationStt: dictationSttProvider, - voiceStt: voiceSttProvider, - voiceTts: voiceTtsProvider, - }, - missingOpenAiCredentialsFor, - }, - "Invalid speech configuration: OpenAI provider selected but credentials are missing" - ); - throw new Error( - `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}` - ); - } - - logger.info( - { - requestedProviders: { - dictationStt: dictationSttProvider, - voiceStt: voiceSttProvider, - voiceTts: voiceTtsProvider, - }, - availability: { - openai: { - stt: Boolean(openaiSttApiKey), - tts: Boolean(openaiTtsApiKey), - dictationStt: Boolean(openaiDictationApiKey), - }, - local: { - configured: Boolean(sherpaConfig), - modelsDir: sherpaConfig?.modelsDir ?? null, - autoDownload: sherpaConfig?.autoDownload ?? null, - }, - }, - }, - "Speech provider reconciliation started" - ); - - if ((wantsLocalDictation || wantsLocalVoiceStt || wantsLocalVoiceTts) && sherpaConfig) { - const autoDownload = sherpaConfig.autoDownload ?? (process.env.VITEST ? false : true); - let sttPreset = (sherpaConfig.stt?.preset ?? "parakeet-tdt-0.6b-v3-int8").trim(); - if ( - sttPreset !== "zipformer-bilingual-zh-en-2023-02-20" && - sttPreset !== "paraformer-bilingual-zh-en" && - sttPreset !== "parakeet-tdt-0.6b-v3-int8" - ) { - throw new Error(`Unknown local STT preset: ${sttPreset}`); - } - - let ttsPreset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim(); - if ( - ttsPreset !== "kitten-nano-en-v0_1-fp16" && - ttsPreset !== "kokoro-en-v0_19" && - ttsPreset !== "pocket-tts-onnx-int8" - ) { - throw new Error(`Unknown local TTS preset: ${ttsPreset}`); - } - - const modelIds: SherpaOnnxModelId[] = []; - if (wantsLocalDictation || wantsLocalVoiceStt) { - modelIds.push(sttPreset as SherpaOnnxModelId); - } - if (wantsLocalVoiceTts) { - modelIds.push(ttsPreset as SherpaOnnxModelId); - } - - try { - logger.info( - { - modelsDir: sherpaConfig.modelsDir, - modelIds, - autoDownload, - }, - "Ensuring local speech models" - ); - await ensureSherpaOnnxModels({ - modelsDir: sherpaConfig.modelsDir, - modelIds, - autoDownload, + const voiceMcpSocketPath = path.join(config.paseoHome, "runtime", "voice-mcp.sock"); + const voiceMcpBridgeCommand = resolveVoiceMcpBridgeCommand(logger); + voiceMcpBridgeServer = createVoiceMcpBridgeSocketServer({ + socketPath: voiceMcpSocketPath, + logger, + createAgentMcpServerForCaller: async (callerAgentId) => { + return createAgentMcpServer({ + agentManager, + agentStorage, + paseoHome: config.paseoHome, + callerAgentId, + enableVoiceTools: false, + resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null, + resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null, logger, }); - } catch (err) { - logger.error( - { - err, - modelsDir: sherpaConfig.modelsDir, - autoDownload, - hint: - "Run: npm run dev --workspace=@getpaseo/server, then run: " + - "`tsx packages/server/scripts/download-speech-models.ts --models-dir --model `", - }, - "Failed to ensure local speech models" - ); - } - } + }, + }); + const { + sttService, + ttsService, + dictationSttService, + cleanup: cleanupSpeechRuntime, + localModelConfig, + } = await initializeSpeechRuntime({ + logger, + openaiConfig: config.openai, + speechConfig: config.speech, + }); - if ((wantsLocalDictation || wantsLocalVoiceStt) && sherpaConfig) { - let preset = (sherpaConfig.stt?.preset ?? "parakeet-tdt-0.6b-v3-int8").trim(); - if ( - preset !== "zipformer-bilingual-zh-en-2023-02-20" && - preset !== "paraformer-bilingual-zh-en" && - preset !== "parakeet-tdt-0.6b-v3-int8" - ) { - throw new Error(`Unknown local STT preset: ${preset}`); - } - const base = sherpaConfig.modelsDir; - - try { - if (preset === "parakeet-tdt-0.6b-v3-int8") { - const modelDir = getSherpaOnnxModelDir(base, "parakeet-tdt-0.6b-v3-int8"); - sherpaOffline = new SherpaOfflineRecognizerEngine( - { - model: { - kind: "nemo_transducer", - encoder: `${modelDir}/encoder.int8.onnx`, - decoder: `${modelDir}/decoder.int8.onnx`, - joiner: `${modelDir}/joiner.int8.onnx`, - tokens: `${modelDir}/tokens.txt`, - }, - numThreads: 2, - debug: 0, - }, - logger - ); - } else { - const model = - preset === "paraformer-bilingual-zh-en" - ? { - kind: "paraformer" as const, - encoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/encoder.int8.onnx`, - decoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/decoder.int8.onnx`, - tokens: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/tokens.txt`, - } - : { - kind: "transducer" as const, - encoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx`, - decoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx`, - joiner: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx`, - tokens: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt`, - modelType: "zipformer", - }; - - sherpaOnline = new SherpaOnlineRecognizerEngine( - { - model, - numThreads: 1, - debug: 0, - }, - logger - ); - } - } catch (err) { - logger.error( - { - err, - modelsDir: sherpaConfig.modelsDir, - preset, - hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`, - }, - "Failed to initialize Sherpa STT (models missing or invalid)" - ); - sherpaOnline = null; - sherpaOffline = null; - } - } else if (wantsLocalDictation || wantsLocalVoiceStt) { - logger.warn( - { configured: Boolean(sherpaConfig) }, - "Local STT selected but local provider config is missing; STT will be unavailable" - ); - } - - if (wantsLocalVoiceTts && sherpaConfig) { - let preset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim(); - if ( - preset !== "kitten-nano-en-v0_1-fp16" && - preset !== "kokoro-en-v0_19" && - preset !== "pocket-tts-onnx-int8" - ) { - throw new Error(`Unknown local TTS preset: ${preset}`); - } - try { - if (preset === "pocket-tts-onnx-int8") { - const modelDir = getSherpaOnnxModelDir(sherpaConfig.modelsDir, "pocket-tts-onnx-int8"); - sherpaTts = await PocketTtsOnnxTTS.create( - { - modelDir, - precision: "int8", - targetChunkMs: 50, - }, - logger - ); - } else { - const modelDir = `${sherpaConfig.modelsDir}/${preset}`; - sherpaTts = new SherpaOnnxTTS( - { - preset: preset as any, - modelDir, - speakerId: sherpaConfig.tts?.speakerId, - speed: sherpaConfig.tts?.speed, - }, - logger - ); - } - } catch (err) { - logger.error( - { - err, - preset, - hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`, - }, - "Failed to initialize Sherpa TTS (models missing or invalid)" - ); - sherpaTts = null; - } - } else if (wantsLocalVoiceTts) { - logger.warn( - { configured: Boolean(sherpaConfig) }, - "Local TTS selected but local provider config is missing; TTS will be unavailable" - ); - } - - if (wantsLocalVoiceStt && sherpaOffline) { - sttService = new SherpaOnnxParakeetSTT({ engine: sherpaOffline }, logger); - } else if (wantsLocalVoiceStt && sherpaOnline) { - sttService = new SherpaOnnxSTT({ engine: sherpaOnline }, logger); - } - - if (wantsLocalVoiceTts && sherpaTts) { - ttsService = sherpaTts; - } - - if (wantsLocalDictation && sherpaOnline) { - dictationSttService = { - id: "local", - createSession: () => new SherpaRealtimeTranscriptionSession({ engine: sherpaOnline! }), - }; - } else if (wantsLocalDictation && sherpaOffline) { - dictationSttService = { - id: "local", - createSession: () => - new SherpaParakeetRealtimeTranscriptionSession({ engine: sherpaOffline! }), - }; - } - - const needsOpenAiStt = !sttService && voiceSttProvider === "openai"; - const needsOpenAiTts = !ttsService && voiceTtsProvider === "openai"; - const needsOpenAiDictation = dictationSttProvider === "openai"; - - if ( - (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) && - (openaiSttApiKey || openaiTtsApiKey || openaiDictationApiKey) - ) { - logger.info("OpenAI speech provider initialized"); - - if (needsOpenAiStt) { - if (openaiSttApiKey) { - const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {}; - sttService = new OpenAISTT( - { - apiKey: openaiSttApiKey, - ...sttConfig, - }, - logger - ); - } - } - - if (needsOpenAiTts) { - if (openaiTtsApiKey) { - const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {}; - ttsService = new OpenAITTS( - { - apiKey: openaiTtsApiKey, - voice: "alloy", - model: "tts-1", - responseFormat: "pcm", - ...ttsConfig, - }, - logger - ); - } - } - - if (needsOpenAiDictation) { - const transcriptionModel = - process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe"; - - dictationSttService = { - id: "openai", - createSession: ({ logger: sessionLogger, language, prompt }) => - new OpenAIRealtimeTranscriptionSession({ - apiKey: openaiDictationApiKey!, - logger: sessionLogger, - transcriptionModel, - ...(language ? { language } : {}), - ...(prompt ? { prompt } : {}), - turnDetection: null, - }), - }; - } - } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) { - logger.warn("OPENAI_API_KEY not set - OpenAI STT/TTS/dictation is unavailable"); - } - - const effectiveProviders = { - dictationStt: dictationSttService?.id ?? "unavailable", - voiceStt: sttService?.id ?? "unavailable", - voiceTts: !ttsService ? "unavailable" : ttsService === sherpaTts ? "local" : "openai", - }; - const unavailableFeatures = [ - !dictationSttService ? "dictation.stt" : null, - !sttService ? "voice.stt" : null, - !ttsService ? "voice.tts" : null, - ].filter((feature): feature is string => feature !== null); - - if (unavailableFeatures.length > 0) { - logger.error( - { - requestedProviders: { - dictationStt: dictationSttProvider, - voiceStt: voiceSttProvider, - voiceTts: voiceTtsProvider, - }, - effectiveProviders, - unavailableFeatures, - }, - "Speech provider reconciliation failed: configured features are unavailable" - ); - throw new Error( - `Configured speech features unavailable: ${unavailableFeatures.join(", ")}` - ); - } else { - logger.info( - { - effectiveProviders, - }, - "Speech provider reconciliation completed" - ); - } - - const wsServer = new VoiceAssistantWebSocketServer( + wsServer = new VoiceAssistantWebSocketServer( httpServer, logger, serverId, @@ -805,12 +565,26 @@ export async function createPaseoDaemon( { stt: sttService, tts: ttsService }, terminalManager, { - openrouterApiKey: config.openrouterApiKey ?? null, - voiceLlmModel: config.voiceLlmModel ?? null, + voiceLlmProvider: resolvedVoiceLlmProvider, + voiceLlmModeId: resolvedVoiceLlmModeId, + voiceLlmProviderExplicit, + voiceLlmModel: resolvedVoiceLlmModel, + voiceAgentMcpStdio: { + command: voiceMcpBridgeCommand.command, + baseArgs: [ + ...voiceMcpBridgeCommand.baseArgs, + "--socket", + voiceMcpSocketPath, + ], + env: { + PASEO_HOME: config.paseoHome, + }, + }, }, { finalTimeoutMs: config.dictationFinalTimeoutMs, stt: dictationSttService, + localModels: localModelConfig ?? undefined, } ); @@ -870,7 +644,12 @@ export async function createPaseoDaemon( relayTransport?.stop().catch(() => undefined); relayTransport = startRelayTransport({ logger, - attachSocket: (ws) => wsServer.attachExternalSocket(ws), + attachSocket: (ws) => { + if (!wsServer) { + throw new Error("WebSocket server not initialized"); + } + return wsServer.attachExternalSocket(ws); + }, relayEndpoint, serverId, daemonKeyPair: daemonKeyPair.keyPair, @@ -896,6 +675,9 @@ export async function createPaseoDaemon( httpServer.listen(listenTarget.path); } }); + if (voiceMcpBridgeServer) { + await voiceMcpBridgeServer.start(); + } }; const stop = async () => { @@ -905,13 +687,14 @@ export async function createPaseoDaemon( await agentStorage.flush().catch(() => undefined); await shutdownProviders(logger); terminalManager.killAll(); - if (sherpaTts && typeof (sherpaTts as any).free === "function") { - (sherpaTts as any).free(); - } - sherpaOnline?.free(); - sherpaOffline?.free(); + cleanupSpeechRuntime(); await relayTransport?.stop().catch(() => undefined); - await wsServer.close(); + if (wsServer) { + await wsServer.close(); + } + if (voiceMcpBridgeServer) { + await voiceMcpBridgeServer.stop().catch(() => undefined); + } await new Promise((resolve) => { httpServer.close(() => resolve()); }); diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index 5f0825a05..e78863184 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -1,9 +1,11 @@ import path from "node:path"; +import { z } from "zod"; import type { PaseoDaemonConfig } from "./bootstrap.js"; -import type { STTConfig } from "./speech/providers/openai/stt.js"; -import type { TTSConfig } from "./speech/providers/openai/tts.js"; import { loadPersistedConfig } from "./persisted-config.js"; +import type { AgentProvider } from "./agent/agent-sdk-types.js"; +import { AgentProviderSchema } from "./agent/provider-manifest.js"; +import { resolveSpeechConfig } from "./speech/speech-config-resolver.js"; import { mergeAllowedHosts, parseAllowedHostsEnv, @@ -13,7 +15,6 @@ import { const DEFAULT_PORT = 6767; const DEFAULT_RELAY_ENDPOINT = "relay.paseo.sh:443"; const DEFAULT_APP_BASE_URL = "https://app.paseo.sh"; - function getDefaultListen(): string { // Main HTTP server defaults to TCP return `127.0.0.1:${DEFAULT_PORT}`; @@ -26,97 +27,14 @@ export type CliConfigOverrides = Partial<{ allowedHosts: AllowedHostsConfig; }>; -function parseOpenAIConfig( - env: NodeJS.ProcessEnv, - configApiKey: string | undefined, - config: { - dictationSttModel?: string; - dictationSttConfidenceThreshold?: number; - voiceSttModel?: string; - voiceTtsVoice?: TTSConfig["voice"]; - voiceTtsModel?: TTSConfig["model"]; - } -) { - const apiKey = env.OPENAI_API_KEY ?? configApiKey; - if (!apiKey) return undefined; +const OptionalVoiceLlmProviderSchema = z + .union([z.string(), z.null(), z.undefined()]) + .transform((value): string | null => (typeof value === "string" ? value.trim().toLowerCase() : null)) + .pipe(z.union([AgentProviderSchema, z.null()])); - const sttConfidenceThreshold = env.STT_CONFIDENCE_THRESHOLD - ? parseFloat(env.STT_CONFIDENCE_THRESHOLD) - : config.dictationSttConfidenceThreshold; - const sttModel = ( - env.STT_MODEL ?? - config.voiceSttModel ?? - config.dictationSttModel - ) as STTConfig["model"] | undefined; - const ttsVoice = (env.TTS_VOICE || "alloy") as - | "alloy" - | "echo" - | "fable" - | "onyx" - | "nova" - | "shimmer"; - const ttsModel = (env.TTS_MODEL || config.voiceTtsModel || "tts-1") as - | "tts-1" - | "tts-1-hd"; - const configuredVoice = config.voiceTtsVoice; - - return { - apiKey, - stt: { - apiKey, - confidenceThreshold: sttConfidenceThreshold, - ...(sttModel ? { model: sttModel } : {}), - }, - tts: { - apiKey, - voice: configuredVoice ?? ttsVoice, - model: ttsModel, - responseFormat: "pcm" as TTSConfig["responseFormat"], - }, - }; -} - -function parseSpeechProviderId(value: unknown): "openai" | "local" | null { - if (typeof value !== "string") { - return null; - } - const normalized = value.trim().toLowerCase(); - if (!normalized) { - return null; - } - if (normalized === "openai") return "openai"; - if (normalized === "local") return "local"; - return null; -} - -function normalizeSherpaSttPreset(value: string): string { - const raw = value.trim(); - const normalized = raw.toLowerCase(); - if (normalized === "zipformer" || normalized === "zipformer-bilingual") { - return "zipformer-bilingual-zh-en-2023-02-20"; - } - if (normalized === "paraformer") { - return "paraformer-bilingual-zh-en"; - } - if (normalized === "parakeet" || normalized === "parakeet-v3" || normalized === "parakeet-tdt") { - return "parakeet-tdt-0.6b-v3-int8"; - } - return raw; -} - -function normalizeSherpaTtsPreset(value: string): string { - const raw = value.trim(); - const normalized = raw.toLowerCase(); - if (normalized === "pocket" || normalized === "pocket-tts") { - return "pocket-tts-onnx-int8"; - } - if (normalized === "kitten") { - return "kitten-nano-en-v0_1-fp16"; - } - if (normalized === "kokoro") { - return "kokoro-en-v0_19"; - } - return raw; +function parseOptionalVoiceLlmProvider(value: unknown): AgentProvider | null { + const parsed = OptionalVoiceLlmProviderSchema.safeParse(value); + return parsed.success ? parsed.data : null; } export function loadConfig( @@ -171,81 +89,19 @@ export function loadConfig( const appBaseUrl = env.PASEO_APP_BASE_URL ?? persisted.app?.baseUrl ?? DEFAULT_APP_BASE_URL; - const openai = parseOpenAIConfig(env, persisted.providers?.openai?.apiKey, { - dictationSttModel: persisted.features?.dictation?.stt?.model, - dictationSttConfidenceThreshold: - persisted.features?.dictation?.stt?.confidenceThreshold, - voiceSttModel: persisted.features?.voiceMode?.stt?.model, - voiceTtsModel: persisted.features?.voiceMode?.tts?.model, - voiceTtsVoice: persisted.features?.voiceMode?.tts?.voice, + const { openai, speech } = resolveSpeechConfig({ + paseoHome, + env, + persisted, }); - const dictationSttProvider = - parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER) ?? - parseSpeechProviderId(persisted.features?.dictation?.stt?.provider) ?? - "local"; - - const voiceSttProvider = - parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER) ?? - parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider) ?? - "local"; - - const voiceTtsProvider = - parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER) ?? - parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider) ?? - "local"; - - const shouldConfigureSherpa = - dictationSttProvider === "local" || - voiceSttProvider === "local" || - voiceTtsProvider === "local" || - typeof env.PASEO_SHERPA_ONNX_MODELS_DIR === "string" || - Boolean(persisted.providers?.sherpaOnnx); - - const sherpaModelsDir = - (env.PASEO_SHERPA_ONNX_MODELS_DIR ?? persisted.providers?.sherpaOnnx?.modelsDir)?.trim() || - path.join(paseoHome, "models", "sherpa-onnx"); - - const sherpaOnnx = shouldConfigureSherpa - ? { - modelsDir: sherpaModelsDir, - autoDownload: - env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD !== undefined - ? env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD === "1" - : persisted.providers?.sherpaOnnx?.autoDownload ?? - // In tests we should never hit the network unexpectedly. - Boolean(env.VITEST) === false, - stt: { - preset: normalizeSherpaSttPreset( - (env.PASEO_SHERPA_STT_PRESET ?? persisted.providers?.sherpaOnnx?.stt?.preset)?.trim() || - (persisted.features?.voiceMode?.stt?.preset ?? - persisted.features?.dictation?.stt?.preset)?.trim() || - "parakeet-tdt-0.6b-v3-int8" - ), - }, - tts: { - preset: normalizeSherpaTtsPreset( - (env.PASEO_SHERPA_TTS_PRESET ?? - persisted.providers?.sherpaOnnx?.tts?.preset ?? - persisted.features?.voiceMode?.tts?.preset)?.trim() || - (env.VITEST ? "kitten-nano-en-v0_1-fp16" : "pocket-tts-onnx-int8") - ), - speakerId: - env.PASEO_SHERPA_TTS_SPEAKER_ID !== undefined - ? Number.parseInt(env.PASEO_SHERPA_TTS_SPEAKER_ID, 10) - : persisted.providers?.sherpaOnnx?.tts?.speakerId ?? - persisted.features?.voiceMode?.tts?.speakerId, - speed: - env.PASEO_SHERPA_TTS_SPEED !== undefined - ? Number.parseFloat(env.PASEO_SHERPA_TTS_SPEED) - : persisted.providers?.sherpaOnnx?.tts?.speed ?? - persisted.features?.voiceMode?.tts?.speed, - }, - } - : undefined; - - const openrouterApiKey = - env.OPENROUTER_API_KEY ?? persisted.providers?.openrouter?.apiKey ?? null; + const envVoiceLlmProvider = parseOptionalVoiceLlmProvider(env.PASEO_VOICE_LLM_PROVIDER); + const persistedVoiceLlmProvider = parseOptionalVoiceLlmProvider( + persisted.features?.voiceMode?.llm?.provider + ); + const voiceLlmProvider = envVoiceLlmProvider ?? persistedVoiceLlmProvider ?? null; + const voiceLlmProviderExplicit = + envVoiceLlmProvider !== null || persistedVoiceLlmProvider !== null; const voiceLlmModel = persisted.features?.voiceMode?.llm?.model ?? null; return { @@ -265,13 +121,9 @@ export function loadConfig( relayPublicEndpoint, appBaseUrl, openai, - speech: { - dictationSttProvider, - voiceSttProvider, - voiceTtsProvider, - ...(sherpaOnnx ? { sherpaOnnx } : {}), - }, - openrouterApiKey, + speech, + voiceLlmProvider, + voiceLlmProviderExplicit, voiceLlmModel, }; } diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index d1119f402..7cd6a0ef0 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, writeFileSync, rmSync, existsSync } from "node:fs"; import { tmpdir, homedir } from "node:os"; import path from "node:path"; import { execSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; import { createDaemonTestContext, @@ -18,9 +19,9 @@ import { const openaiApiKey = process.env.OPENAI_API_KEY ?? null; -const sherpaModelsDir = - process.env.PASEO_SHERPA_ONNX_MODELS_DIR ?? - path.join(homedir(), ".paseo", "models", "sherpa-onnx"); +const localModelsDir = + process.env.PASEO_LOCAL_MODELS_DIR ?? + path.join(homedir(), ".paseo", "models", "local-speech"); function hasSherpaZipformerModels(modelsDir: string): boolean { return ( @@ -49,7 +50,7 @@ function hasSherpaKittenModels(modelsDir: string): boolean { ); } -const hasLocalSpeech = hasSherpaZipformerModels(sherpaModelsDir) && hasSherpaKittenModels(sherpaModelsDir); +const hasLocalSpeech = hasSherpaZipformerModels(localModelsDir) && hasSherpaKittenModels(localModelsDir); const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey); const speechTest = hasAnySpeech ? test : test.skip; @@ -92,23 +93,42 @@ describe("daemon client E2E", () => { let ctx: DaemonTestContext; beforeAll(async () => { + const speechConfig = + openaiApiKey + ? { + providers: { + dictationStt: { provider: "openai" as const, explicit: true }, + voiceStt: { provider: "openai" as const, explicit: true }, + voiceTts: { provider: "openai" as const, explicit: true }, + }, + } + : hasLocalSpeech + ? { + providers: { + dictationStt: { provider: "local" as const, explicit: true }, + voiceStt: { provider: "local" as const, explicit: true }, + voiceTts: { provider: "local" as const, explicit: true }, + }, + local: { + modelsDir: localModelsDir, + models: { + dictationStt: + process.env.PASEO_DICTATION_LOCAL_STT_MODEL ?? + "zipformer-bilingual-zh-en-2023-02-20", + voiceStt: + process.env.PASEO_VOICE_LOCAL_STT_MODEL ?? + "zipformer-bilingual-zh-en-2023-02-20", + voiceTts: + process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16", + }, + }, + } + : undefined; + ctx = await createDaemonTestContext({ dictationFinalTimeoutMs: 5000, ...(openaiApiKey ? { openai: { apiKey: openaiApiKey } } : {}), - speech: { - dictationSttProvider: "local", - voiceSttProvider: "local", - voiceTtsProvider: "local", - sherpaOnnx: { - modelsDir: sherpaModelsDir, - stt: { - preset: process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20", - }, - tts: { - preset: process.env.PASEO_SHERPA_TTS_PRESET ?? "kitten-nano-en-v0_1-fp16", - }, - }, - }, + ...(speechConfig ? { speech: speechConfig } : {}), }); }, 60000); @@ -119,22 +139,18 @@ describe("daemon client E2E", () => { test("handles session actions", async () => { expect(ctx.client.isConnected).toBe(true); - const voiceConversationId = `voice-${Date.now()}`; - const loadResult = await ctx.client.loadVoiceConversation(voiceConversationId); - expect(loadResult.voiceConversationId).toBe(voiceConversationId); - expect(typeof loadResult.messageCount).toBe("number"); - const agents = await ctx.client.fetchAgents(); expect(Array.isArray(agents)).toBe(true); - const listResult = await ctx.client.listVoiceConversations(); - expect(Array.isArray(listResult.conversations)).toBe(true); + const voiceAgents = await ctx.client.fetchAgents({ + filter: { labels: { surface: "voice" } }, + }); + expect(Array.isArray(voiceAgents)).toBe(true); - const missingId = `missing-${Date.now()}`; - const deleteResult = await ctx.client.deleteVoiceConversation(missingId); - expect(deleteResult.voiceConversationId).toBe(missingId); - expect(deleteResult.success).toBe(false); - expect(deleteResult.error).toBeTruthy(); + await expect(ctx.client.setVoiceMode(true)).resolves.toBeUndefined(); + await expect(ctx.client.setVoiceMode(false)).resolves.toBeUndefined(); + + await ctx.client.deleteAgent(randomUUID()); }, 30000); test("emits server_info on websocket connect", async () => { @@ -160,19 +176,23 @@ describe("daemon client E2E", () => { await client.close(); }, 15000); - test("matches request IDs for concurrent session requests", async () => { - const firstRequestId = `list-${Date.now()}-a`; - const secondRequestId = `list-${Date.now()}-b`; + test("handles concurrent filtered agent fetch requests", async () => { + const firstRequestId = `fetch-${Date.now()}-a`; + const secondRequestId = `fetch-${Date.now()}-b`; const [first, second] = await Promise.all([ - ctx.client.listVoiceConversations(firstRequestId), - ctx.client.listVoiceConversations(secondRequestId), + ctx.client.fetchAgents({ + requestId: firstRequestId, + filter: { labels: { surface: "voice" } }, + }), + ctx.client.fetchAgents({ + requestId: secondRequestId, + filter: { labels: { surface: "voice" } }, + }), ]); - expect(Array.isArray(first.conversations)).toBe(true); - expect(Array.isArray(second.conversations)).toBe(true); - expect(first.requestId).toBe(firstRequestId); - expect(second.requestId).toBe(secondRequestId); + expect(Array.isArray(first)).toBe(true); + expect(Array.isArray(second)).toBe(true); }, 15000); test( @@ -355,7 +375,7 @@ describe("daemon client E2E", () => { expect(sawAssistantMessage).toBe(true); expect(sawRawAssistantMessage).toBe(true); - await ctx.client.setVoiceConversation(false); + await ctx.client.setVoiceMode(false); await ctx.client.abortRequest(); await ctx.client.audioPlayed("audio-1"); @@ -566,16 +586,16 @@ describe("daemon client E2E", () => { 120000 ); - test.runIf(Boolean(process.env.OPENROUTER_API_KEY))( - "streams session activity logs and chunks", + speechTest( + "does not process non-voice audio through the voice agent path", async () => { - await ctx.client.setVoiceConversation(false); + await ctx.client.setVoiceMode(false); - let sawAssistantChunk = false; let sawTranscriptLog = false; + let sawAssistantChunk = false; let sawAssistantLog = false; - const completion = waitForSignal(60000, (resolve) => { + const transcriptSeen = waitForSignal(60000, (resolve) => { const unsubscribeChunk = ctx.client.on("assistant_chunk", (message) => { if (message.type !== "assistant_chunk") { return; @@ -583,9 +603,6 @@ describe("daemon client E2E", () => { if (message.payload.chunk.length > 0) { sawAssistantChunk = true; } - if (sawAssistantChunk && sawTranscriptLog && sawAssistantLog) { - resolve(); - } }); const unsubscribeActivity = ctx.client.on("activity_log", (message) => { @@ -594,13 +611,11 @@ describe("daemon client E2E", () => { } if (message.payload.type === "transcript") { sawTranscriptLog = true; + resolve(); } if (message.payload.type === "assistant") { sawAssistantLog = true; } - if (sawAssistantChunk && sawTranscriptLog && sawAssistantLog) { - resolve(); - } }); return () => { @@ -609,16 +624,30 @@ describe("daemon client E2E", () => { }; }); - await ctx.client.sendUserMessage("Say 'hello' and nothing else"); - await completion; + const fixturePath = path.resolve( + process.cwd(), + "..", + "app", + "e2e", + "fixtures", + "recording.wav" + ); + const wav = await import("node:fs/promises").then((fs) => fs.readFile(fixturePath)); + await ctx.client.sendVoiceAudioChunk(wav.toString("base64"), "audio/wav", true); + await transcriptSeen; + await new Promise((resolve) => setTimeout(resolve, 1500)); + + expect(sawTranscriptLog).toBe(true); + expect(sawAssistantChunk).toBe(false); + expect(sawAssistantLog).toBe(false); }, - 120000 + 90000 ); speechTest( "voice mode buffers audio until isLast and emits transcription_result", async () => { - await ctx.client.setVoiceConversation(true, `voice-${Date.now()}`); + await ctx.client.setVoiceMode(true, randomUUID()); const transcription = waitForSignal(30_000, (resolve) => { const unsubscribe = ctx.client.on("transcription_result", (message) => { @@ -713,7 +742,7 @@ describe("daemon client E2E", () => { } } finally { await Promise.allSettled([transcription, errorSignal]); - await ctx.client.setVoiceConversation(false); + await ctx.client.setVoiceMode(false); } }, 90_000 diff --git a/packages/server/src/server/daemon-e2e/connection-offer.e2e.test.ts b/packages/server/src/server/daemon-e2e/connection-offer.e2e.test.ts index 3ba4cda14..6913d0c88 100644 --- a/packages/server/src/server/daemon-e2e/connection-offer.e2e.test.ts +++ b/packages/server/src/server/daemon-e2e/connection-offer.e2e.test.ts @@ -186,6 +186,7 @@ describe("ConnectionOfferV2 (daemon E2E)", () => { PASEO_HOME: tempHome, PASEO_LISTEN: `0.0.0.0:${port}`, OPENAI_API_KEY: "", + PASEO_LOCAL_AUTO_DOWNLOAD: "0", PASEO_LOG_FORMAT: "json", }; diff --git a/packages/server/src/server/exports.ts b/packages/server/src/server/exports.ts index 31262a6a0..c0779c865 100644 --- a/packages/server/src/server/exports.ts +++ b/packages/server/src/server/exports.ts @@ -5,6 +5,7 @@ export { resolvePaseoHome } from "./paseo-home.js"; export { createRootLogger, type LogLevel, type LogFormat } from "./logger.js"; export { loadPersistedConfig, type PersistedConfig } from "./persisted-config.js"; export { DaemonClient, type DaemonClientConfig, type ConnectionState, type DaemonEvent } from "../client/daemon-client.js"; +export { runVoiceMcpBridgeCli } from "./voice-mcp-bridge.js"; // Agent SDK types for CLI commands export type { diff --git a/packages/server/src/server/index.ts b/packages/server/src/server/index.ts index 7737855f8..60c91a602 100644 --- a/packages/server/src/server/index.ts +++ b/packages/server/src/server/index.ts @@ -10,8 +10,15 @@ import { resolvePaseoHome } from "./paseo-home.js"; import { createRootLogger } from "./logger.js"; import { loadPersistedConfig } from "./persisted-config.js"; import { PidLockError } from "./pid-lock.js"; +import { runVoiceMcpBridgeCli } from "./voice-mcp-bridge.js"; async function main() { + const bridgeArgIndex = process.argv.findIndex((arg) => arg === "__paseo_voice_mcp_bridge"); + if (bridgeArgIndex >= 0) { + await runVoiceMcpBridgeCli(process.argv.slice(bridgeArgIndex + 1)); + return; + } + let paseoHome: string; let logger: ReturnType; let config: ReturnType; diff --git a/packages/server/src/server/path-utils.ts b/packages/server/src/server/path-utils.ts new file mode 100644 index 000000000..76403ef36 --- /dev/null +++ b/packages/server/src/server/path-utils.ts @@ -0,0 +1,22 @@ +import { homedir } from "node:os"; +import { isAbsolute, resolve } from "node:path"; + +function hasHomePrefix(value: string): boolean { + return value === "~" || value.startsWith("~/"); +} + +export function expandUserPath(value: string): string { + const trimmed = value.trim(); + if (hasHomePrefix(trimmed)) { + return resolve(homedir(), trimmed.slice(2)); + } + return resolve(trimmed); +} + +export function resolvePathFromBase(baseCwd: string, requestedPath: string): string { + const trimmed = requestedPath.trim(); + if (hasHomePrefix(trimmed) || isAbsolute(trimmed)) { + return expandUserPath(trimmed); + } + return resolve(baseCwd, trimmed); +} diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index 4b34cbbcd..a9fc694f6 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -1,6 +1,7 @@ import { existsSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; import { z } from "zod"; +import { AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js"; const LogConfigSchema = z .object({ @@ -17,44 +18,25 @@ const ProviderCredentialsSchema = z }) .strict(); -const SherpaOnnxProviderSchema = z +const LocalSpeechProviderSchema = z .object({ modelsDir: z.string().min(1).optional(), autoDownload: z.boolean().optional(), - stt: z - .object({ - preset: z.string().min(1).optional(), - }) - .strict() - .optional(), - tts: z - .object({ - preset: z.string().min(1).optional(), - speakerId: z.number().int().optional(), - speed: z.number().optional(), - }) - .strict() - .optional(), }) .strict(); const ProvidersSchema = z .object({ openai: ProviderCredentialsSchema.optional(), - openrouter: ProviderCredentialsSchema.optional(), - sherpaOnnx: SherpaOnnxProviderSchema.optional(), + local: LocalSpeechProviderSchema.optional(), }) .strict(); -const SpeechProviderIdSchema = z.preprocess( - (value) => { - if (typeof value !== "string") { - return value; - } - return value.trim().toLowerCase(); - }, - z.enum(["openai", "local"]) -); +const SpeechProviderIdSchema = z + .string() + .trim() + .toLowerCase() + .pipe(z.enum(["openai", "local"])); const FeatureDictationSchema = z .object({ @@ -62,7 +44,6 @@ const FeatureDictationSchema = z .object({ provider: SpeechProviderIdSchema.optional(), model: z.string().min(1).optional(), - preset: z.string().min(1).optional(), confidenceThreshold: z.number().optional(), }) .strict() @@ -74,7 +55,7 @@ const FeatureVoiceModeSchema = z .object({ llm: z .object({ - provider: z.enum(["openrouter"]).optional(), + provider: z.enum(AGENT_PROVIDER_IDS as [string, ...string[]]).optional(), model: z.string().min(1).optional(), }) .strict() @@ -83,16 +64,14 @@ const FeatureVoiceModeSchema = z .object({ provider: SpeechProviderIdSchema.optional(), model: z.string().min(1).optional(), - preset: z.string().min(1).optional(), }) .strict() .optional(), tts: z .object({ provider: SpeechProviderIdSchema.optional(), - model: z.enum(["tts-1", "tts-1-hd"]).optional(), + model: z.string().min(1).optional(), voice: z.enum(["alloy", "echo", "fable", "onyx", "nova", "shimmer"]).optional(), - preset: z.string().min(1).optional(), speakerId: z.number().int().optional(), speed: z.number().optional(), }) diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 7cff12a80..2465a5807 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -1,17 +1,10 @@ import { v4 as uuidv4 } from "uuid"; -import { readFile, mkdir, writeFile, stat } from "fs/promises"; +import { mkdir, stat } from "fs/promises"; import { exec } from "child_process"; -import { promisify, inspect } from "util"; +import { promisify } from "util"; import { join, resolve, sep } from "path"; -import invariant from "tiny-invariant"; import { z } from "zod"; -import { streamText, stepCountIs } from "ai"; import type { ToolSet } from "ai"; -import type { ModelMessage } from "@ai-sdk/provider-utils"; -import { - createOpenRouter, - OpenRouterProviderOptions, -} from "@openrouter/ai-sdk-provider"; import { serializeAgentStreamEvent, type AgentSnapshotPayload, @@ -29,8 +22,6 @@ import { } from "./messages.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; import { parseAndHighlightDiff, type ParsedDiffFile } from "./utils/diff-highlighter.js"; -import { getSystemPrompt } from "./agent/system-prompt.js"; -import { getAllTools } from "./agent/llm-openai.js"; import { TTSManager } from "./agent/tts-manager.js"; import { STTManager } from "./agent/stt-manager.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; @@ -39,7 +30,6 @@ import { isPaseoDictationDebugEnabled } from "./agent/recordings-debug.js"; import { DictationStreamManager, } from "./dictation/dictation-stream-manager.js"; -import type { VoiceConversationStore } from "./voice-conversation-store.js"; import { buildConfigOverrides, buildSessionConfig, @@ -47,6 +37,11 @@ import { } from "./persistence-hooks.js"; import { experimental_createMCPClient } from "ai"; import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js"; +import type { + VoiceCallerContext, + VoiceMcpStdioConfig, + VoiceSpeakHandler, +} from "./voice-types.js"; export type AgentMcpTransportFactory = () => Promise; import { buildProviderRegistry } from "./agent/provider-registry.js"; @@ -59,6 +54,7 @@ import { generateStructuredAgentResponse, } from "./agent/agent-response-loop.js"; import type { + AgentPermissionRequest, AgentPermissionResponse, AgentPromptContentBlock, AgentPromptInput, @@ -70,6 +66,7 @@ import type { } from "./agent/agent-sdk-types.js"; import { AgentStorage, type StoredAgentRecord } from "./agent/agent-storage.js"; import { isValidAgentProvider, AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js"; +import { isVoicePermissionAllowed } from "./voice-permission-policy.js"; import { listDirectoryEntries, readExplorerFile, @@ -106,13 +103,11 @@ import { import { getProjectIcon } from "../utils/project-icon.js"; import { expandTilde } from "../utils/path.js"; import { - ensureSherpaOnnxModels, - getSherpaOnnxModelDir, -} from "./speech/providers/local/sherpa/model-downloader.js"; -import { - listSherpaOnnxModels, - type SherpaOnnxModelId, -} from "./speech/providers/local/sherpa/model-catalog.js"; + ensureLocalSpeechModels, + getLocalSpeechModelDir, + listLocalSpeechModels, + type LocalSpeechModelId, +} from "./speech/providers/local/models.js"; import type pino from "pino"; const execAsync = promisify(exec); @@ -130,6 +125,33 @@ const RESTART_EXIT_DELAY_MS = 250; * Uses Claude Haiku for speed and cost efficiency. */ const AUTO_GEN_MODEL = "haiku"; +const VOICE_AGENT_SYSTEM_INSTRUCTION = [ + "You are the Paseo voice assistant.", + "The user cannot see your chat messages or tool calls.", + "Always narrate everything through the speak tool.", + "Use concise plain language suitable for speech output.", + "Never use bash, file-edit, or web tools directly.", + "Only use the paseo MCP tools.", +].join(" "); + +export function buildVoiceAgentMcpServerConfig(params: { + callerAgentId: string; + command: string; + baseArgs: string[]; + env?: Record; +}): { + type: "stdio"; + command: string; + args: string[]; + env?: Record; +} { + return { + type: "stdio", + command: params.command, + args: [...params.baseArgs, "--caller-agent-id", params.callerAgentId], + ...(params.env ? { env: params.env } : {}), + }; +} type ProcessingPhase = "idle" | "transcribing" | "llm"; @@ -158,18 +180,7 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round( PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS ); const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/; - -/** - * Type for present_artifact tool arguments - */ -interface PresentArtifactArgs { - type: "markdown" | "diff" | "image" | "code"; - source: - | { type: "file"; path: string } - | { type: "command_output"; command: string } - | { type: "text"; text: string }; - title: string; -} +const VoiceAgentIdSchema = z.string().uuid(); interface AudioBufferState { chunks: Buffer[]; @@ -178,6 +189,42 @@ interface AudioBufferState { totalPCMBytes: number; } +export type SessionOptions = { + clientId: string; + onMessage: (msg: SessionOutboundMessage) => void; + logger: pino.Logger; + downloadTokenStore: DownloadTokenStore; + pushTokenStore: PushTokenStore; + paseoHome: string; + agentManager: AgentManager; + agentStorage: AgentStorage; + createAgentMcpTransport: AgentMcpTransportFactory; + stt: SpeechToTextProvider | null; + tts: TextToSpeechProvider | null; + terminalManager: TerminalManager | null; + voice?: { + voiceLlmProvider?: AgentProvider | null; + voiceLlmModeId?: string | null; + voiceLlmProviderExplicit?: boolean; + voiceLlmModel?: string | null; + voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; + }; + voiceBridge?: { + registerVoiceSpeakHandler?: (agentId: string, handler: VoiceSpeakHandler) => void; + unregisterVoiceSpeakHandler?: (agentId: string) => void; + registerVoiceCallerContext?: (agentId: string, context: VoiceCallerContext) => void; + unregisterVoiceCallerContext?: (agentId: string) => void; + }; + dictation?: { + finalTimeoutMs?: number; + stt?: SpeechToTextProvider | null; + localModels?: { + modelsDir: string; + defaultModelIds: LocalSpeechModelId[]; + }; + }; +}; + function convertPCMToWavBuffer( pcmBuffer: Buffer, sampleRate: number, @@ -250,7 +297,7 @@ function toAgentPersistenceHandle( } /** - * Session represents a single client conversation session. + * Session represents a single connected client session. * It owns all state management, orchestration logic, and message processing. * Session has no knowledge of WebSockets - it only emits and receives messages. */ @@ -259,7 +306,6 @@ export class Session { private readonly sessionId: string; private readonly onMessage: (msg: SessionOutboundMessage) => void; private readonly sessionLogger: pino.Logger; - private readonly voiceConversationStore: VoiceConversationStore; private readonly paseoHome: string; // State machine @@ -270,7 +316,6 @@ export class Session { // Voice mode state private isVoiceMode = false; private speechInProgress = false; - private voiceConversationId: string | null = null; private readonly dictationStreamManager: DictationStreamManager; @@ -286,7 +331,6 @@ export class Session { >(); // Conversation history - private messages: ModelMessage[] = []; private turnIndex = 0; // Per-session managers @@ -319,32 +363,43 @@ export class Session { } | null = null; private readonly terminalManager: TerminalManager | null; private terminalSubscriptions: Map void> = new Map(); - private readonly openrouterApiKey: string | null; + private readonly voiceLlmProvider: AgentProvider | null; + private readonly voiceLlmModeId: string | null; + private readonly voiceLlmProviderExplicit: boolean; private readonly voiceLlmModel: string | null; + private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null; + private readonly localSpeechModelsDir: string; + private readonly defaultLocalSpeechModelIds: LocalSpeechModelId[]; + private readonly registerVoiceSpeakHandler?: ( + agentId: string, + handler: VoiceSpeakHandler + ) => void; + private readonly unregisterVoiceSpeakHandler?: (agentId: string) => void; + private readonly registerVoiceCallerContext?: ( + agentId: string, + context: VoiceCallerContext + ) => void; + private readonly unregisterVoiceCallerContext?: (agentId: string) => void; + private voiceAssistantAgentId: string | null = null; - constructor( - clientId: string, - onMessage: (msg: SessionOutboundMessage) => void, - logger: pino.Logger, - downloadTokenStore: DownloadTokenStore, - pushTokenStore: PushTokenStore, - paseoHome: string, - agentManager: AgentManager, - agentStorage: AgentStorage, - createAgentMcpTransport: AgentMcpTransportFactory, - stt: SpeechToTextProvider | null, - tts: TextToSpeechProvider | null, - terminalManager: TerminalManager | null, - voiceConversationStore: VoiceConversationStore, - voice?: { - openrouterApiKey?: string | null; - voiceLlmModel?: string | null; - }, - dictation?: { - finalTimeoutMs?: number; - stt?: SpeechToTextProvider | null; - } - ) { + constructor(options: SessionOptions) { + const { + clientId, + onMessage, + logger, + downloadTokenStore, + pushTokenStore, + paseoHome, + agentManager, + agentStorage, + createAgentMcpTransport, + stt, + tts, + terminalManager, + voice, + voiceBridge, + dictation, + } = options; this.clientId = clientId; this.sessionId = uuidv4(); this.onMessage = onMessage; @@ -355,9 +410,24 @@ export class Session { this.agentStorage = agentStorage; this.createAgentMcpTransport = createAgentMcpTransport; this.terminalManager = terminalManager; - this.voiceConversationStore = voiceConversationStore; - this.openrouterApiKey = voice?.openrouterApiKey ?? null; + this.voiceLlmProvider = voice?.voiceLlmProvider ?? null; + this.voiceLlmModeId = voice?.voiceLlmModeId ?? null; + this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false; this.voiceLlmModel = voice?.voiceLlmModel ?? null; + this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null; + const configuredModelsDir = dictation?.localModels?.modelsDir?.trim(); + this.localSpeechModelsDir = + configuredModelsDir && configuredModelsDir.length > 0 + ? configuredModelsDir + : join(this.paseoHome, "models", "local-speech"); + this.defaultLocalSpeechModelIds = + dictation?.localModels?.defaultModelIds && dictation.localModels.defaultModelIds.length > 0 + ? [...new Set(dictation.localModels.defaultModelIds)] + : ["parakeet-tdt-0.6b-v3-int8", "pocket-tts-onnx-int8"]; + this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler; + this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler; + this.registerVoiceCallerContext = voiceBridge?.registerVoiceCallerContext; + this.unregisterVoiceCallerContext = voiceBridge?.unregisterVoiceCallerContext; this.abortController = new AbortController(); this.sessionLogger = logger.child({ module: "session", @@ -384,28 +454,6 @@ export class Session { this.sessionLogger.info("Session created"); } - private escapeXmlText(value: string): string { - return value - .replace(/&/g, "&") - .replace(//g, ">"); - } - - private escapeXmlAttribute(value: string): string { - return this.escapeXmlText(value) - .replace(/"/g, """) - .replace(/'/g, "'"); - } - - private formatVoiceTranscriptionForLLM(text: string): string { - const trimmed = text.trim(); - const focusedAgentId = this.clientActivity?.focusedAgentId ?? null; - const focusedAttr = focusedAgentId - ? ` focused-agent-id="${this.escapeXmlAttribute(focusedAgentId)}"` - : ""; - return `${this.escapeXmlText(trimmed)}`; - } - /** * Get the client's current activity state */ @@ -787,10 +835,6 @@ export class Session { public async handleMessage(msg: SessionInboundMessage): Promise { try { switch (msg.type) { - case "user_text": - await this.handleUserText(msg.text); - break; - case "voice_audio_chunk": await this.handleAudioChunk(msg); break; @@ -826,21 +870,6 @@ export class Session { } break; - case "load_voice_conversation_request": - await this.handleLoadVoiceConversation(msg.voiceConversationId, msg.requestId); - break; - - case "list_voice_conversations_request": - await this.handleListVoiceConversations(msg.requestId); - break; - - case "delete_voice_conversation_request": - await this.handleDeleteVoiceConversation( - msg.voiceConversationId, - msg.requestId - ); - break; - case "delete_agent_request": await this.handleDeleteAgentRequest(msg.agentId, msg.requestId); break; @@ -849,8 +878,8 @@ export class Session { await this.handleArchiveAgentRequest(msg.agentId, msg.requestId); break; - case "set_voice_conversation": - await this.handleSetVoiceConversation(msg.enabled, msg.voiceConversationId); + case "set_voice_mode": + await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId); break; case "send_agent_message_request": @@ -1105,103 +1134,6 @@ export class Session { } } - /** - * Load a voice conversation into this session (best-effort). - */ - public async handleLoadVoiceConversation( - voiceConversationId: string, - requestId: string - ): Promise { - const loaded = await this.voiceConversationStore.load( - this.sessionLogger, - voiceConversationId - ); - - this.voiceConversationId = voiceConversationId; - this.messages = loaded ?? []; - - this.emit({ - type: "voice_conversation_loaded", - payload: { - voiceConversationId, - messageCount: this.messages.length, - requestId, - }, - }); - } - - /** - * List all voice conversations - */ - public async handleListVoiceConversations(requestId: string): Promise { - try { - const conversations = await this.voiceConversationStore.list(this.sessionLogger); - this.emit({ - type: "list_voice_conversations_response", - payload: { - conversations: conversations.map((conv) => ({ - id: conv.id, - lastUpdated: conv.lastUpdated.toISOString(), - messageCount: conv.messageCount, - })), - requestId, - }, - }); - } catch (error: any) { - this.sessionLogger.error( - { err: error }, - "Failed to list voice conversations" - ); - this.emit({ - type: "activity_log", - payload: { - id: uuidv4(), - timestamp: new Date(), - type: "error", - content: `Failed to list voice conversations: ${error.message}`, - }, - }); - } - } - - /** - * Delete a voice conversation - */ - public async handleDeleteVoiceConversation( - voiceConversationId: string, - requestId: string - ): Promise { - try { - await this.voiceConversationStore.delete(this.sessionLogger, voiceConversationId); - this.emit({ - type: "delete_voice_conversation_response", - payload: { - voiceConversationId, - success: true, - requestId, - }, - }); - this.sessionLogger.info( - { voiceConversationId }, - `Deleted voice conversation ${voiceConversationId}` - ); - } catch (error: any) { - this.sessionLogger.error( - { err: error, voiceConversationId }, - `Failed to delete voice conversation ${voiceConversationId}` - ); - this.emit({ - type: "delete_voice_conversation_response", - payload: { - voiceConversationId, - success: false, - error: error.message, - requestId, - }, - }); - } - } - private async handleRestartServerRequest( requestId: string, reason?: string @@ -1325,47 +1257,74 @@ export class Session { /** * Handle voice mode toggle */ - private async handleSetVoiceConversation( + private async handleSetVoiceMode( enabled: boolean, - voiceConversationId?: string + voiceAgentId?: string ): Promise { if (enabled) { - if (!voiceConversationId || voiceConversationId.trim().length === 0) { - this.sessionLogger.warn("set_voice_conversation missing voiceConversationId; ignoring"); - return; + let normalizedVoiceAgentId: string | null = null; + if (voiceAgentId && voiceAgentId.trim().length > 0) { + normalizedVoiceAgentId = this.parseVoiceAgentId( + voiceAgentId, + "set_voice_mode" + ); + } else { + normalizedVoiceAgentId = await this.findExistingVoiceAssistantAgentId(); } this.isVoiceMode = true; - this.voiceConversationId = voiceConversationId; - - const loaded = await this.voiceConversationStore.load( - this.sessionLogger, - voiceConversationId - ); - this.messages = loaded ?? []; - + this.voiceAssistantAgentId = normalizedVoiceAgentId; this.sessionLogger.info( - { voiceConversationId, messageCount: this.messages.length }, - "Voice conversation enabled" + { + voiceAssistantAgentId: this.voiceAssistantAgentId, + resumedVoiceAgent: Boolean(normalizedVoiceAgentId), + providedVoiceAgentId: Boolean(voiceAgentId?.trim()), + }, + "Voice mode enabled (agent-backed)" ); return; } this.isVoiceMode = false; - const idToPersist = this.voiceConversationId; - if (idToPersist) { - try { - await this.voiceConversationStore.save( - this.sessionLogger, - idToPersist, - this.messages - ); - } catch (error) { - this.sessionLogger.warn({ err: error }, "Failed to persist voice conversation"); - } - } + this.sessionLogger.info( + { voiceAssistantAgentId: this.voiceAssistantAgentId }, + "Voice mode disabled (agent-backed)" + ); + } - this.sessionLogger.info({ voiceConversationId: idToPersist }, "Voice conversation disabled"); + private parseVoiceAgentId(rawId: string, source: string): string { + const parsed = VoiceAgentIdSchema.safeParse(rawId.trim()); + if (!parsed.success) { + throw new Error(`${source}: voiceAgentId must be a UUID`); + } + return parsed.data; + } + + private async findExistingVoiceAssistantAgentId(): Promise { + const scopedCandidates = await this.listAgentPayloads({ + labels: { + surface: "voice", + voiceClientId: this.clientId, + }, + }); + const fallbackCandidates = await this.listAgentPayloads({ + labels: { surface: "voice" }, + }); + const available = [ + ...scopedCandidates, + ...fallbackCandidates.filter( + (candidate) => !scopedCandidates.some((scoped) => scoped.id === candidate.id) + ), + ].filter((agent) => !agent.archivedAt); + if (available.length === 0) { + return null; + } + available.sort((left, right) => { + const leftTime = Date.parse(left.updatedAt); + const rightTime = Date.parse(right.updatedAt); + return (Number.isFinite(rightTime) ? rightTime : 0) - (Number.isFinite(leftTime) ? leftTime : 0); + }); + return available[0]?.id ?? null; } /** @@ -1482,24 +1441,6 @@ export class Session { ); try { - // Validate that the working directory exists - const resolvedCwd = expandTilde(config.cwd); - try { - const stats = await stat(resolvedCwd); - if (!stats.isDirectory()) { - throw new Error( - `Working directory is not a directory: ${config.cwd}` - ); - } - } catch (statError: any) { - if (statError.code === "ENOENT") { - throw new Error( - `Working directory does not exist: ${config.cwd}` - ); - } - throw statError; - } - const { sessionConfig, worktreeConfig } = await this.buildAgentSessionConfig( config, git, @@ -1944,13 +1885,11 @@ export class Session { private async handleSpeechModelsListRequest( msg: Extract ): Promise { - const modelsDir = - process.env.PASEO_SHERPA_ONNX_MODELS_DIR?.trim() || - join(this.paseoHome, "models", "sherpa-onnx"); + const modelsDir = this.localSpeechModelsDir; const models = await Promise.all( - listSherpaOnnxModels().map(async (model) => { - const modelDir = getSherpaOnnxModelDir(modelsDir, model.id); + listLocalSpeechModels().map(async (model) => { + const modelDir = getLocalSpeechModelDir(modelsDir, model.id); const missingFiles: string[] = []; for (const rel of model.requiredFiles) { const filePath = join(modelDir, rel); @@ -1991,20 +1930,15 @@ export class Session { private async handleSpeechModelsDownloadRequest( msg: Extract ): Promise { - const modelsDir = - process.env.PASEO_SHERPA_ONNX_MODELS_DIR?.trim() || - join(this.paseoHome, "models", "sherpa-onnx"); + const modelsDir = this.localSpeechModelsDir; const modelIdsRaw = msg.modelIds && msg.modelIds.length > 0 ? msg.modelIds - : [ - process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20", - process.env.PASEO_SHERPA_TTS_PRESET ?? "pocket-tts-onnx-int8", - ]; + : this.defaultLocalSpeechModelIds; - const allModelIds = new Set(listSherpaOnnxModels().map((m) => m.id)); - const invalid = modelIdsRaw.filter((id) => !allModelIds.has(id as SherpaOnnxModelId)); + const allModelIds = new Set(listLocalSpeechModels().map((m) => m.id)); + const invalid = modelIdsRaw.filter((id) => !allModelIds.has(id as LocalSpeechModelId)); if (invalid.length > 0) { this.emit({ type: "speech_models_download_response", @@ -2018,9 +1952,9 @@ export class Session { return; } - const modelIds = modelIdsRaw as SherpaOnnxModelId[]; + const modelIds = modelIdsRaw as LocalSpeechModelId[]; try { - await ensureSherpaOnnxModels({ + await ensureLocalSpeechModels({ modelsDir, modelIds, autoDownload: true, @@ -4212,39 +4146,6 @@ export class Session { return timeline.length; } - /** - * Handle text message from user - */ - private async handleUserText(text: string): Promise { - // Abort any in-progress stream immediately - this.createAbortController(); - - // Wait for aborted stream to finish cleanup (save partial response) - if (this.currentStreamPromise) { - this.sessionLogger.debug("Waiting for aborted stream to finish cleanup"); - await this.currentStreamPromise; - this.sessionLogger.debug("Aborted stream finished cleanup"); - } - - // Emit user message activity log - this.emit({ - type: "activity_log", - payload: { - id: uuidv4(), - timestamp: new Date(), - type: "transcript", - content: text, - }, - }); - - // Add to conversation - this.messages.push({ role: "user", content: text }); - - // Process through LLM (TTS enabled in voice mode for voice conversations) - this.currentStreamPromise = this.processWithLLM(this.isVoiceMode); - await this.currentStreamPromise; - } - /** * Handle audio chunk for buffering and transcription */ @@ -4507,18 +4408,10 @@ export class Session { }, }); - // Add to conversation - this.messages.push({ - role: "user", - content: this.isVoiceMode - ? this.formatVoiceTranscriptionForLLM(result.text) - : result.text, - }); - - // Set phase to LLM and process (TTS enabled in voice mode for voice conversations) + // Set phase to LLM and process (TTS enabled in voice mode for voice agents) this.clearSpeechInProgress("transcription complete"); this.setPhase("llm"); - this.currentStreamPromise = this.processWithLLM(this.isVoiceMode); + this.currentStreamPromise = this.processVoiceTurn(result.text); await this.currentStreamPromise; this.setPhase("idle"); } catch (error: any) { @@ -4537,367 +4430,212 @@ export class Session { } } - /** - * Process user message through LLM with streaming and tool execution - */ - private async processWithLLM(enableTTS: boolean): Promise { - let assistantResponse = ""; - let pendingTTS: Promise | null = null; - let textBuffer = ""; - let sawTextDelta = false; + private async ensureVoiceAssistantAgent(): Promise { + if (this.voiceAssistantAgentId) { + const existing = this.agentManager.getAgent(this.voiceAssistantAgentId); + if (existing) { + return existing.id; + } + try { + const hydrated = await this.ensureAgentLoaded(this.voiceAssistantAgentId); + this.voiceAssistantAgentId = hydrated.id; + return hydrated.id; + } catch (error) { + this.sessionLogger.debug( + { err: error, voiceAssistantAgentId: this.voiceAssistantAgentId }, + "Voice assistant agent not found in active/persisted state; creating new session" + ); + } + } - const flushTextBuffer = () => { - if (textBuffer.length > 0) { - // TTS handling (capture mode at generation time for drift protection) - if (enableTTS && !this.speechInProgress) { - const modeAtGeneration = this.isVoiceMode; - pendingTTS = this.ttsManager.generateAndWaitForPlayback( - textBuffer, - (msg) => this.emit(msg), - this.abortController.signal, - modeAtGeneration - ); - } else if (enableTTS && this.speechInProgress) { - this.sessionLogger.debug("Skipping TTS chunk while speech in progress"); - } + const provider = this.voiceLlmProvider; + if (!provider) { + throw new Error("Voice LLM provider is not configured"); + } + const voiceAgentId = this.voiceAssistantAgentId ?? uuidv4(); + const cwd = join(this.paseoHome, "voice-agent-workspace"); + await mkdir(cwd, { recursive: true }); - // Emit activity log + const mcpStdio = this.voiceAgentMcpStdio; + if (!mcpStdio) { + throw new Error("Voice MCP stdio bridge is not configured"); + } + + const model = this.voiceLlmModel?.trim() || undefined; + const config: AgentSessionConfig = { + provider, + cwd, + modeId: this.voiceLlmModeId ?? "default", + ...(model ? { model } : {}), + mcpServers: { + paseo: buildVoiceAgentMcpServerConfig({ + callerAgentId: voiceAgentId, + command: mcpStdio.command, + baseArgs: mcpStdio.baseArgs, + env: mcpStdio.env, + }), + }, + }; + + const created = await this.agentManager.createAgent(config, voiceAgentId, { + labels: { + surface: "voice", + ui: "false", + voiceClientId: this.clientId, + }, + }); + this.voiceAssistantAgentId = created.id; + let speakHandlerRegistered = false; + let callerContextRegistered = false; + + try { + this.registerVoiceSpeakHandler?.(created.id, async ({ text, signal }) => { + const abortSignal = signal ?? this.abortController.signal; + await this.ttsManager.generateAndWaitForPlayback( + text, + (msg) => this.emit(msg), + abortSignal, + true + ); this.emit({ type: "activity_log", payload: { id: uuidv4(), timestamp: new Date(), type: "assistant", - content: textBuffer, + content: text, }, }); - } - textBuffer = ""; - }; - - try { - // Debug: dump conversation before LLM call - await this.dumpConversation(); - - const openrouterApiKey = - this.openrouterApiKey ?? process.env.OPENROUTER_API_KEY ?? null; - invariant( - openrouterApiKey, - "OpenRouter API key is required (set providers.openrouter.apiKey in config.json or OPENROUTER_API_KEY)" - ); - - const openrouter = createOpenRouter({ - apiKey: openrouterApiKey, }); + speakHandlerRegistered = true; - // Wait for agent MCP to initialize if needed - if (!this.agentTools) { - this.sessionLogger.debug("Waiting for agent MCP initialization..."); - const startTime = Date.now(); - while (!this.agentTools && Date.now() - startTime < 5000) { - await new Promise((resolve) => setTimeout(resolve, 100)); - } - if (!this.agentTools) { - this.sessionLogger.info("Agent MCP tools unavailable; continuing with default tool set"); - } - } - - const allTools = getAllTools(this.agentTools ?? undefined); - - const result = await streamText({ - model: openrouter(this.voiceLlmModel ?? "anthropic/claude-haiku-4.5"), - system: getSystemPrompt(), - providerOptions: { - openrouter: { - transforms: ["middle-out"], // Compress prompts that are > context size. - } as OpenRouterProviderOptions, - }, - messages: this.messages, - tools: allTools, - abortSignal: this.abortController.signal, - onFinish: async (event) => { - const newMessages = event.response.messages; - if (newMessages.length > 0) { - this.messages.push(...newMessages); - this.sessionLogger.debug( - { messageCount: newMessages.length }, - `onFinish - saved message with ${newMessages.length} steps` - ); - } - - // Persist voice conversation to disk (best-effort; voice-only) - if (enableTTS && this.voiceConversationId) { - try { - await this.voiceConversationStore.save( - this.sessionLogger, - this.voiceConversationId, - this.messages - ); - } catch (error) { - this.sessionLogger.warn( - { err: error, voiceConversationId: this.voiceConversationId }, - "Failed to persist voice conversation" - ); - } - } - }, - onChunk: async ({ chunk }) => { - if (chunk.type === "text-delta") { - sawTextDelta = true; - // Accumulate text in buffer - textBuffer += chunk.text; - assistantResponse += chunk.text; - - // Emit chunk for UI streaming - this.emit({ - type: "assistant_chunk", - payload: { chunk: chunk.text }, - }); - } else if (chunk.type === "tool-call") { - // Flush accumulated text as a segment before tool call - flushTextBuffer(); - - // Wait for pending TTS before executing tool - if (pendingTTS) { - this.sessionLogger.debug( - { toolName: chunk.toolName }, - `Waiting for TTS before executing ${chunk.toolName}` - ); - await pendingTTS; - } - - // Handle present_artifact tool specially - if (chunk.toolName === "present_artifact") { - await this.handlePresentArtifact( - chunk.toolCallId, - chunk.input as PresentArtifactArgs - ); - } - - // Emit tool call activity log - this.emit({ - type: "activity_log", - payload: { - id: chunk.toolCallId, - timestamp: new Date(), - type: "tool_call", - content: `Calling ${chunk.toolName}`, - metadata: { - toolCallId: chunk.toolCallId, - toolName: chunk.toolName, - arguments: chunk.input, - }, - }, - }); - } else if (chunk.type === "tool-result") { - // Check if this is a create_agent result - if (chunk.toolName === "create_agent" && chunk.output) { - const result = chunk.output as any; - if (result.structuredContent?.agentId) { - const agentId = result.structuredContent.agentId; - this.emit({ - type: "status", - payload: { - status: "agent_created", - agentId, - }, - }); - } - } - - // Emit tool result event - this.emit({ - type: "activity_log", - payload: { - id: chunk.toolCallId, - timestamp: new Date(), - type: "tool_result", - content: `Tool ${chunk.toolName} completed`, - metadata: { - toolCallId: chunk.toolCallId, - toolName: chunk.toolName, - result: chunk.output, - }, - }, - }); - } - }, - onError: async (error) => { - this.sessionLogger.error({ err: error }, "Stream error"); - - this.emit({ - type: "activity_log", - payload: { - id: uuidv4(), - timestamp: new Date(), - type: "error", - content: `Stream error: ${ - error instanceof Error ? error.message : String(error) - }`, - }, - }); - }, - stopWhen: stepCountIs(10), + this.registerVoiceCallerContext?.(created.id, { + childAgentDefaultLabels: { ui: "true" }, + allowCustomCwd: true, + enableVoiceTools: true, }); + callerContextRegistered = true; + } catch (error) { + if (speakHandlerRegistered) { + this.unregisterVoiceSpeakHandler?.(created.id); + } + if (callerContextRegistered) { + this.unregisterVoiceCallerContext?.(created.id); + } + await this.agentManager.closeAgent(created.id).catch(() => undefined); + this.voiceAssistantAgentId = null; + throw error; + } - // Consume the fullStream to handle tool-error chunks - for await (const part of result.fullStream) { - if (part.type === "tool-error") { - this.emit({ - type: "activity_log", - payload: { - id: part.toolCallId, - timestamp: new Date(), - type: "error", - content: `Tool ${part.toolName} failed: ${ - part.error instanceof Error - ? part.error.message - : String(part.error) - }`, - metadata: { - toolCallId: part.toolCallId, - toolName: part.toolName, - error: part.error, - }, - }, - }); + this.sessionLogger.info( + { + voiceAssistantAgentId: created.id, + provider, + model: model ?? null, + providerExplicit: this.voiceLlmProviderExplicit, + }, + "Voice assistant agent initialized" + ); + return created.id; + } + + private buildVoiceAgentPrompt(userText: string): string { + return [ + VOICE_AGENT_SYSTEM_INSTRUCTION, + "", + `User said: ${userText.trim()}`, + ].join("\n"); + } + + private shouldAllowVoicePermission(request: AgentPermissionRequest): boolean { + return isVoicePermissionAllowed(request); + } + + private async processWithVoiceAgent(userText: string): Promise { + const agentId = await this.ensureVoiceAssistantAgent(); + + await this.interruptAgentIfRunning(agentId); + + const prompt = this.buildVoiceAgentPrompt(userText); + this.agentManager.recordUserMessage(agentId, userText); + + let sawSpeakToolCall = false; + const assistantTextChunks: string[] = []; + const iterator = this.agentManager.streamAgent(agentId, prompt); + for await (const event of iterator) { + if (event.type === "turn_failed") { + throw new Error(event.error); + } + if (event.type === "timeline") { + if (event.item.type === "tool_call" && typeof event.item.name === "string") { + if (event.item.name.toLowerCase().includes("speak")) { + sawSpeakToolCall = true; + } + } + if (event.item.type === "assistant_message" && event.item.text.trim().length > 0) { + assistantTextChunks.push(event.item.text.trim()); } } - - if (!sawTextDelta) { - let fallbackText = ""; - try { - fallbackText = (await result.text).trim(); - } catch { - fallbackText = ""; - } - if (fallbackText.length > 0) { - textBuffer += fallbackText; - assistantResponse += fallbackText; - this.emit({ - type: "assistant_chunk", - payload: { chunk: fallbackText }, + if (event.type === "permission_requested") { + if (this.shouldAllowVoicePermission(event.request)) { + await this.agentManager.respondToPermission(agentId, event.request.id, { + behavior: "allow", }); - } - } - - // Flush any remaining text at the end - flushTextBuffer(); - - // Note: Message is saved by onFinish callback with proper tool calls - - // Now wait for any pending TTS, but don't fail if it times out - if (pendingTTS) { - try { - await pendingTTS; - } catch (ttsError) { - this.sessionLogger.error( - { err: ttsError }, - "TTS playback failed (message already saved)" + } else { + await this.agentManager.respondToPermission(agentId, event.request.id, { + behavior: "deny", + message: "Voice assistant policy only allows MCP paseo tools.", + interrupt: true, + }); + throw new Error( + `Voice assistant denied non-MCP tool request: ${event.request.name}` ); } } - } catch (error) { - // Note: Partial messages are saved by onAbort callback with proper tool calls + } - // Check if this is an abort error - const isAbortError = - error instanceof Error && error.name === "AbortError"; - - // Only emit error log for non-abort errors - if (!isAbortError) { - this.emit({ - type: "activity_log", - payload: { - id: uuidv4(), - timestamp: new Date(), - type: "error", - content: `Error: ${ - error instanceof Error ? error.message : String(error) - }`, - }, - }); - } - - // Don't re-throw abort errors (they're expected during interruptions) - if (isAbortError) { - this.sessionLogger.debug("Stream aborted (partial response saved)"); - return; - } - - // Re-throw unexpected errors - throw error; - } finally { - // Increment turn index for next LLM call - this.turnIndex++; - - // Clear the stream promise tracker - this.currentStreamPromise = null; + if (!sawSpeakToolCall && assistantTextChunks.length > 0) { + const fallbackText = assistantTextChunks.join(" ").trim(); + await this.ttsManager.generateAndWaitForPlayback( + fallbackText, + (msg) => this.emit(msg), + this.abortController.signal, + true + ); + this.emit({ + type: "activity_log", + payload: { + id: uuidv4(), + timestamp: new Date(), + type: "assistant", + content: fallbackText, + }, + }); + this.sessionLogger.warn( + { voiceAssistantAgentId: agentId }, + "Voice agent responded without speak tool; used fallback TTS from assistant text" + ); } } /** - * Handle present_artifact tool execution + * Process user message through LLM with streaming and tool execution */ - private async handlePresentArtifact( - toolCallId: string, - args: PresentArtifactArgs - ): Promise { - let content: string; - let isBase64 = false; - + private async processVoiceTurn(latestUserText?: string): Promise { try { - if (args.source.type === "file") { - const fileBuffer = await readFile(args.source.path); - content = fileBuffer.toString("base64"); - isBase64 = true; - } else if (args.source.type === "command_output") { - const { stdout } = await execAsync(args.source.command, { - encoding: "buffer", - }); - content = stdout.toString("base64"); - isBase64 = true; - } else if (args.source.type === "text") { - content = args.source.text; - isBase64 = false; - } else { - content = "[Unknown source type]"; - isBase64 = false; + if (!this.isVoiceMode) { + this.sessionLogger.warn("Ignoring processVoiceTurn call while voice mode is disabled"); + return; } - } catch (error) { - this.sessionLogger.error( - { err: error }, - "Failed to resolve artifact source" - ); - content = `[Error: ${ - error instanceof Error ? error.message : String(error) - }]`; - isBase64 = false; + const normalized = (latestUserText ?? "").trim(); + if (!normalized) { + return; + } + await this.processWithVoiceAgent(normalized); + } finally { + this.turnIndex++; + this.currentStreamPromise = null; } - - // Emit artifact message - this.emit({ - type: "artifact", - payload: { - type: args.type, - id: toolCallId, - title: args.title, - content, - isBase64, - }, - }); - - // Emit activity log for artifact - this.emit({ - type: "activity_log", - payload: { - id: toolCallId, - timestamp: new Date(), - type: "system", - content: `${args.type} artifact: ${args.title}`, - metadata: { artifactId: toolCallId, artifactType: args.type }, - }, - }); } /** @@ -5107,38 +4845,6 @@ export class Session { this.onMessage(msg); } - /** - * Debug helper: dump conversation to disk - */ - private async dumpConversation(): Promise { - try { - const dumpDir = join(process.cwd(), ".debug.conversations"); - await mkdir(dumpDir, { recursive: true }); - - const filename = `${this.voiceConversationId ?? this.sessionId}-${this.turnIndex}.json`; - const filepath = join(dumpDir, filename); - - const dump = { - voiceConversationId: this.voiceConversationId, - sessionId: this.sessionId, - turnIndex: this.turnIndex, - timestamp: new Date().toISOString(), - messages: this.messages, - }; - - await writeFile(filepath, inspect(dump, { depth: null }), "utf-8"); - this.sessionLogger.debug( - { filepath }, - `Dumped conversation to ${filepath}` - ); - } catch (error) { - this.sessionLogger.error( - { err: error }, - "Failed to dump conversation" - ); - } - } - /** * Clean up session resources */ @@ -5180,6 +4886,21 @@ export class Session { this.agentTools = null; } + if (this.voiceAssistantAgentId) { + try { + await this.agentManager.closeAgent(this.voiceAssistantAgentId); + } catch (error) { + this.sessionLogger.warn( + { err: error, voiceAssistantAgentId: this.voiceAssistantAgentId }, + "Failed to close voice assistant agent" + ); + } finally { + this.unregisterVoiceSpeakHandler?.(this.voiceAssistantAgentId); + this.unregisterVoiceCallerContext?.(this.voiceAssistantAgentId); + this.voiceAssistantAgentId = null; + } + } + // Unsubscribe from all terminals for (const unsubscribe of this.terminalSubscriptions.values()) { unsubscribe(); diff --git a/packages/server/src/server/session.voice-mcp-config.test.ts b/packages/server/src/server/session.voice-mcp-config.test.ts new file mode 100644 index 000000000..14f4c84e5 --- /dev/null +++ b/packages/server/src/server/session.voice-mcp-config.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, test } from "vitest"; + +import { buildVoiceAgentMcpServerConfig } from "./session.js"; + +describe("voice MCP stdio config", () => { + test("builds stdio MCP config for voice agent", () => { + const config = buildVoiceAgentMcpServerConfig({ + callerAgentId: "voice-agent-123", + command: "/usr/local/bin/paseo", + baseArgs: ["__paseo_voice_mcp_bridge", "--socket", "/tmp/paseo-voice.sock"], + env: { + PASEO_HOME: "/tmp/paseo-home", + }, + }); + + expect(config.type).toBe("stdio"); + expect(config.command).toBe("/usr/local/bin/paseo"); + expect(config.args).toEqual([ + "__paseo_voice_mcp_bridge", + "--socket", + "/tmp/paseo-voice.sock", + "--caller-agent-id", + "voice-agent-123", + ]); + expect(config.env).toEqual({ + PASEO_HOME: "/tmp/paseo-home", + }); + }); +}); diff --git a/packages/server/src/server/speech/providers/local/config.ts b/packages/server/src/server/speech/providers/local/config.ts new file mode 100644 index 000000000..5399111ce --- /dev/null +++ b/packages/server/src/server/speech/providers/local/config.ts @@ -0,0 +1,167 @@ +import path from "node:path"; + +import { z } from "zod"; + +import type { PersistedConfig } from "../../../persisted-config.js"; +import type { RequestedSpeechProviders } from "../../speech-types.js"; +import { + DEFAULT_LOCAL_STT_MODEL, + DEFAULT_LOCAL_TTS_MODEL, + LocalSttModelIdSchema, + LocalTtsModelIdSchema, + type LocalSpeechModelId, + type LocalSttModelId, + type LocalTtsModelId, +} from "./models.js"; + +export type LocalSpeechModelConfig = { + dictationStt: LocalSttModelId; + voiceStt: LocalSttModelId; + voiceTts: LocalTtsModelId; + voiceTtsSpeakerId?: number; + voiceTtsSpeed?: number; +}; + +export type LocalSpeechProviderConfig = { + modelsDir: string; + autoDownload?: boolean; + models: LocalSpeechModelConfig; +}; + +export type ResolvedLocalSpeechConfig = { + local: LocalSpeechProviderConfig | undefined; +}; + +export type { LocalSpeechModelId, LocalSttModelId, LocalTtsModelId }; + +const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech"); + +const BooleanStringSchema = z + .string() + .trim() + .toLowerCase() + .pipe(z.enum(["1", "0", "true", "false", "yes", "no"])) + .transform((value) => value === "1" || value === "true" || value === "yes"); + +const OptionalBooleanFlagSchema = z + .union([z.boolean(), BooleanStringSchema]) + .optional(); + +const NumberLikeSchema = z.union([ + z.number(), + z.string().trim().min(1), +]); + +const OptionalFiniteNumberSchema = NumberLikeSchema + .pipe(z.coerce.number().finite()) + .optional(); + +const OptionalIntegerSchema = NumberLikeSchema + .pipe(z.coerce.number().int()) + .optional(); + +const LocalSpeechResolutionSchema = z.object({ + includeProviderConfig: z.boolean(), + modelsDir: z.string().trim().min(1), + autoDownload: OptionalBooleanFlagSchema.default(true), + dictationLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL), + voiceLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL), + voiceLocalTtsModel: LocalTtsModelIdSchema.default(DEFAULT_LOCAL_TTS_MODEL), + voiceLocalTtsSpeakerId: OptionalIntegerSchema, + voiceLocalTtsSpeed: OptionalFiniteNumberSchema, +}); + +function persistedLocalFeatureModel( + provider: RequestedSpeechProviders[keyof RequestedSpeechProviders]["provider"], + model: string | undefined +): string | undefined { + if (provider !== "local") { + return undefined; + } + return model; +} + +function shouldIncludeLocalProviderConfig(params: { + providers: RequestedSpeechProviders; + env: NodeJS.ProcessEnv; + persisted: PersistedConfig; +}): boolean { + const localRequestedByFeature = + params.providers.dictationStt.provider === "local" || + params.providers.voiceStt.provider === "local" || + params.providers.voiceTts.provider === "local"; + + return ( + localRequestedByFeature || + params.env.PASEO_LOCAL_MODELS_DIR !== undefined || + params.persisted.providers?.local !== undefined + ); +} + +export function resolveLocalSpeechConfig(params: { + paseoHome: string; + env: NodeJS.ProcessEnv; + persisted: PersistedConfig; + providers: RequestedSpeechProviders; +}): ResolvedLocalSpeechConfig { + const includeProviderConfig = shouldIncludeLocalProviderConfig(params); + + const parsed = LocalSpeechResolutionSchema.parse({ + includeProviderConfig, + modelsDir: + params.env.PASEO_LOCAL_MODELS_DIR ?? + params.persisted.providers?.local?.modelsDir ?? + path.join(params.paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR), + autoDownload: + params.env.PASEO_LOCAL_AUTO_DOWNLOAD ?? + params.persisted.providers?.local?.autoDownload, + dictationLocalSttModel: + params.env.PASEO_DICTATION_LOCAL_STT_MODEL ?? + persistedLocalFeatureModel( + params.providers.dictationStt.provider, + params.persisted.features?.dictation?.stt?.model + ) ?? + DEFAULT_LOCAL_STT_MODEL, + voiceLocalSttModel: + params.env.PASEO_VOICE_LOCAL_STT_MODEL ?? + persistedLocalFeatureModel( + params.providers.voiceStt.provider, + params.persisted.features?.voiceMode?.stt?.model + ) ?? + DEFAULT_LOCAL_STT_MODEL, + voiceLocalTtsModel: + params.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? + persistedLocalFeatureModel( + params.providers.voiceTts.provider, + params.persisted.features?.voiceMode?.tts?.model + ) ?? + DEFAULT_LOCAL_TTS_MODEL, + voiceLocalTtsSpeakerId: + params.env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID ?? + params.persisted.features?.voiceMode?.tts?.speakerId, + voiceLocalTtsSpeed: + params.env.PASEO_VOICE_LOCAL_TTS_SPEED ?? + params.persisted.features?.voiceMode?.tts?.speed, + }); + + return { + local: + parsed.includeProviderConfig + ? { + modelsDir: parsed.modelsDir, + autoDownload: parsed.autoDownload, + models: { + dictationStt: parsed.dictationLocalSttModel, + voiceStt: parsed.voiceLocalSttModel, + voiceTts: parsed.voiceLocalTtsModel, + ...(parsed.voiceLocalTtsSpeakerId !== undefined + ? { voiceTtsSpeakerId: parsed.voiceLocalTtsSpeakerId } + : {}), + ...(parsed.voiceLocalTtsSpeed !== undefined + ? { voiceTtsSpeed: parsed.voiceLocalTtsSpeed } + : {}), + }, + } + : undefined, + }; +} diff --git a/packages/server/src/server/speech/providers/local/models.ts b/packages/server/src/server/speech/providers/local/models.ts new file mode 100644 index 000000000..ee2dee911 --- /dev/null +++ b/packages/server/src/server/speech/providers/local/models.ts @@ -0,0 +1,48 @@ +import { + ensureSherpaOnnxModels, + getSherpaOnnxModelDir, +} from "./sherpa/model-downloader.js"; +import { + DEFAULT_LOCAL_STT_MODEL, + DEFAULT_LOCAL_TTS_MODEL, + LocalSttModelIdSchema, + LocalTtsModelIdSchema, + listSherpaOnnxModels, + type LocalSpeechModelId, + type LocalSttModelId, + type LocalTtsModelId, +} from "./sherpa/model-catalog.js"; + +export { + DEFAULT_LOCAL_STT_MODEL, + DEFAULT_LOCAL_TTS_MODEL, + LocalSttModelIdSchema, + LocalTtsModelIdSchema, + type LocalSpeechModelId, + type LocalSttModelId, + type LocalTtsModelId, +}; + +export type LocalSpeechModelSpec = ReturnType[number]; + +export function listLocalSpeechModels(): LocalSpeechModelSpec[] { + return listSherpaOnnxModels(); +} + +export function getLocalSpeechModelDir(modelsDir: string, modelId: LocalSpeechModelId): string { + return getSherpaOnnxModelDir(modelsDir, modelId); +} + +export async function ensureLocalSpeechModels(options: { + modelsDir: string; + modelIds: LocalSpeechModelId[]; + autoDownload?: boolean; + logger: import("pino").Logger; +}): Promise> { + return ensureSherpaOnnxModels({ + modelsDir: options.modelsDir, + modelIds: options.modelIds, + autoDownload: options.autoDownload ?? true, + logger: options.logger, + }); +} diff --git a/packages/server/src/server/speech/providers/local/runtime.ts b/packages/server/src/server/speech/providers/local/runtime.ts new file mode 100644 index 000000000..13bd4ef9d --- /dev/null +++ b/packages/server/src/server/speech/providers/local/runtime.ts @@ -0,0 +1,370 @@ +import type { Logger } from "pino"; + +import type { PaseoSpeechConfig } from "../../../bootstrap.js"; +import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js"; +import type { RequestedSpeechProviders } from "../../speech-types.js"; +import { PocketTtsOnnxTTS } from "./pocket/pocket-tts-onnx.js"; +import { + ensureLocalSpeechModels, + getLocalSpeechModelDir, + DEFAULT_LOCAL_STT_MODEL, + DEFAULT_LOCAL_TTS_MODEL, + LocalSttModelIdSchema, + LocalTtsModelIdSchema, + type LocalSpeechModelId, + type LocalSttModelId, + type LocalTtsModelId, +} from "./models.js"; +import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js"; +import { SherpaOnlineRecognizerEngine } from "./sherpa/sherpa-online-recognizer.js"; +import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js"; +import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js"; +import { SherpaRealtimeTranscriptionSession } from "./sherpa/sherpa-realtime-session.js"; +import { SherpaOnnxSTT } from "./sherpa/sherpa-stt.js"; +import { SherpaOnnxTTS } from "./sherpa/sherpa-tts.js"; + +type LocalSttEngine = + | { kind: "offline"; engine: SherpaOfflineRecognizerEngine } + | { kind: "online"; engine: SherpaOnlineRecognizerEngine }; + +type ResolvedLocalModels = { + dictationLocalSttModel: LocalSttModelId; + voiceLocalSttModel: LocalSttModelId; + voiceLocalTtsModel: LocalTtsModelId; +}; + +type LocalSpeechAvailability = { + configured: boolean; + modelsDir: string | null; + autoDownload: boolean | null; +}; + +export type InitializedLocalSpeech = { + sttService: SpeechToTextProvider | null; + ttsService: TextToSpeechProvider | null; + dictationSttService: SpeechToTextProvider | null; + localVoiceTtsProvider: TextToSpeechProvider | null; + localModelConfig: { + modelsDir: string; + defaultModelIds: LocalSpeechModelId[]; + } | null; + availability: LocalSpeechAvailability; + cleanup: () => void; +}; + +function buildModelDownloadHint(modelId: LocalSpeechModelId): string { + return `Use 'paseo speech download --model ${modelId}' to download this model.`; +} + +function resolveConfiguredLocalModels( + speechConfig: PaseoSpeechConfig | null +): ResolvedLocalModels { + return { + dictationLocalSttModel: LocalSttModelIdSchema.parse( + speechConfig?.local?.models.dictationStt ?? DEFAULT_LOCAL_STT_MODEL + ), + voiceLocalSttModel: LocalSttModelIdSchema.parse( + speechConfig?.local?.models.voiceStt ?? DEFAULT_LOCAL_STT_MODEL + ), + voiceLocalTtsModel: LocalTtsModelIdSchema.parse( + speechConfig?.local?.models.voiceTts ?? DEFAULT_LOCAL_TTS_MODEL + ), + }; +} + +export function getLocalSpeechAvailability( + speechConfig: PaseoSpeechConfig | null +): LocalSpeechAvailability { + const localConfig = speechConfig?.local ?? null; + return { + configured: Boolean(localConfig), + modelsDir: localConfig?.modelsDir ?? null, + autoDownload: localConfig?.autoDownload ?? null, + }; +} + +function computeRequiredLocalModelIds(params: { + providers: RequestedSpeechProviders; + models: ResolvedLocalModels; +}): LocalSpeechModelId[] { + const ids = new Set(); + if (params.providers.dictationStt.provider === "local") { + ids.add(params.models.dictationLocalSttModel); + } + if (params.providers.voiceStt.provider === "local") { + ids.add(params.models.voiceLocalSttModel); + } + if (params.providers.voiceTts.provider === "local") { + ids.add(params.models.voiceLocalTtsModel); + } + return Array.from(ids); +} + +async function createLocalSttEngine(params: { + modelId: LocalSttModelId; + modelsDir: string; + logger: Logger; +}): Promise { + const { modelId, modelsDir, logger } = params; + + if (modelId === "parakeet-tdt-0.6b-v3-int8") { + const modelDir = getLocalSpeechModelDir(modelsDir, modelId); + return { + kind: "offline", + engine: new SherpaOfflineRecognizerEngine( + { + model: { + kind: "nemo_transducer", + encoder: `${modelDir}/encoder.int8.onnx`, + decoder: `${modelDir}/decoder.int8.onnx`, + joiner: `${modelDir}/joiner.int8.onnx`, + tokens: `${modelDir}/tokens.txt`, + }, + numThreads: 2, + debug: 0, + }, + logger + ), + }; + } + + if (modelId === "paraformer-bilingual-zh-en") { + const modelDir = getLocalSpeechModelDir(modelsDir, modelId); + return { + kind: "online", + engine: new SherpaOnlineRecognizerEngine( + { + model: { + kind: "paraformer", + encoder: `${modelDir}/encoder.int8.onnx`, + decoder: `${modelDir}/decoder.int8.onnx`, + tokens: `${modelDir}/tokens.txt`, + }, + numThreads: 1, + debug: 0, + }, + logger + ), + }; + } + + if (modelId === "zipformer-bilingual-zh-en-2023-02-20") { + const modelDir = getLocalSpeechModelDir(modelsDir, modelId); + return { + kind: "online", + engine: new SherpaOnlineRecognizerEngine( + { + model: { + kind: "transducer", + encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`, + decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`, + joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`, + tokens: `${modelDir}/tokens.txt`, + modelType: "zipformer", + }, + numThreads: 1, + debug: 0, + }, + logger + ), + }; + } + + throw new Error(`Unsupported local STT model '${modelId}'`); +} + +export async function initializeLocalSpeechServices(params: { + providers: RequestedSpeechProviders; + speechConfig: PaseoSpeechConfig | null; + logger: Logger; +}): Promise { + const { providers, logger, speechConfig } = params; + const localConfig = speechConfig?.local ?? null; + const localModels = resolveConfiguredLocalModels(speechConfig); + + let sttService: SpeechToTextProvider | null = null; + let ttsService: TextToSpeechProvider | null = null; + let dictationSttService: SpeechToTextProvider | null = null; + let localVoiceTtsProvider: TextToSpeechProvider | null = null; + + const requiredLocalModelIds = computeRequiredLocalModelIds({ + providers, + models: localModels, + }); + + if (requiredLocalModelIds.length > 0 && localConfig) { + try { + logger.info( + { + modelsDir: localConfig.modelsDir, + modelIds: requiredLocalModelIds, + autoDownload: localConfig.autoDownload ?? true, + }, + "Ensuring local speech models" + ); + await ensureLocalSpeechModels({ + modelsDir: localConfig.modelsDir, + modelIds: requiredLocalModelIds, + autoDownload: localConfig.autoDownload ?? true, + logger, + }); + } catch (err) { + logger.error( + { + err, + modelsDir: localConfig.modelsDir, + modelIds: requiredLocalModelIds, + autoDownload: localConfig.autoDownload ?? true, + hint: + "Use `paseo speech models` to inspect status and " + + "`paseo speech download --model ` to fetch missing models.", + }, + "Failed to ensure local speech models" + ); + } + } + + const localSttEngines = new Map(); + + const getLocalSttEngine = async ( + modelId: LocalSttModelId + ): Promise => { + const existing = localSttEngines.get(modelId); + if (existing) { + return existing; + } + if (!localConfig) { + return null; + } + try { + const created = await createLocalSttEngine({ + modelId, + modelsDir: localConfig.modelsDir, + logger, + }); + localSttEngines.set(modelId, created); + return created; + } catch (err) { + logger.error( + { + err, + modelsDir: localConfig.modelsDir, + modelId, + hint: buildModelDownloadHint(modelId), + }, + "Failed to initialize local STT engine (models missing or invalid)" + ); + return null; + } + }; + + if (providers.voiceStt.provider === "local") { + if (!localConfig) { + logger.warn( + { configured: false }, + "Local STT selected for voice but local provider config is missing; STT will be unavailable" + ); + } else { + const voiceEngine = await getLocalSttEngine(localModels.voiceLocalSttModel); + if (voiceEngine?.kind === "offline") { + sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger); + } else if (voiceEngine?.kind === "online") { + sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger); + } + } + } + + if (providers.dictationStt.provider === "local") { + if (!localConfig) { + logger.warn( + { configured: false }, + "Local STT selected for dictation but local provider config is missing; dictation STT will be unavailable" + ); + } else { + const dictationEngine = await getLocalSttEngine(localModels.dictationLocalSttModel); + if (dictationEngine?.kind === "offline") { + dictationSttService = { + id: "local", + createSession: () => + new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }), + }; + } else if (dictationEngine?.kind === "online") { + dictationSttService = { + id: "local", + createSession: () => new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }), + }; + } + } + } + + if (providers.voiceTts.provider === "local") { + if (!localConfig) { + logger.warn( + { configured: false }, + "Local TTS selected for voice but local provider config is missing; TTS will be unavailable" + ); + } else { + try { + if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") { + const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); + localVoiceTtsProvider = await PocketTtsOnnxTTS.create( + { + modelDir, + precision: "int8", + targetChunkMs: 50, + }, + logger + ); + } else { + const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel); + localVoiceTtsProvider = new SherpaOnnxTTS( + { + preset: localModels.voiceLocalTtsModel, + modelDir, + speakerId: speechConfig?.local?.models.voiceTtsSpeakerId, + speed: speechConfig?.local?.models.voiceTtsSpeed, + }, + logger + ); + } + ttsService = localVoiceTtsProvider; + } catch (err) { + logger.error( + { + err, + modelsDir: localConfig.modelsDir, + modelId: localModels.voiceLocalTtsModel, + hint: buildModelDownloadHint(localModels.voiceLocalTtsModel), + }, + "Failed to initialize local TTS engine (models missing or invalid)" + ); + } + } + } + + const cleanup = () => { + const maybeFreeable = localVoiceTtsProvider as unknown as { free?: () => void } | null; + if (typeof maybeFreeable?.free === "function") { + maybeFreeable.free(); + } + for (const engine of localSttEngines.values()) { + engine.engine.free(); + } + }; + + return { + sttService, + ttsService, + dictationSttService, + localVoiceTtsProvider, + localModelConfig: + localConfig + ? { + modelsDir: localConfig.modelsDir, + defaultModelIds: requiredLocalModelIds, + } + : null, + availability: getLocalSpeechAvailability(speechConfig), + cleanup, + }; +} diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts index 94c6d62bd..084c89b9d 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts @@ -1,26 +1,22 @@ +import { z } from "zod"; + export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts"; -export type SherpaOnnxModelId = - | "zipformer-bilingual-zh-en-2023-02-20" - | "paraformer-bilingual-zh-en" - | "parakeet-tdt-0.6b-v3-int8" - | "kitten-nano-en-v0_1-fp16" - | "kokoro-en-v0_19" - | "pocket-tts-onnx-int8"; +type DefaultModelRole = "stt" | "tts"; -export type SherpaOnnxModelSpec = { - id: SherpaOnnxModelId; +type SherpaOnnxCatalogEntry = { kind: SherpaOnnxModelKind; archiveUrl?: string; downloadFiles?: Array<{ url: string; relPath: string }>; extractedDir: string; requiredFiles: string[]; description: string; + aliases?: readonly string[]; + defaultFor?: DefaultModelRole; }; -export const SHERPA_ONNX_MODEL_CATALOG: Record = { +export const SHERPA_ONNX_MODEL_CATALOG = { "zipformer-bilingual-zh-en-2023-02-20": { - id: "zipformer-bilingual-zh-en-2023-02-20", kind: "stt-online", archiveUrl: "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2", @@ -32,44 +28,45 @@ export const SHERPA_ONNX_MODEL_CATALOG: Record; + +export type SherpaOnnxModelId = keyof typeof SHERPA_ONNX_MODEL_CATALOG; +export type LocalSpeechModelId = SherpaOnnxModelId; + +type ModelIdByKind = { + [Id in SherpaOnnxModelId]: (typeof SHERPA_ONNX_MODEL_CATALOG)[Id]["kind"] extends K + ? Id + : never; +}[SherpaOnnxModelId]; + +export type LocalSttModelId = ModelIdByKind<"stt-online"> | ModelIdByKind<"stt-offline">; +export type LocalTtsModelId = ModelIdByKind<"tts">; + +const ALL_MODEL_IDS = Object.keys(SHERPA_ONNX_MODEL_CATALOG) as SherpaOnnxModelId[]; + +export const LOCAL_STT_MODEL_IDS = ALL_MODEL_IDS.filter( + (id): id is LocalSttModelId => SHERPA_ONNX_MODEL_CATALOG[id].kind !== "tts" +); + +export const LOCAL_TTS_MODEL_IDS = ALL_MODEL_IDS.filter( + (id): id is LocalTtsModelId => SHERPA_ONNX_MODEL_CATALOG[id].kind === "tts" +); + +function resolveDefaultModelId(role: "stt"): LocalSttModelId; +function resolveDefaultModelId(role: "tts"): LocalTtsModelId; +function resolveDefaultModelId(role: DefaultModelRole): SherpaOnnxModelId { + const match = ALL_MODEL_IDS.find((id) => { + const entry: SherpaOnnxCatalogEntry = SHERPA_ONNX_MODEL_CATALOG[id]; + return entry.defaultFor === role; + }); + if (!match) { + throw new Error(`No default model configured for role '${role}'`); + } + return match; +} + +export const DEFAULT_LOCAL_STT_MODEL = resolveDefaultModelId("stt"); +export const DEFAULT_LOCAL_TTS_MODEL = resolveDefaultModelId("tts"); + +function buildAliasMap(modelIds: readonly T[]): Record { + const aliasMap: Record = {}; + for (const modelId of modelIds) { + const aliases = SHERPA_ONNX_MODEL_CATALOG[modelId as SherpaOnnxModelId].aliases ?? []; + for (const alias of aliases) { + aliasMap[alias.trim().toLowerCase()] = modelId; + } + } + return aliasMap; +} + +function createAliasedModelIdSchema(params: { + modelIds: readonly T[]; + aliases: Record; +}): z.ZodType { + const validIds = new Set(params.modelIds); + return z + .string() + .trim() + .toLowerCase() + .refine( + (value): value is T => + validIds.has(value as T) || Object.prototype.hasOwnProperty.call(params.aliases, value), + { + message: "Invalid model id", + } + ) + .transform((value) => params.aliases[value] ?? (value as T)); +} + +const STT_MODEL_ALIASES = buildAliasMap(LOCAL_STT_MODEL_IDS); +const TTS_MODEL_ALIASES = buildAliasMap(LOCAL_TTS_MODEL_IDS); + +export const LocalSttModelIdSchema = createAliasedModelIdSchema({ + modelIds: LOCAL_STT_MODEL_IDS, + aliases: STT_MODEL_ALIASES, +}); + +export const LocalTtsModelIdSchema = createAliasedModelIdSchema({ + modelIds: LOCAL_TTS_MODEL_IDS, + aliases: TTS_MODEL_ALIASES, +}); + +export type SherpaOnnxModelSpec = SherpaOnnxCatalogEntry & { + id: SherpaOnnxModelId; }; export function listSherpaOnnxModels(): SherpaOnnxModelSpec[] { - return Object.values(SHERPA_ONNX_MODEL_CATALOG); + return ALL_MODEL_IDS.map((id) => ({ + id, + ...SHERPA_ONNX_MODEL_CATALOG[id], + })); } export function getSherpaOnnxModelSpec(id: SherpaOnnxModelId): SherpaOnnxModelSpec { const spec = SHERPA_ONNX_MODEL_CATALOG[id]; if (!spec) { - throw new Error(`Unknown sherpa-onnx model id: ${id}`); + throw new Error(`Unknown local speech model id: ${id}`); } - return spec; + return { + id, + ...spec, + }; } diff --git a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts index 9c434c4be..66298751d 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts @@ -96,7 +96,7 @@ async function isNonEmptyFile(filePath: string): Promise { export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOptions): Promise { const logger = options.logger.child({ module: "speech", - provider: "sherpa-onnx", + provider: "local", component: "model-downloader", modelId: options.modelId, }); @@ -109,8 +109,8 @@ export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOption if (!options.autoDownload) { throw new Error( - `Missing sherpa-onnx model files for ${options.modelId} in ${modelDir}. ` + - `Set PASEO_SHERPA_ONNX_AUTO_DOWNLOAD=1 to auto-download.` + `Missing local speech model files for ${options.modelId} in ${modelDir}. ` + + `Set PASEO_LOCAL_AUTO_DOWNLOAD=1 to auto-download.` ); } diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts index 810b8d165..4cb40416c 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts @@ -34,7 +34,7 @@ export class SherpaOfflineRecognizerEngine { private readonly logger: pino.Logger; constructor(config: SherpaOfflineRecognizerConfig, logger: pino.Logger) { - this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "offline-recognizer" }); + this.logger = logger.child({ module: "speech", provider: "local", component: "offline-recognizer" }); assertFileExists(config.model.encoder, "offline encoder"); assertFileExists(config.model.decoder, "offline decoder"); diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts index c7c8c3b1b..4148ca860 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts @@ -46,7 +46,7 @@ export class SherpaOnlineRecognizerEngine { private readonly logger: pino.Logger; constructor(config: SherpaOnlineRecognizerConfig, logger: pino.Logger) { - this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "online-recognizer" }); + this.logger = logger.child({ module: "speech", provider: "local", component: "online-recognizer" }); const { model } = config; if (model.kind === "transducer") { diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts index 80bf1abe2..f90df65ee 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts @@ -12,6 +12,7 @@ export function loadSherpaOnnx(): SherpaOnnxModule { if (cached) { return cached; } + const require = createRequire(import.meta.url); cached = require("sherpa-onnx") as SherpaOnnxModule; return cached; diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts index 8318b8673..81498c079 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts @@ -14,13 +14,16 @@ function platformArch(): string { return `${platform}-${process.arch}`; } -function prependLibraryPath(envKey: "DYLD_LIBRARY_PATH" | "LD_LIBRARY_PATH", dir: string): void { +function prependLibraryPath( + envKey: "DYLD_LIBRARY_PATH" | "LD_LIBRARY_PATH" | "PATH", + dir: string +): void { const current = process.env[envKey] ?? ""; - const parts = current.split(":").filter(Boolean); + const parts = current.split(path.delimiter).filter(Boolean); if (parts.includes(dir)) { return; } - process.env[envKey] = [dir, ...parts].join(":"); + process.env[envKey] = [dir, ...parts].join(path.delimiter); } export function loadSherpaOnnxNode(): SherpaOnnxNodeModule { @@ -31,8 +34,7 @@ export function loadSherpaOnnxNode(): SherpaOnnxNodeModule { const require = createRequire(import.meta.url); // sherpa-onnx-node depends on a platform-specific package (e.g. sherpa-onnx-darwin-arm64) - // that contains the native addon + its shared libraries. On macOS/Linux we need to ensure - // the appropriate library path env var includes that directory before requiring the addon. + // that contains the native addon and shared libraries. Ensure the OS loader path includes it. const arch = platformArch(); const pkgName = `sherpa-onnx-${arch}`; @@ -43,6 +45,8 @@ export function loadSherpaOnnxNode(): SherpaOnnxNodeModule { prependLibraryPath("DYLD_LIBRARY_PATH", pkgDir); } else if (process.platform === "linux") { prependLibraryPath("LD_LIBRARY_PATH", pkgDir); + } else if (process.platform === "win32") { + prependLibraryPath("PATH", pkgDir); } } catch { // Best effort - if the platform package isn't present, require() below will throw a useful error. diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts index 3b23fa8e1..8bc8bb52c 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts @@ -25,7 +25,7 @@ export class SherpaOnnxParakeetSTT implements SpeechToTextProvider { constructor(config: SherpaParakeetSttConfig, logger: pino.Logger) { this.engine = config.engine; this.silencePeakThreshold = config.silencePeakThreshold ?? 300; - this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "parakeet-stt" }); + this.logger = logger.child({ module: "speech", provider: "local", component: "parakeet-stt" }); } public createSession(params: { diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts index ff308f51d..05dbf1d2a 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts @@ -28,7 +28,7 @@ export class SherpaOnnxSTT implements SpeechToTextProvider { this.engine = config.engine; this.silencePeakThreshold = config.silencePeakThreshold ?? 300; this.tailPaddingMs = config.tailPaddingMs ?? 500; - this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "stt" }); + this.logger = logger.child({ module: "speech", provider: "local", component: "stt" }); } public createSession(params: { diff --git a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts index 99d9dab6a..8c05461b8 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts @@ -33,7 +33,7 @@ export class SherpaOnnxTTS implements TextToSpeechProvider { if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") { throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`); } - this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "tts" }); + this.logger = logger.child({ module: "speech", provider: "local", component: "tts" }); this.speakerId = config.speakerId ?? 0; this.speed = config.speed ?? 1.0; diff --git a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts index 8d66efdc5..8af66a672 100644 --- a/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts +++ b/packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "vitest"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; +import { randomUUID } from "node:crypto"; import pino from "pino"; import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js"; @@ -96,7 +97,7 @@ describe("speech models (download E2E)", () => { const set = getModelSet(); const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-")); - const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "sherpa-onnx"); + const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "local-speech"); const modelIds: SherpaOnnxModelId[] = set === "parakeet-pocket" @@ -114,14 +115,26 @@ describe("speech models (download E2E)", () => { paseoHomeRoot, dictationFinalTimeoutMs: 8000, speech: { - dictationSttProvider: "local", - voiceSttProvider: "local", - voiceTtsProvider: "local", - sherpaOnnx: { + providers: { + dictationStt: { provider: "local", explicit: true }, + voiceStt: { provider: "local", explicit: true }, + voiceTts: { provider: "local", explicit: true }, + }, + local: { modelsDir, autoDownload: false, - stt: { preset: set === "parakeet-pocket" ? "parakeet-tdt-0.6b-v3-int8" : "zipformer-bilingual-zh-en-2023-02-20" }, - tts: { preset: set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16" }, + models: { + dictationStt: + set === "parakeet-pocket" + ? "parakeet-tdt-0.6b-v3-int8" + : "zipformer-bilingual-zh-en-2023-02-20", + voiceStt: + set === "parakeet-pocket" + ? "parakeet-tdt-0.6b-v3-int8" + : "zipformer-bilingual-zh-en-2023-02-20", + voiceTts: + set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16", + }, }, }, }); @@ -178,7 +191,7 @@ describe("speech models (download E2E)", () => { }; }); - await ctx.client.setVoiceConversation(true, `voice-download-${Date.now()}`); + await ctx.client.setVoiceMode(true, randomUUID()); for (let offset = 0; offset < pcm16.length; offset += chunkBytes) { const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes)); const isLast = offset + chunkBytes >= pcm16.length; @@ -188,7 +201,7 @@ describe("speech models (download E2E)", () => { if (voiceText.length > 0) { expect(voiceText).toContain("voice note"); } - await ctx.client.setVoiceConversation(false); + await ctx.client.setVoiceMode(false); // Streaming TTS: generate locally from downloaded model and validate chunking. const ttsText = "This is a voice note."; diff --git a/packages/server/src/server/speech/providers/openai/config.test.ts b/packages/server/src/server/speech/providers/openai/config.test.ts new file mode 100644 index 000000000..10b04364f --- /dev/null +++ b/packages/server/src/server/speech/providers/openai/config.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, test } from "vitest"; + +import { PersistedConfigSchema } from "../../../persisted-config.js"; +import { resolveOpenAiSpeechConfig } from "./config.js"; + +describe("resolveOpenAiSpeechConfig", () => { + test("treats empty OPENAI_API_KEY as unset", () => { + const persisted = PersistedConfigSchema.parse({}); + const env = { + OPENAI_API_KEY: "", + } as NodeJS.ProcessEnv; + + const resolved = resolveOpenAiSpeechConfig({ + env, + persisted, + providers: { + dictationStt: { provider: "local", explicit: false }, + voiceStt: { provider: "local", explicit: false }, + voiceTts: { provider: "local", explicit: false }, + }, + }); + + expect(resolved).toBeUndefined(); + }); + + test("uses trimmed OPENAI_API_KEY when configured", () => { + const persisted = PersistedConfigSchema.parse({}); + const env = { + OPENAI_API_KEY: " sk-test ", + } as NodeJS.ProcessEnv; + + const resolved = resolveOpenAiSpeechConfig({ + env, + persisted, + providers: { + dictationStt: { provider: "openai", explicit: true }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "openai", explicit: true }, + }, + }); + + expect(resolved?.apiKey).toBe("sk-test"); + expect(resolved?.stt?.apiKey).toBe("sk-test"); + expect(resolved?.tts?.apiKey).toBe("sk-test"); + }); +}); diff --git a/packages/server/src/server/speech/providers/openai/config.ts b/packages/server/src/server/speech/providers/openai/config.ts new file mode 100644 index 000000000..b71b0ad09 --- /dev/null +++ b/packages/server/src/server/speech/providers/openai/config.ts @@ -0,0 +1,126 @@ +import { z } from "zod"; + +import type { PersistedConfig } from "../../../persisted-config.js"; +import type { RequestedSpeechProviders } from "../../speech-types.js"; +import type { STTConfig } from "./stt.js"; +import type { TTSConfig } from "./tts.js"; + +export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe"; +export const DEFAULT_OPENAI_TTS_MODEL = "tts-1"; + +export type OpenAiSpeechProviderConfig = { + apiKey?: string; + stt?: Partial & { apiKey?: string }; + tts?: Partial & { apiKey?: string }; + realtimeTranscriptionModel?: string; +}; + +const OpenAiTtsVoiceSchema = z.enum([ + "alloy", + "echo", + "fable", + "onyx", + "nova", + "shimmer", +]); + +const OpenAiTtsModelSchema = z.enum(["tts-1", "tts-1-hd"]); + +const NumberLikeSchema = z.union([ + z.number(), + z.string().trim().min(1), +]); + +const OptionalFiniteNumberSchema = NumberLikeSchema + .pipe(z.coerce.number().finite()) + .optional(); + +const OptionalTrimmedStringSchema = z + .string() + .trim() + .optional() + .transform((value) => (value && value.length > 0 ? value : undefined)); + +const OpenAiSpeechResolutionSchema = z.object({ + apiKey: OptionalTrimmedStringSchema, + sttConfidenceThreshold: OptionalFiniteNumberSchema, + sttModel: OptionalTrimmedStringSchema, + ttsVoice: z + .string() + .trim() + .toLowerCase() + .pipe(OpenAiTtsVoiceSchema) + .default("alloy"), + ttsModel: z + .string() + .trim() + .toLowerCase() + .pipe(OpenAiTtsModelSchema) + .default(DEFAULT_OPENAI_TTS_MODEL), + realtimeTranscriptionModel: OptionalTrimmedStringSchema.default( + DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL + ), +}); + +export function resolveOpenAiSpeechConfig(params: { + env: NodeJS.ProcessEnv; + persisted: PersistedConfig; + providers: RequestedSpeechProviders; +}): OpenAiSpeechProviderConfig | undefined { + const parsed = OpenAiSpeechResolutionSchema.parse({ + apiKey: params.env.OPENAI_API_KEY ?? params.persisted.providers?.openai?.apiKey, + sttConfidenceThreshold: + params.env.STT_CONFIDENCE_THRESHOLD ?? + params.persisted.features?.dictation?.stt?.confidenceThreshold, + sttModel: + params.env.STT_MODEL ?? + (params.providers.voiceStt.provider === "openai" + ? params.persisted.features?.voiceMode?.stt?.model + : undefined) ?? + (params.providers.dictationStt.provider === "openai" + ? params.persisted.features?.dictation?.stt?.model + : undefined), + ttsVoice: + params.env.TTS_VOICE ?? + (params.providers.voiceTts.provider === "openai" + ? params.persisted.features?.voiceMode?.tts?.voice + : undefined) ?? + "alloy", + ttsModel: + params.env.TTS_MODEL ?? + (params.providers.voiceTts.provider === "openai" + ? params.persisted.features?.voiceMode?.tts?.model + : undefined) ?? + DEFAULT_OPENAI_TTS_MODEL, + realtimeTranscriptionModel: + params.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? + (params.providers.dictationStt.provider === "openai" + ? params.persisted.features?.dictation?.stt?.model + : undefined) ?? + DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, + }); + + if (!parsed.apiKey) { + return undefined; + } + + return { + apiKey: parsed.apiKey, + stt: { + apiKey: parsed.apiKey, + ...(parsed.sttConfidenceThreshold !== undefined + ? { confidenceThreshold: parsed.sttConfidenceThreshold } + : {}), + ...(parsed.sttModel + ? { model: parsed.sttModel } + : {}), + }, + tts: { + apiKey: parsed.apiKey, + voice: parsed.ttsVoice, + model: parsed.ttsModel, + responseFormat: "pcm", + }, + realtimeTranscriptionModel: parsed.realtimeTranscriptionModel, + }; +} diff --git a/packages/server/src/server/speech/providers/openai/runtime.ts b/packages/server/src/server/speech/providers/openai/runtime.ts new file mode 100644 index 000000000..64d29eaf6 --- /dev/null +++ b/packages/server/src/server/speech/providers/openai/runtime.ts @@ -0,0 +1,171 @@ +import type { Logger } from "pino"; + +import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js"; +import type { RequestedSpeechProviders } from "../../speech-types.js"; +import { + DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, + DEFAULT_OPENAI_TTS_MODEL, + type OpenAiSpeechProviderConfig, +} from "./config.js"; +import { OpenAIRealtimeTranscriptionSession } from "./realtime-transcription-session.js"; +import { OpenAISTT } from "./stt.js"; +import { OpenAITTS } from "./tts.js"; + +type OpenAiCredentialState = { + openaiSttApiKey: string | undefined; + openaiTtsApiKey: string | undefined; + openaiDictationApiKey: string | undefined; +}; + +export type OpenAiSpeechAvailability = { + stt: boolean; + tts: boolean; + dictationStt: boolean; +}; + +export type SpeechServices = { + sttService: SpeechToTextProvider | null; + ttsService: TextToSpeechProvider | null; + dictationSttService: SpeechToTextProvider | null; +}; + +function resolveOpenAiCredentials( + openaiConfig: OpenAiSpeechProviderConfig | undefined +): OpenAiCredentialState { + const openaiApiKey = openaiConfig?.apiKey; + return { + openaiSttApiKey: openaiConfig?.stt?.apiKey ?? openaiApiKey, + openaiTtsApiKey: openaiConfig?.tts?.apiKey ?? openaiApiKey, + openaiDictationApiKey: openaiApiKey, + }; +} + +export function getOpenAiSpeechAvailability( + openaiConfig: OpenAiSpeechProviderConfig | undefined +): OpenAiSpeechAvailability { + const credentials = resolveOpenAiCredentials(openaiConfig); + return { + stt: Boolean(credentials.openaiSttApiKey), + tts: Boolean(credentials.openaiTtsApiKey), + dictationStt: Boolean(credentials.openaiDictationApiKey), + }; +} + +export function validateOpenAiCredentialRequirements(params: { + providers: RequestedSpeechProviders; + openaiConfig: OpenAiSpeechProviderConfig | undefined; + logger: Logger; +}): void { + const { providers, logger, openaiConfig } = params; + const openAiCredentials = resolveOpenAiCredentials(openaiConfig); + + const missingOpenAiCredentialsFor: string[] = []; + if (providers.voiceStt.provider === "openai" && !openAiCredentials.openaiSttApiKey) { + missingOpenAiCredentialsFor.push("voice.stt"); + } + if (providers.voiceTts.provider === "openai" && !openAiCredentials.openaiTtsApiKey) { + missingOpenAiCredentialsFor.push("voice.tts"); + } + if ( + providers.dictationStt.provider === "openai" && + !openAiCredentials.openaiDictationApiKey + ) { + missingOpenAiCredentialsFor.push("dictation.stt"); + } + + if (missingOpenAiCredentialsFor.length > 0) { + logger.error( + { + requestedProviders: { + dictationStt: providers.dictationStt.provider, + voiceStt: providers.voiceStt.provider, + voiceTts: providers.voiceTts.provider, + }, + missingOpenAiCredentialsFor, + }, + "Invalid speech configuration: OpenAI provider selected but credentials are missing" + ); + throw new Error( + `Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}` + ); + } +} + +export function initializeOpenAiSpeechServices(params: { + providers: RequestedSpeechProviders; + openaiConfig: OpenAiSpeechProviderConfig | undefined; + existing: SpeechServices; + logger: Logger; +}): SpeechServices { + const { providers, openaiConfig, existing, logger } = params; + const openAiCredentials = resolveOpenAiCredentials(openaiConfig); + + let sttService = existing.sttService; + let ttsService = existing.ttsService; + let dictationSttService = existing.dictationSttService; + + const needsOpenAiStt = !sttService && providers.voiceStt.provider === "openai"; + const needsOpenAiTts = !ttsService && providers.voiceTts.provider === "openai"; + const needsOpenAiDictation = + !dictationSttService && providers.dictationStt.provider === "openai"; + + if ( + (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) && + (openAiCredentials.openaiSttApiKey || + openAiCredentials.openaiTtsApiKey || + openAiCredentials.openaiDictationApiKey) + ) { + logger.info("OpenAI speech provider initialized"); + + if (needsOpenAiStt && openAiCredentials.openaiSttApiKey) { + const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {}; + sttService = new OpenAISTT( + { + apiKey: openAiCredentials.openaiSttApiKey, + ...sttConfig, + }, + logger + ); + } + + if (needsOpenAiTts && openAiCredentials.openaiTtsApiKey) { + const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {}; + ttsService = new OpenAITTS( + { + apiKey: openAiCredentials.openaiTtsApiKey, + voice: "alloy", + model: DEFAULT_OPENAI_TTS_MODEL, + responseFormat: "pcm", + ...ttsConfig, + }, + logger + ); + } + + const dictationApiKey = openAiCredentials.openaiDictationApiKey; + if (needsOpenAiDictation && dictationApiKey) { + dictationSttService = { + id: "openai", + createSession: ({ logger: sessionLogger, language, prompt }) => + new OpenAIRealtimeTranscriptionSession({ + apiKey: dictationApiKey, + logger: sessionLogger, + transcriptionModel: + openaiConfig?.realtimeTranscriptionModel ?? + DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, + ...(language ? { language } : {}), + ...(prompt ? { prompt } : {}), + turnDetection: null, + }), + }; + } + } else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) { + logger.warn("OpenAI speech providers are configured but credentials are missing"); + } + + return { + sttService, + ttsService, + dictationSttService, + }; +} diff --git a/packages/server/src/server/speech/speech-config-resolver.test.ts b/packages/server/src/server/speech/speech-config-resolver.test.ts new file mode 100644 index 000000000..99c5a955f --- /dev/null +++ b/packages/server/src/server/speech/speech-config-resolver.test.ts @@ -0,0 +1,127 @@ +import path from "node:path"; + +import { describe, expect, test } from "vitest"; + +import { PersistedConfigSchema } from "../persisted-config.js"; +import { resolveSpeechConfig } from "./speech-config-resolver.js"; + +describe("resolveSpeechConfig", () => { + test("resolves local-first defaults without env overrides", () => { + const paseoHome = "/tmp/paseo-home"; + const persisted = PersistedConfigSchema.parse({}); + const env = {} as NodeJS.ProcessEnv; + + const result = resolveSpeechConfig({ + paseoHome, + env, + persisted, + }); + + expect(result.openai).toBeUndefined(); + expect(result.speech.providers.dictationStt).toEqual({ + provider: "local", + explicit: false, + }); + expect(result.speech.providers.voiceStt).toEqual({ + provider: "local", + explicit: false, + }); + expect(result.speech.providers.voiceTts).toEqual({ + provider: "local", + explicit: false, + }); + expect(result.speech.local).toEqual({ + modelsDir: path.join(paseoHome, "models", "local-speech"), + autoDownload: true, + models: { + dictationStt: "parakeet-tdt-0.6b-v3-int8", + voiceStt: "parakeet-tdt-0.6b-v3-int8", + voiceTts: "pocket-tts-onnx-int8", + }, + }); + expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8"); + }); + + test("resolves feature-scoped local model env vars", () => { + const persisted = PersistedConfigSchema.parse({ + features: { + voiceMode: { + stt: { provider: "openai", model: "gpt-4o-transcribe" }, + }, + }, + providers: { + openai: { apiKey: "persisted-key" }, + }, + }); + const env = { + PASEO_DICTATION_LOCAL_STT_MODEL: "zipformer", + PASEO_VOICE_LOCAL_STT_MODEL: "parakeet", + PASEO_VOICE_LOCAL_TTS_MODEL: "kitten", + PASEO_VOICE_LOCAL_TTS_SPEAKER_ID: "5", + PASEO_VOICE_LOCAL_TTS_SPEED: "1.35", + PASEO_LOCAL_MODELS_DIR: "/tmp/models", + PASEO_LOCAL_AUTO_DOWNLOAD: "0", + OPENAI_API_KEY: "env-key", + PASEO_VOICE_STT_PROVIDER: "openai", + PASEO_DICTATION_STT_PROVIDER: "local", + PASEO_VOICE_TTS_PROVIDER: "local", + } as NodeJS.ProcessEnv; + + const result = resolveSpeechConfig({ + paseoHome: "/tmp/paseo-home", + env, + persisted, + }); + + expect(result.speech.local).toEqual({ + modelsDir: "/tmp/models", + autoDownload: false, + models: { + dictationStt: "zipformer-bilingual-zh-en-2023-02-20", + voiceStt: "parakeet-tdt-0.6b-v3-int8", + voiceTts: "kitten-nano-en-v0_1-fp16", + voiceTtsSpeakerId: 5, + voiceTtsSpeed: 1.35, + }, + }); + expect(result.speech.providers.dictationStt).toEqual({ + provider: "local", + explicit: true, + }); + expect(result.speech.providers.voiceStt).toEqual({ + provider: "openai", + explicit: true, + }); + expect(result.speech.providers.voiceTts).toEqual({ + provider: "local", + explicit: true, + }); + expect(result.speech.local?.models.dictationStt).toBe("zipformer-bilingual-zh-en-2023-02-20"); + expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.local?.models.voiceTts).toBe("kitten-nano-en-v0_1-fp16"); + expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(5); + expect(result.speech.local?.models.voiceTtsSpeed).toBe(1.35); + expect(result.openai?.apiKey).toBe("env-key"); + expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe"); + }); + + test("ignores deprecated shared local model env vars", () => { + const persisted = PersistedConfigSchema.parse({}); + const env = { + PASEO_LOCAL_STT_MODEL: "zipformer-bilingual-zh-en-2023-02-20", + PASEO_LOCAL_TTS_MODEL: "kitten-nano-en-v0_1-fp16", + } as NodeJS.ProcessEnv; + + const result = resolveSpeechConfig({ + paseoHome: "/tmp/paseo-home", + env, + persisted, + }); + + expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8"); + expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8"); + }); +}); diff --git a/packages/server/src/server/speech/speech-config-resolver.ts b/packages/server/src/server/speech/speech-config-resolver.ts new file mode 100644 index 000000000..0d374954a --- /dev/null +++ b/packages/server/src/server/speech/speech-config-resolver.ts @@ -0,0 +1,105 @@ +import { z } from "zod"; + +import type { PersistedConfig } from "../persisted-config.js"; +import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js"; +import { resolveLocalSpeechConfig } from "./providers/local/config.js"; +import { resolveOpenAiSpeechConfig } from "./providers/openai/config.js"; +import { + SpeechProviderIdSchema, + type RequestedSpeechProvider, + type RequestedSpeechProviders, +} from "./speech-types.js"; + +const OptionalSpeechProviderSchema = z + .string() + .trim() + .toLowerCase() + .pipe(SpeechProviderIdSchema) + .optional(); + +const RequestedSpeechProvidersSchema = z.object({ + dictationStt: OptionalSpeechProviderSchema.default("local"), + voiceStt: OptionalSpeechProviderSchema.default("local"), + voiceTts: OptionalSpeechProviderSchema.default("local"), +}); + +function resolveRequestedSpeechProviders(params: { + env: NodeJS.ProcessEnv; + persisted: PersistedConfig; +}): RequestedSpeechProviders { + const resolveFeatureProvider = ( + configuredValue: string | undefined, + parsedValue: z.infer + ): RequestedSpeechProvider => ({ + provider: parsedValue, + explicit: configuredValue !== undefined, + }); + + const dictationSttProviderFromConfig = + params.env.PASEO_DICTATION_STT_PROVIDER ?? + params.persisted.features?.dictation?.stt?.provider; + const voiceSttProviderFromConfig = + params.env.PASEO_VOICE_STT_PROVIDER ?? + params.persisted.features?.voiceMode?.stt?.provider; + const voiceTtsProviderFromConfig = + params.env.PASEO_VOICE_TTS_PROVIDER ?? + params.persisted.features?.voiceMode?.tts?.provider; + + const parsed = RequestedSpeechProvidersSchema.parse({ + dictationStt: dictationSttProviderFromConfig ?? "local", + voiceStt: voiceSttProviderFromConfig ?? "local", + voiceTts: voiceTtsProviderFromConfig ?? "local", + }); + + return { + dictationStt: resolveFeatureProvider( + dictationSttProviderFromConfig, + parsed.dictationStt + ), + voiceStt: resolveFeatureProvider( + voiceSttProviderFromConfig, + parsed.voiceStt + ), + voiceTts: resolveFeatureProvider( + voiceTtsProviderFromConfig, + parsed.voiceTts + ), + }; +} + +export function resolveSpeechConfig(params: { + paseoHome: string; + env: NodeJS.ProcessEnv; + persisted: PersistedConfig; +}): { + openai: PaseoOpenAIConfig | undefined; + speech: PaseoSpeechConfig; +} { + const providers = resolveRequestedSpeechProviders({ + env: params.env, + persisted: params.persisted, + }); + + const local = resolveLocalSpeechConfig({ + paseoHome: params.paseoHome, + env: params.env, + persisted: params.persisted, + providers, + }); + + const openai = resolveOpenAiSpeechConfig({ + env: params.env, + persisted: params.persisted, + providers, + }); + + return { + openai, + speech: { + providers, + ...(local.local + ? { local: local.local } + : {}), + }, + }; +} diff --git a/packages/server/src/server/speech/speech-runtime.ts b/packages/server/src/server/speech/speech-runtime.ts new file mode 100644 index 000000000..9b1280cbd --- /dev/null +++ b/packages/server/src/server/speech/speech-runtime.ts @@ -0,0 +1,173 @@ +import type { Logger } from "pino"; + +import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js"; +import { + getLocalSpeechAvailability, + initializeLocalSpeechServices, +} from "./providers/local/runtime.js"; +import type { LocalSpeechModelId } from "./providers/local/config.js"; +import { + getOpenAiSpeechAvailability, + initializeOpenAiSpeechServices, + validateOpenAiCredentialRequirements, +} from "./providers/openai/runtime.js"; +import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js"; +import type { RequestedSpeechProviders } from "./speech-types.js"; + +function resolveRequestedSpeechProviders( + speechConfig: PaseoSpeechConfig | null +): RequestedSpeechProviders { + const fromConfig = speechConfig?.providers; + if (fromConfig) { + return fromConfig; + } + + return { + dictationStt: { provider: "local", explicit: false }, + voiceStt: { provider: "local", explicit: false }, + voiceTts: { provider: "local", explicit: false }, + }; +} + +export type InitializedSpeechRuntime = { + sttService: SpeechToTextProvider | null; + ttsService: TextToSpeechProvider | null; + dictationSttService: SpeechToTextProvider | null; + cleanup: () => void; + localModelConfig: { + modelsDir: string; + defaultModelIds: LocalSpeechModelId[]; + } | null; +}; + +export async function initializeSpeechRuntime(params: { + logger: Logger; + openaiConfig?: PaseoOpenAIConfig; + speechConfig?: PaseoSpeechConfig; +}): Promise { + const logger = params.logger; + const speechConfig = params.speechConfig ?? null; + const openaiConfig = params.openaiConfig; + const providers = resolveRequestedSpeechProviders(speechConfig); + + validateOpenAiCredentialRequirements({ + providers, + openaiConfig, + logger, + }); + + logger.info( + { + requestedProviders: { + dictationStt: providers.dictationStt.provider, + voiceStt: providers.voiceStt.provider, + voiceTts: providers.voiceTts.provider, + }, + availability: { + openai: getOpenAiSpeechAvailability(openaiConfig), + local: getLocalSpeechAvailability(speechConfig), + }, + }, + "Speech provider reconciliation started" + ); + + const localSpeech = await initializeLocalSpeechServices({ + providers, + speechConfig, + logger, + }); + + const openAiSpeech = initializeOpenAiSpeechServices({ + providers, + openaiConfig, + existing: { + sttService: localSpeech.sttService, + ttsService: localSpeech.ttsService, + dictationSttService: localSpeech.dictationSttService, + }, + logger, + }); + + const effectiveProviders = { + dictationStt: openAiSpeech.dictationSttService?.id ?? "unavailable", + voiceStt: openAiSpeech.sttService?.id ?? "unavailable", + voiceTts: + !openAiSpeech.ttsService + ? "unavailable" + : openAiSpeech.ttsService === localSpeech.localVoiceTtsProvider + ? "local" + : "openai", + }; + const unavailableFeatures = [ + !openAiSpeech.dictationSttService ? "dictation.stt" : null, + !openAiSpeech.sttService ? "voice.stt" : null, + !openAiSpeech.ttsService ? "voice.tts" : null, + ].filter((feature): feature is string => feature !== null); + const explicitlyConfiguredUnavailableFeatures = unavailableFeatures.filter((feature) => { + if (feature === "dictation.stt") { + return providers.dictationStt.explicit; + } + if (feature === "voice.stt") { + return providers.voiceStt.explicit; + } + return providers.voiceTts.explicit; + }); + + if (explicitlyConfiguredUnavailableFeatures.length > 0) { + logger.error( + { + requestedProviders: { + dictationStt: providers.dictationStt.provider, + voiceStt: providers.voiceStt.provider, + voiceTts: providers.voiceTts.provider, + }, + explicitProviders: { + dictationStt: providers.dictationStt.explicit, + voiceStt: providers.voiceStt.explicit, + voiceTts: providers.voiceTts.explicit, + }, + effectiveProviders, + unavailableFeatures: explicitlyConfiguredUnavailableFeatures, + }, + "Speech provider reconciliation failed: configured features are unavailable" + ); + throw new Error( + `Configured speech features unavailable: ${explicitlyConfiguredUnavailableFeatures.join(", ")}` + ); + } + + if (unavailableFeatures.length > 0) { + logger.warn( + { + requestedProviders: { + dictationStt: providers.dictationStt.provider, + voiceStt: providers.voiceStt.provider, + voiceTts: providers.voiceTts.provider, + }, + explicitProviders: { + dictationStt: providers.dictationStt.explicit, + voiceStt: providers.voiceStt.explicit, + voiceTts: providers.voiceTts.explicit, + }, + effectiveProviders, + unavailableFeatures, + }, + "Speech provider reconciliation completed with unavailable default features" + ); + } else { + logger.info( + { + effectiveProviders, + }, + "Speech provider reconciliation completed" + ); + } + + return { + sttService: openAiSpeech.sttService, + ttsService: openAiSpeech.ttsService, + dictationSttService: openAiSpeech.dictationSttService, + cleanup: localSpeech.cleanup, + localModelConfig: localSpeech.localModelConfig, + }; +} diff --git a/packages/server/src/server/speech/speech-types.ts b/packages/server/src/server/speech/speech-types.ts new file mode 100644 index 000000000..d9606addc --- /dev/null +++ b/packages/server/src/server/speech/speech-types.ts @@ -0,0 +1,16 @@ +import { z } from "zod"; + +export const SpeechProviderIdSchema = z.enum(["openai", "local"]); +export type SpeechProviderId = z.infer; + +export const RequestedSpeechProviderSchema = z.object({ + provider: SpeechProviderIdSchema, + explicit: z.boolean(), +}); +export type RequestedSpeechProvider = z.infer; + +export type RequestedSpeechProviders = { + dictationStt: RequestedSpeechProvider; + voiceStt: RequestedSpeechProvider; + voiceTts: RequestedSpeechProvider; +}; diff --git a/packages/server/src/server/test-utils/paseo-daemon.ts b/packages/server/src/server/test-utils/paseo-daemon.ts index 7024991b8..44c1a69b8 100644 --- a/packages/server/src/server/test-utils/paseo-daemon.ts +++ b/packages/server/src/server/test-utils/paseo-daemon.ts @@ -21,6 +21,9 @@ type TestPaseoDaemonOptions = { cleanup?: boolean; openai?: PaseoOpenAIConfig; speech?: PaseoSpeechConfig; + voiceLlmProvider?: PaseoDaemonConfig["voiceLlmProvider"]; + voiceLlmProviderExplicit?: boolean; + voiceLlmModel?: string | null; dictationFinalTimeoutMs?: number; }; @@ -78,7 +81,9 @@ export async function createTestPaseoDaemon( appBaseUrl: "https://app.paseo.sh", openai: options.openai, speech: options.speech, - openrouterApiKey: null, + voiceLlmProvider: options.voiceLlmProvider ?? null, + voiceLlmProviderExplicit: options.voiceLlmProviderExplicit ?? false, + voiceLlmModel: options.voiceLlmModel ?? null, dictationFinalTimeoutMs: options.dictationFinalTimeoutMs, downloadTokenTtlMs: options.downloadTokenTtlMs, }; diff --git a/packages/server/src/server/voice-conversation-store.ts b/packages/server/src/server/voice-conversation-store.ts deleted file mode 100644 index c18ad654a..000000000 --- a/packages/server/src/server/voice-conversation-store.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { readFile, writeFile, readdir, unlink, mkdir, stat } from "fs/promises"; -import { join } from "path"; -import type { ModelMessage } from "@ai-sdk/provider-utils"; -import { standardizePrompt } from "ai/internal"; - -type LoggerLike = { - child(bindings: Record): LoggerLike; - info(...args: any[]): void; - debug(...args: any[]): void; - warn(...args: any[]): void; - error(...args: any[]): void; -}; - -function getLogger(logger: LoggerLike): LoggerLike { - return logger.child({ module: "voice-conversation-store" }); -} - -export interface VoiceConversationMetadata { - id: string; - lastUpdated: Date; - messageCount: number; -} - -interface VoiceConversationData { - voiceConversationId: string; - lastUpdated: string; - messageCount: number; - messages: ModelMessage[]; -} - -export class VoiceConversationStore { - private readonly baseDir: string; - - constructor(baseDir: string) { - this.baseDir = baseDir; - } - - private async ensureBaseDir(): Promise { - await mkdir(this.baseDir, { recursive: true }); - } - - public async save( - logger: LoggerLike, - voiceConversationId: string, - messages: ModelMessage[] - ): Promise { - const log = getLogger(logger); - await this.ensureBaseDir(); - - const filepath = join(this.baseDir, `${voiceConversationId}.json`); - const data: VoiceConversationData = { - voiceConversationId, - lastUpdated: new Date().toISOString(), - messageCount: messages.length, - messages, - }; - - await writeFile(filepath, JSON.stringify(data, null, 2), "utf-8"); - log.debug({ voiceConversationId, messageCount: messages.length }, "Saved voice conversation"); - } - - /** - * Load voice conversation from disk. - * Returns null when missing or invalid (best-effort). - */ - public async load( - logger: LoggerLike, - voiceConversationId: string - ): Promise { - const log = getLogger(logger); - const filepath = join(this.baseDir, `${voiceConversationId}.json`); - - try { - await stat(filepath); - } catch { - log.debug({ voiceConversationId }, "Voice conversation not found"); - return null; - } - - try { - const fileContent = await readFile(filepath, "utf-8"); - const data: VoiceConversationData = JSON.parse(fileContent); - - const result = await standardizePrompt({ prompt: data.messages }); - - log.debug( - { voiceConversationId, messageCount: data.messageCount }, - "Loaded voice conversation" - ); - - return result.messages as ModelMessage[]; - } catch (error) { - log.warn({ err: error, voiceConversationId }, "Failed to load voice conversation"); - return null; - } - } - - public async list(logger: LoggerLike): Promise { - const log = getLogger(logger); - try { - await this.ensureBaseDir(); - - const files = await readdir(this.baseDir); - const jsonFiles = files.filter((f) => f.endsWith(".json")); - const conversations: VoiceConversationMetadata[] = []; - - for (const file of jsonFiles) { - try { - const filepath = join(this.baseDir, file); - const fileContent = await readFile(filepath, "utf-8"); - const data: VoiceConversationData = JSON.parse(fileContent); - - conversations.push({ - id: data.voiceConversationId, - lastUpdated: new Date(data.lastUpdated), - messageCount: data.messageCount, - }); - } catch (error) { - log.warn({ err: error, file }, "Failed to read voice conversation file"); - } - } - - conversations.sort( - (a, b) => b.lastUpdated.getTime() - a.lastUpdated.getTime() - ); - return conversations; - } catch (error) { - log.warn({ err: error }, "Failed to list voice conversations"); - return []; - } - } - - public async delete(logger: LoggerLike, voiceConversationId: string): Promise { - const log = getLogger(logger); - const filepath = join(this.baseDir, `${voiceConversationId}.json`); - await unlink(filepath); - log.debug({ voiceConversationId }, "Deleted voice conversation"); - } -} - diff --git a/packages/server/src/server/voice-conversations.e2e.test.ts b/packages/server/src/server/voice-conversations.e2e.test.ts deleted file mode 100644 index 8557315b7..000000000 --- a/packages/server/src/server/voice-conversations.e2e.test.ts +++ /dev/null @@ -1,147 +0,0 @@ -import { describe, test, expect } from "vitest"; -import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; -import { v4 as uuidv4 } from "uuid"; - -import { createTestPaseoDaemon } from "./test-utils/paseo-daemon.js"; -import { DaemonClient } from "./test-utils/daemon-client.js"; - -async function waitForFile(filepath: string, timeoutMs = 5000): Promise { - const start = Date.now(); - // eslint-disable-next-line no-constant-condition - while (true) { - if (existsSync(filepath)) { - return; - } - if (Date.now() - start > timeoutMs) { - throw new Error(`Timed out waiting for file: ${filepath}`); - } - await new Promise((r) => setTimeout(r, 50)); - } -} - -async function waitForJsonFile( - filepath: string, - timeoutMs = 5000 -): Promise { - const start = Date.now(); - // eslint-disable-next-line no-constant-condition - while (true) { - if (existsSync(filepath)) { - try { - const raw = readFileSync(filepath, "utf8"); - if (raw.trim().length > 0) { - return JSON.parse(raw) as T; - } - } catch { - // File may exist but still be mid-write; retry. - } - } - if (Date.now() - start > timeoutMs) { - throw new Error(`Timed out waiting for valid JSON: ${filepath}`); - } - await new Promise((r) => setTimeout(r, 50)); - } -} - -describe("voice conversations - daemon E2E", () => { - test( - "two concurrent clients persist independently under paseoHome/voice-conversations", - async () => { - const daemon = await createTestPaseoDaemon(); - const url = `ws://127.0.0.1:${daemon.port}/ws`; - - const clientA = new DaemonClient({ url }); - const clientB = new DaemonClient({ url }); - await clientA.connect(); - await clientB.connect(); - - try { - const voiceConversationIdA = uuidv4(); - const voiceConversationIdB = uuidv4(); - - await clientA.setVoiceConversation(true, voiceConversationIdA); - await clientB.setVoiceConversation(true, voiceConversationIdB); - - // Minimal traffic to cause a persist without requiring external APIs. - await clientA.setVoiceConversation(false); - await clientB.setVoiceConversation(false); - - const fileA = join( - daemon.paseoHome, - "voice-conversations", - `${voiceConversationIdA}.json` - ); - const fileB = join( - daemon.paseoHome, - "voice-conversations", - `${voiceConversationIdB}.json` - ); - - await waitForFile(fileA); - await waitForFile(fileB); - - const dataA = await waitForJsonFile<{ - voiceConversationId: string; - messageCount: number; - messages: unknown[]; - }>(fileA); - const dataB = await waitForJsonFile<{ - voiceConversationId: string; - messageCount: number; - messages: unknown[]; - }>(fileB); - - expect(dataA.voiceConversationId).toBe(voiceConversationIdA); - expect(dataB.voiceConversationId).toBe(voiceConversationIdB); - expect(dataA.messageCount).toBe(0); - expect(dataB.messageCount).toBe(0); - expect(Array.isArray(dataA.messages)).toBe(true); - expect(Array.isArray(dataB.messages)).toBe(true); - } finally { - await clientA.close().catch(() => undefined); - await clientB.close().catch(() => undefined); - await daemon.close(); - } - }, - 30000 - ); - - test( - "WS attach ignores URL conversationId param for voice conversation state", - async () => { - const daemon = await createTestPaseoDaemon(); - const urlConversationId = `url-${uuidv4()}`; - const url = `ws://127.0.0.1:${daemon.port}/ws?conversationId=${encodeURIComponent( - urlConversationId - )}`; - - const client = new DaemonClient({ url }); - await client.connect(); - - try { - const voiceConversationId = `client-${uuidv4()}`; - await client.setVoiceConversation(true, voiceConversationId); - await client.setVoiceConversation(false); - - const file = join( - daemon.paseoHome, - "voice-conversations", - `${voiceConversationId}.json` - ); - const urlFile = join( - daemon.paseoHome, - "voice-conversations", - `${urlConversationId}.json` - ); - - await waitForFile(file); - expect(existsSync(urlFile)).toBe(false); - } finally { - await client.close().catch(() => undefined); - await daemon.close(); - } - }, - 30000 - ); -}); diff --git a/packages/server/src/server/voice-local-agent.e2e.test.ts b/packages/server/src/server/voice-local-agent.e2e.test.ts new file mode 100644 index 000000000..59e23ee8e --- /dev/null +++ b/packages/server/src/server/voice-local-agent.e2e.test.ts @@ -0,0 +1,136 @@ +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { randomUUID } from "node:crypto"; +import path from "node:path"; +import { readFile } from "node:fs/promises"; + +import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js"; + +const openaiApiKey = process.env.OPENAI_API_KEY ?? null; +const shouldRun = + process.env.PASEO_VOICE_LOCAL_AGENT_E2E === "1" && + Boolean(openaiApiKey) && + !process.env.CI; + +function waitForSignal( + timeoutMs: number, + setup: ( + resolve: (value: T) => void, + reject: (error: Error) => void + ) => () => void +): Promise { + return new Promise((resolve, reject) => { + let cleanup: (() => void) | null = null; + const timeout = setTimeout(() => { + cleanup?.(); + reject(new Error(`Timeout waiting for event after ${timeoutMs}ms`)); + }, timeoutMs); + + cleanup = setup( + (value) => { + clearTimeout(timeout); + cleanup?.(); + resolve(value); + }, + (error) => { + clearTimeout(timeout); + cleanup?.(); + reject(error); + } + ); + }); +} + +(shouldRun ? describe : describe.skip)( + "voice local-agent e2e", + () => { + let ctx: DaemonTestContext; + + beforeAll(async () => { + ctx = await createDaemonTestContext({ + agentClients: {}, + openai: { apiKey: openaiApiKey! }, + speech: { + providers: { + dictationStt: { provider: "openai", explicit: true }, + voiceStt: { provider: "openai", explicit: true }, + voiceTts: { provider: "openai", explicit: true }, + }, + }, + voiceLlmProvider: "codex", + voiceLlmProviderExplicit: true, + voiceLlmModel: "gpt-5.2-mini", + }); + }, 120000); + + afterAll(async () => { + await ctx.cleanup(); + }, 60000); + + test( + "routes voice turns through local agent speak tool", + async () => { + await ctx.client.setVoiceMode(true, randomUUID()); + + const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => { + const offAudio = ctx.client.on("audio_output", (msg) => { + if (msg.type !== "audio_output") return; + resolve({ chunkId: msg.payload.id }); + }); + const offError = ctx.client.on("activity_log", (msg) => { + if (msg.type !== "activity_log") return; + if (msg.payload.type !== "error") return; + reject(new Error(String(msg.payload.content))); + }); + return () => { + offAudio(); + offError(); + }; + }); + + const assistantLogPromise = waitForSignal(120000, (resolve, reject) => { + const offLog = ctx.client.on("activity_log", (msg) => { + if (msg.type !== "activity_log") return; + if (msg.payload.type !== "assistant") return; + const content = String(msg.payload.content ?? ""); + if (!content.trim()) return; + resolve(content); + }); + const offError = ctx.client.on("activity_log", (msg) => { + if (msg.type !== "activity_log") return; + if (msg.payload.type !== "error") return; + reject(new Error(String(msg.payload.content))); + }); + return () => { + offLog(); + offError(); + }; + }); + + const fixturePath = path.resolve( + process.cwd(), + "..", + "app", + "e2e", + "fixtures", + "recording.wav" + ); + const wav = await readFile(fixturePath); + await ctx.client.sendVoiceAudioChunk(wav.toString("base64"), "audio/wav", true); + + const [{ chunkId }, assistantText] = await Promise.all([ + audioPromise, + assistantLogPromise, + ]); + + expect(chunkId.length).toBeGreaterThan(0); + expect(assistantText.trim().length).toBeGreaterThan(0); + + const agents = await ctx.client.fetchAgents(); + expect( + agents.some((agent) => String(agent.labels?.surface ?? "") === "voice") + ).toBe(false); + }, + 180000 + ); + } +); diff --git a/packages/server/src/server/voice-mcp-bridge.test.ts b/packages/server/src/server/voice-mcp-bridge.test.ts new file mode 100644 index 000000000..942d55925 --- /dev/null +++ b/packages/server/src/server/voice-mcp-bridge.test.ts @@ -0,0 +1,93 @@ +import os from "node:os"; +import path from "node:path"; +import { mkdtemp, rm } from "node:fs/promises"; +import { describe, expect, test } from "vitest"; +import { experimental_createMCPClient } from "ai"; +import { z } from "zod"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import pino from "pino"; + +import { createVoiceMcpBridgeSocketServer } from "./voice-mcp-bridge.js"; + +describe("voice MCP bridge", () => { + test("proxies stdio MCP messages through unix socket bridge", async () => { + const tmpRoot = await mkdtemp(path.join(os.tmpdir(), "paseo-voice-mcp-bridge-")); + const socketPath = path.join(tmpRoot, "voice-mcp.sock"); + const callerAgentId = "voice-agent-bridge-test"; + + const bridge = createVoiceMcpBridgeSocketServer({ + socketPath, + logger: pino({ level: "silent" }), + createAgentMcpServerForCaller: async (callerId) => { + const server = new McpServer({ + name: "bridge-test-server", + version: "1.0.0", + }); + + server.registerTool( + "echo_caller", + { + value: z.string().optional(), + }, + async (args) => { + return { + content: [ + { + type: "text", + text: JSON.stringify({ + callerAgentId: callerId, + value: args.value ?? null, + }), + }, + ], + structuredContent: { + callerAgentId: callerId, + value: args.value ?? null, + }, + }; + } + ); + + return server; + }, + }); + + await bridge.start(); + + const tsxBin = path.resolve(process.cwd(), "../../node_modules/.bin/tsx"); + const serverIndex = path.resolve(process.cwd(), "src/server/index.ts"); + + const transport = new StdioClientTransport({ + command: process.execPath, + args: [ + tsxBin, + serverIndex, + "__paseo_voice_mcp_bridge", + "--socket", + socketPath, + "--caller-agent-id", + callerAgentId, + ], + }); + + const client = await experimental_createMCPClient({ transport }); + + try { + const result = await client.callTool({ + name: "echo_caller", + args: { value: "ok" }, + }); + + const payload = + ((result as { structuredContent?: { callerAgentId?: string; value?: string | null } }) + .structuredContent) ?? null; + + expect(payload?.callerAgentId).toBe(callerAgentId); + } finally { + await client.close(); + await bridge.stop(); + await rm(tmpRoot, { recursive: true, force: true }); + } + }, 30_000); +}); diff --git a/packages/server/src/server/voice-mcp-bridge.ts b/packages/server/src/server/voice-mcp-bridge.ts new file mode 100644 index 000000000..ed6728fb6 --- /dev/null +++ b/packages/server/src/server/voice-mcp-bridge.ts @@ -0,0 +1,337 @@ +import net from "node:net"; +import path from "node:path"; +import { mkdir, rm } from "node:fs/promises"; +import type { Logger } from "pino"; +import pino from "pino"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import type { JSONRPCMessage } from "@modelcontextprotocol/sdk/types.js"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; + + +type BridgeEnvelope = + | { type: "init"; callerAgentId: string } + | { type: "mcp"; message: JSONRPCMessage } + | { type: "ready" } + | { type: "error"; message: string }; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function parseEnvelope(raw: string): BridgeEnvelope { + const parsed = JSON.parse(raw); + if (!isRecord(parsed) || typeof parsed.type !== "string") { + throw new Error("Invalid bridge envelope"); + } + if (parsed.type === "init") { + const callerAgentId = typeof parsed.callerAgentId === "string" ? parsed.callerAgentId.trim() : ""; + if (!callerAgentId) { + throw new Error("Invalid init payload: callerAgentId is required"); + } + return { type: "init", callerAgentId }; + } + if (parsed.type === "mcp") { + if (!("message" in parsed)) { + throw new Error("Invalid mcp payload: message is required"); + } + return { type: "mcp", message: parsed.message as JSONRPCMessage }; + } + if (parsed.type === "ready") return { type: "ready" }; + if (parsed.type === "error") { + return { type: "error", message: typeof parsed.message === "string" ? parsed.message : "Unknown error" }; + } + throw new Error(`Unknown envelope type: ${parsed.type}`); +} + +function encodeEnvelope(envelope: BridgeEnvelope): string { + return `${JSON.stringify(envelope)}\n`; +} + +export type VoiceMcpBridgeSocketServer = { + socketPath: string; + start: () => Promise; + stop: () => Promise; +}; + +export function createVoiceMcpBridgeSocketServer(params: { + socketPath: string; + logger: Logger; + createAgentMcpServerForCaller: (callerAgentId: string) => Promise<{ connect: (transport: InMemoryTransport) => Promise; close?: () => Promise }>; +}): VoiceMcpBridgeSocketServer { + const logger = params.logger.child({ module: "voice-mcp-bridge" }); + const sockets = new Set(); + + const server = net.createServer((socket) => { + sockets.add(socket); + const connectionLogger = logger.child({ component: "connection" }); + let readBuffer = ""; + let initialized = false; + let clientTransport: InMemoryTransport | null = null; + let mcpServer: { close?: () => Promise } | null = null; + + const send = (payload: BridgeEnvelope) => { + socket.write(encodeEnvelope(payload)); + }; + + const fail = (message: string) => { + send({ type: "error", message }); + socket.end(); + }; + + const cleanup = async () => { + sockets.delete(socket); + const closeTasks: Promise[] = []; + if (clientTransport) { + closeTasks.push(clientTransport.close().catch(() => undefined)); + } + if (mcpServer?.close) { + closeTasks.push(mcpServer.close().catch(() => undefined)); + } + await Promise.all(closeTasks); + }; + + socket.on("data", (chunk) => { + readBuffer += chunk.toString("utf8"); + while (true) { + const newlineIndex = readBuffer.indexOf("\n"); + if (newlineIndex < 0) break; + const line = readBuffer.slice(0, newlineIndex).trim(); + readBuffer = readBuffer.slice(newlineIndex + 1); + if (!line) continue; + + let message: BridgeEnvelope; + try { + message = parseEnvelope(line); + } catch (error) { + fail(error instanceof Error ? error.message : String(error)); + return; + } + + if (message.type === "init") { + if (initialized) { + fail("Bridge already initialized"); + return; + } + initialized = true; + void (async () => { + try { + const [proxyClient, proxyServer] = InMemoryTransport.createLinkedPair(); + const serverInstance = await params.createAgentMcpServerForCaller(message.callerAgentId); + await serverInstance.connect(proxyServer); + await proxyClient.start(); + proxyClient.onmessage = (jsonrpcMessage) => { + send({ type: "mcp", message: jsonrpcMessage }); + }; + clientTransport = proxyClient; + mcpServer = serverInstance; + send({ type: "ready" }); + } catch (error) { + connectionLogger.error({ err: error }, "Failed to initialize voice MCP bridge connection"); + fail(error instanceof Error ? error.message : String(error)); + } + })(); + continue; + } + + if (message.type === "mcp") { + if (!clientTransport) { + fail("Bridge is not initialized"); + return; + } + void clientTransport.send(message.message).catch((error) => { + connectionLogger.error({ err: error }, "Failed to forward MCP message"); + fail(error instanceof Error ? error.message : String(error)); + }); + continue; + } + } + }); + + socket.on("error", (error) => { + connectionLogger.error({ err: error }, "Voice MCP bridge socket error"); + }); + socket.on("close", () => { + void cleanup(); + }); + }); + + return { + socketPath: params.socketPath, + async start() { + await mkdir(path.dirname(params.socketPath), { recursive: true }); + await rm(params.socketPath, { force: true }).catch(() => undefined); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(params.socketPath, () => { + server.off("error", reject); + resolve(); + }); + }); + logger.info({ socketPath: params.socketPath }, "Voice MCP bridge socket server listening"); + }, + async stop() { + for (const socket of sockets) { + socket.destroy(); + } + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) reject(error); + else resolve(); + }); + }); + await rm(params.socketPath, { force: true }).catch(() => undefined); + }, + }; +} + +function parseBridgeCliArgs(argv: string[]): { socketPath: string; callerAgentId: string } { + let socketPath: string | null = null; + let callerAgentId: string | null = null; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--socket") { + socketPath = argv[index + 1] ?? null; + index += 1; + continue; + } + if (arg === "--caller-agent-id") { + callerAgentId = argv[index + 1] ?? null; + index += 1; + continue; + } + } + + if (!socketPath?.trim()) { + throw new Error("Missing required --socket "); + } + if (!callerAgentId?.trim()) { + throw new Error("Missing required --caller-agent-id "); + } + + return { + socketPath: socketPath.trim(), + callerAgentId: callerAgentId.trim(), + }; +} + +export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Promise { + const bridgeLogger = logger ?? pino({ level: "error" }); + const parsed = parseBridgeCliArgs(argv); + + const socket = net.createConnection(parsed.socketPath); + const stdioTransport = new StdioServerTransport(process.stdin, process.stdout); + let socketBuffer = ""; + let stdioStarted = false; + const pendingMcpMessages: JSONRPCMessage[] = []; + + let resolveReady: (() => void) | null = null; + let rejectReady: ((error: Error) => void) | null = null; + const readyPromise = new Promise((resolve, reject) => { + resolveReady = () => { + resolveReady = null; + rejectReady = null; + resolve(); + }; + rejectReady = (error: Error) => { + resolveReady = null; + rejectReady = null; + reject(error); + }; + }); + + const failReady = (message: string) => { + if (rejectReady) { + rejectReady(new Error(message)); + } + }; + + const sendEnvelope = (payload: BridgeEnvelope) => { + socket.write(encodeEnvelope(payload)); + }; + + socket.on("data", (chunk) => { + socketBuffer += chunk.toString("utf8"); + while (true) { + const newlineIndex = socketBuffer.indexOf("\n"); + if (newlineIndex < 0) break; + const line = socketBuffer.slice(0, newlineIndex).trim(); + socketBuffer = socketBuffer.slice(newlineIndex + 1); + if (!line) continue; + + let envelope: BridgeEnvelope; + try { + envelope = parseEnvelope(line); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + bridgeLogger.error({ err: error }, "Failed to parse voice MCP bridge envelope"); + failReady(`Failed to parse voice MCP bridge envelope: ${message}`); + socket.destroy(); + return; + } + if (envelope.type === "ready") { + resolveReady?.(); + continue; + } + if (envelope.type === "error") { + failReady(`Voice MCP bridge error: ${envelope.message}`); + socket.destroy(); + return; + } + if (envelope.type === "mcp") { + if (!stdioStarted) { + pendingMcpMessages.push(envelope.message); + continue; + } + void stdioTransport.send(envelope.message).catch((error) => { + bridgeLogger.error({ err: error }, "Failed to forward MCP message to stdio transport"); + socket.destroy(); + }); + } + } + }); + + socket.on("error", (error) => { + bridgeLogger.error({ err: error }, "Voice MCP bridge socket client error"); + failReady(`Voice MCP bridge socket client error: ${error.message}`); + }); + + await new Promise((resolve, reject) => { + socket.once("connect", () => resolve()); + socket.once("error", reject); + }); + + sendEnvelope({ type: "init", callerAgentId: parsed.callerAgentId }); + + const readyTimeoutPromise = new Promise((_, reject) => { + const timeout = setTimeout(() => { + reject(new Error("Timed out waiting for voice MCP bridge initialization")); + }, 10000); + readyPromise.finally(() => clearTimeout(timeout)).catch(() => undefined); + }); + await Promise.race([readyPromise, readyTimeoutPromise]); + + stdioTransport.onmessage = (message) => { + sendEnvelope({ type: "mcp", message }); + }; + stdioTransport.onerror = (error) => { + bridgeLogger.error({ err: error }, "Voice MCP stdio transport error"); + socket.destroy(); + }; + stdioTransport.onclose = () => { + socket.end(); + }; + + await stdioTransport.start(); + stdioStarted = true; + for (const message of pendingMcpMessages) { + await stdioTransport.send(message); + } + + await new Promise((resolve) => { + socket.once("close", () => resolve()); + process.stdin.once("end", () => resolve()); + }); + + await stdioTransport.close().catch(() => undefined); +} diff --git a/packages/server/src/server/voice-permission-policy.test.ts b/packages/server/src/server/voice-permission-policy.test.ts new file mode 100644 index 000000000..6ca551183 --- /dev/null +++ b/packages/server/src/server/voice-permission-policy.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, test } from "vitest"; + +import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js"; +import { isVoicePermissionAllowed } from "./voice-permission-policy.js"; + +function buildRequest(partial: Partial): AgentPermissionRequest { + return { + id: "req-1", + provider: "codex", + name: "unknown", + kind: "tool", + ...partial, + }; +} + +describe("isVoicePermissionAllowed", () => { + test("allows speak tool", () => { + const result = isVoicePermissionAllowed( + buildRequest({ name: "speak" }) + ); + expect(result).toBe(true); + }); + + test("allows explicit MCP/paseo tool names", () => { + expect( + isVoicePermissionAllowed(buildRequest({ name: "mcp__paseo__create_agent" })) + ).toBe(true); + expect( + isVoicePermissionAllowed(buildRequest({ name: "paseo_create_agent" })) + ).toBe(true); + }); + + test("denies non-tool permission kinds", () => { + const result = isVoicePermissionAllowed( + buildRequest({ kind: "mode", name: "mcp__paseo__create_agent" }) + ); + expect(result).toBe(false); + }); + + test("allows codextool only when metadata references MCP/paseo", () => { + const allowed = isVoicePermissionAllowed( + buildRequest({ + name: "codextool", + metadata: { + questions: [{ question: "Allow codextool to call mcp paseo create_agent?" }], + }, + }) + ); + expect(allowed).toBe(true); + }); + + test("denies codextool when metadata includes shell-like operations", () => { + const denied = isVoicePermissionAllowed( + buildRequest({ + name: "codextool", + metadata: { + questions: [{ question: "Allow codextool to execute shell command?" }], + }, + }) + ); + expect(denied).toBe(false); + }); +}); diff --git a/packages/server/src/server/voice-permission-policy.ts b/packages/server/src/server/voice-permission-policy.ts new file mode 100644 index 000000000..0c0f77da0 --- /dev/null +++ b/packages/server/src/server/voice-permission-policy.ts @@ -0,0 +1,66 @@ +import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js"; + +const ALLOWED_TOKEN_SET = ["mcp", "paseo", "speak"]; +const DENIED_TOKEN_SET = [ + "bash", + "shell", + "terminal", + "command", + "execute", + "edit", + "write", + "read", + "fetch", + "http", + "web", +]; + +function containsAny(text: string, tokens: readonly string[]): boolean { + return tokens.some((token) => text.includes(token)); +} + +function stringifyMetadata(metadata: unknown): string { + try { + return JSON.stringify(metadata ?? {})?.toLowerCase() ?? ""; + } catch { + return ""; + } +} + +/** + * Voice assistant policy: allow only MCP/paseo/speak tool requests. + * All non-tool permission requests are denied. + */ +export function isVoicePermissionAllowed(request: AgentPermissionRequest): boolean { + if (request.kind !== "tool") { + return false; + } + + const normalizedName = request.name.trim().toLowerCase(); + if (!normalizedName) { + return false; + } + + if (normalizedName === "speak") { + return true; + } + if (normalizedName.includes("mcp") || normalizedName.includes("paseo")) { + return true; + } + + if (normalizedName !== "codextool") { + return false; + } + + const metadataText = stringifyMetadata({ + metadata: request.metadata ?? null, + input: request.input ?? null, + }); + if (!metadataText) { + return false; + } + + const mentionsAllowedTooling = containsAny(metadataText, ALLOWED_TOKEN_SET); + const mentionsDeniedTooling = containsAny(metadataText, DENIED_TOKEN_SET); + return mentionsAllowedTooling && !mentionsDeniedTooling; +} diff --git a/packages/server/src/server/voice-types.ts b/packages/server/src/server/voice-types.ts new file mode 100644 index 000000000..b5e75fa58 --- /dev/null +++ b/packages/server/src/server/voice-types.ts @@ -0,0 +1,18 @@ +export type VoiceSpeakHandler = (params: { + text: string; + callerAgentId: string; + signal?: AbortSignal; +}) => Promise; + +export type VoiceCallerContext = { + childAgentDefaultLabels?: Record; + lockedCwd?: string; + allowCustomCwd?: boolean; + enableVoiceTools?: boolean; +}; + +export type VoiceMcpStdioConfig = { + command: string; + baseArgs: string[]; + env?: Record; +}; diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 4d347694e..610420c1a 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -19,8 +19,13 @@ import { Session } from "./session.js"; import type { AgentProvider } from "./agent/agent-sdk-types.js"; import { PushTokenStore } from "./push/token-store.js"; import { PushService } from "./push/push-service.js"; -import { VoiceConversationStore } from "./voice-conversation-store.js"; import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js"; +import type { LocalSpeechModelId } from "./speech/providers/local/models.js"; +import type { + VoiceCallerContext, + VoiceMcpStdioConfig, + VoiceSpeakHandler, +} from "./voice-types.js"; export type AgentMcpTransportFactory = () => Promise; @@ -69,15 +74,26 @@ export class VoiceAssistantWebSocketServer { private readonly stt: SpeechToTextProvider | null; private readonly tts: TextToSpeechProvider | null; private readonly terminalManager: TerminalManager | null; - private readonly voiceConversationStore: VoiceConversationStore; private readonly dictation: { finalTimeoutMs?: number; stt?: SpeechToTextProvider | null; + localModels?: { + modelsDir: string; + defaultModelIds: LocalSpeechModelId[]; + }; } | null; private readonly voice: { - openrouterApiKey?: string | null; + voiceLlmProvider?: AgentProvider | null; + voiceLlmModeId?: string | null; + voiceLlmProviderExplicit?: boolean; voiceLlmModel?: string | null; + voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; } | null; + private readonly voiceSpeakHandlers = new Map< + string, + VoiceSpeakHandler + >(); + private readonly voiceCallerContexts = new Map(); constructor( server: HTTPServer, @@ -92,12 +108,19 @@ export class VoiceAssistantWebSocketServer { speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null }, terminalManager?: TerminalManager | null, voice?: { - openrouterApiKey?: string | null; + voiceLlmProvider?: AgentProvider | null; + voiceLlmModeId?: string | null; + voiceLlmProviderExplicit?: boolean; voiceLlmModel?: string | null; + voiceAgentMcpStdio?: VoiceMcpStdioConfig | null; }, dictation?: { finalTimeoutMs?: number; stt?: SpeechToTextProvider | null; + localModels?: { + modelsDir: string; + defaultModelIds: LocalSpeechModelId[]; + }; } ) { this.logger = logger.child({ module: "websocket-server" }); @@ -110,9 +133,6 @@ export class VoiceAssistantWebSocketServer { this.stt = speech?.stt ?? null; this.tts = speech?.tts ?? null; this.terminalManager = terminalManager ?? null; - this.voiceConversationStore = new VoiceConversationStore( - join(paseoHome, "voice-conversations") - ); this.voice = voice ?? null; this.dictation = dictation ?? null; @@ -208,25 +228,38 @@ export class VoiceAssistantWebSocketServer { const clientId = `client-${++this.clientIdCounter}`; const connectionLogger = this.logger.child({ clientId }); - const session = new Session( + const session = new Session({ clientId, - (msg) => { + onMessage: (msg) => { this.sendToClient(ws, wrapSessionMessage(msg)); }, - connectionLogger.child({ module: "session" }), - this.downloadTokenStore, - this.pushTokenStore, - this.paseoHome, - this.agentManager, - this.agentStorage, - this.createAgentMcpTransport, - this.stt, - this.tts, - this.terminalManager, - this.voiceConversationStore, - this.voice ?? undefined, - this.dictation ?? undefined - ); + logger: connectionLogger.child({ module: "session" }), + downloadTokenStore: this.downloadTokenStore, + pushTokenStore: this.pushTokenStore, + paseoHome: this.paseoHome, + agentManager: this.agentManager, + agentStorage: this.agentStorage, + createAgentMcpTransport: this.createAgentMcpTransport, + stt: this.stt, + tts: this.tts, + terminalManager: this.terminalManager, + voice: this.voice ?? undefined, + voiceBridge: { + registerVoiceSpeakHandler: (agentId, handler) => { + this.voiceSpeakHandlers.set(agentId, handler); + }, + unregisterVoiceSpeakHandler: (agentId) => { + this.voiceSpeakHandlers.delete(agentId); + }, + registerVoiceCallerContext: (agentId, context) => { + this.voiceCallerContexts.set(agentId, context); + }, + unregisterVoiceCallerContext: (agentId) => { + this.voiceCallerContexts.delete(agentId); + }, + }, + dictation: this.dictation ?? undefined, + }); this.sessions.set(ws, session); @@ -263,6 +296,18 @@ export class VoiceAssistantWebSocketServer { }); } + public resolveVoiceSpeakHandler( + callerAgentId: string + ): VoiceSpeakHandler | null { + return this.voiceSpeakHandlers.get(callerAgentId) ?? null; + } + + public resolveVoiceCallerContext( + callerAgentId: string + ): VoiceCallerContext | null { + return this.voiceCallerContexts.get(callerAgentId) ?? null; + } + private async detachSocket( ws: WebSocketLike, connectionLogger: pino.Logger, diff --git a/packages/server/src/shared/messages.ts b/packages/server/src/shared/messages.ts index a8f2a3d03..3b8f88252 100644 --- a/packages/server/src/shared/messages.ts +++ b/packages/server/src/shared/messages.ts @@ -294,11 +294,6 @@ export type AgentStreamEventPayload = z.infer< // Session Inbound Messages (Session receives these) // ============================================================================ -export const UserTextMessageSchema = z.object({ - type: z.literal("user_text"), - text: z.string(), -}); - export const VoiceAudioChunkMessageSchema = z.object({ type: z.literal("voice_audio_chunk"), audio: z.string(), // base64 encoded @@ -337,23 +332,6 @@ export const UnsubscribeAgentUpdatesMessageSchema = z.object({ subscriptionId: z.string(), }); -export const LoadVoiceConversationRequestMessageSchema = z.object({ - type: z.literal("load_voice_conversation_request"), - voiceConversationId: z.string(), - requestId: z.string(), -}); - -export const ListVoiceConversationsRequestMessageSchema = z.object({ - type: z.literal("list_voice_conversations_request"), - requestId: z.string(), -}); - -export const DeleteVoiceConversationRequestMessageSchema = z.object({ - type: z.literal("delete_voice_conversation_request"), - voiceConversationId: z.string(), - requestId: z.string(), -}); - export const DeleteAgentRequestMessageSchema = z.object({ type: z.literal("delete_agent_request"), agentId: z.string(), @@ -366,10 +344,10 @@ export const ArchiveAgentRequestMessageSchema = z.object({ requestId: z.string(), }); -export const SetVoiceConversationMessageSchema = z.object({ - type: z.literal("set_voice_conversation"), +export const SetVoiceModeMessageSchema = z.object({ + type: z.literal("set_voice_mode"), enabled: z.boolean(), - voiceConversationId: z.string().optional(), + voiceAgentId: z.string().optional(), }); export const SendAgentMessageSchema = z.object({ @@ -876,7 +854,6 @@ export const KillTerminalRequestSchema = z.object({ }); export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ - UserTextMessageSchema, VoiceAudioChunkMessageSchema, AbortRequestMessageSchema, AudioPlayedMessageSchema, @@ -884,12 +861,9 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ FetchAgentRequestMessageSchema, SubscribeAgentUpdatesMessageSchema, UnsubscribeAgentUpdatesMessageSchema, - LoadVoiceConversationRequestMessageSchema, - ListVoiceConversationsRequestMessageSchema, - DeleteVoiceConversationRequestMessageSchema, DeleteAgentRequestMessageSchema, ArchiveAgentRequestMessageSchema, - SetVoiceConversationMessageSchema, + SetVoiceModeMessageSchema, SendAgentMessageRequestSchema, WaitForFinishRequestSchema, DictationStreamStartMessageSchema, @@ -1127,15 +1101,6 @@ export const ArtifactMessageSchema = z.object({ }), }); -export const VoiceConversationLoadedMessageSchema = z.object({ - type: z.literal("voice_conversation_loaded"), - payload: z.object({ - voiceConversationId: z.string(), - messageCount: z.number(), - requestId: z.string(), - }), -}); - export const AgentUpdateMessageSchema = z.object({ type: z.literal("agent_update"), payload: z.discriminatedUnion("kind", [ @@ -1225,30 +1190,6 @@ export const WaitForFinishResponseMessageSchema = z.object({ }), }); -export const ListVoiceConversationsResponseMessageSchema = z.object({ - type: z.literal("list_voice_conversations_response"), - payload: z.object({ - conversations: z.array( - z.object({ - id: z.string(), - lastUpdated: z.string(), - messageCount: z.number(), - }) - ), - requestId: z.string(), - }), -}); - -export const DeleteVoiceConversationResponseMessageSchema = z.object({ - type: z.literal("delete_voice_conversation_response"), - payload: z.object({ - voiceConversationId: z.string(), - success: z.boolean(), - error: z.string().optional(), - requestId: z.string(), - }), -}); - export const AgentPermissionRequestMessageSchema = z.object({ type: z.literal("agent_permission_request"), payload: z.object({ @@ -1679,7 +1620,6 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [ RpcErrorMessageSchema, InitializeAgentResponseMessageSchema, ArtifactMessageSchema, - VoiceConversationLoadedMessageSchema, AgentUpdateMessageSchema, AgentStreamMessageSchema, AgentStreamSnapshotMessageSchema, @@ -1691,8 +1631,6 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [ SetAgentModelResponseMessageSchema, SetAgentThinkingResponseMessageSchema, WaitForFinishResponseMessageSchema, - ListVoiceConversationsResponseMessageSchema, - DeleteVoiceConversationResponseMessageSchema, AgentPermissionRequestMessageSchema, AgentPermissionResolvedMessageSchema, AgentDeletedMessageSchema, @@ -1737,9 +1675,6 @@ export type TranscriptionResultMessage = z.infer; export type RpcErrorMessage = z.infer; export type ArtifactMessage = z.infer; -export type VoiceConversationLoadedMessage = z.infer< - typeof VoiceConversationLoadedMessageSchema ->; export type AgentUpdateMessage = z.infer; export type AgentStreamMessage = z.infer; export type AgentStreamSnapshotMessage = z.infer< @@ -1758,12 +1693,6 @@ export type SendAgentMessageResponseMessage = z.infer< export type WaitForFinishResponseMessage = z.infer< typeof WaitForFinishResponseMessageSchema >; -export type ListVoiceConversationsResponseMessage = z.infer< - typeof ListVoiceConversationsResponseMessageSchema ->; -export type DeleteVoiceConversationResponseMessage = z.infer< - typeof DeleteVoiceConversationResponseMessageSchema ->; export type AgentPermissionRequestMessage = z.infer; export type AgentPermissionResolvedMessage = z.infer; export type AgentDeletedMessage = z.infer; @@ -1778,7 +1707,6 @@ export type InitializeAgentResponseMessage = z.infer; // Type exports for inbound message types -export type UserTextMessage = z.infer; export type VoiceAudioChunkMessage = z.infer; export type FetchAgentsRequestMessage = z.infer; export type FetchAgentRequestMessage = z.infer; diff --git a/packages/server/src/test-utils/vitest-setup.ts b/packages/server/src/test-utils/vitest-setup.ts index 3ecf822c6..8f0beb4ad 100644 --- a/packages/server/src/test-utils/vitest-setup.ts +++ b/packages/server/src/test-utils/vitest-setup.ts @@ -1,7 +1,8 @@ import path from "node:path"; import dotenv from "dotenv"; -// Load repo-root .env for integration/E2E tests (OpenAI, etc.) +// Load package-local .env.test first for integration/E2E credentials, then repo-root .env fallback. +dotenv.config({ path: path.resolve(process.cwd(), ".env.test"), override: true }); dotenv.config({ path: path.resolve(process.cwd(), "../.env") }); process.env.GIT_TERMINAL_PROMPT = "0"; diff --git a/packages/server/src/utils/tool-call-parsers.ts b/packages/server/src/utils/tool-call-parsers.ts index cc320b27b..a8cf81df0 100644 --- a/packages/server/src/utils/tool-call-parsers.ts +++ b/packages/server/src/utils/tool-call-parsers.ts @@ -28,6 +28,7 @@ const PrincipalParamSchema = z.union([ z.object({ pattern: z.string() }).transform((d) => ({ type: "text" as const, value: d.pattern })), z.object({ query: z.string() }).transform((d) => ({ type: "text" as const, value: d.query })), z.object({ url: z.string() }).transform((d) => ({ type: "text" as const, value: d.url })), + z.object({ text: z.string() }).transform((d) => ({ type: "text" as const, value: d.text })), // Files array (Codex apply_patch) z.object({ files: z.array(FileEntrySchema).nonempty() }).transform((d) => ({ type: "path" as const, value: d.files[0].path })), // TodoWrite - show in_progress item or count diff --git a/packages/website/public/schemas/paseo.config.v1.json b/packages/website/public/schemas/paseo.config.v1.json index 14a45f390..cb853f22c 100644 --- a/packages/website/public/schemas/paseo.config.v1.json +++ b/packages/website/public/schemas/paseo.config.v1.json @@ -86,8 +86,18 @@ }, "additionalProperties": false }, - "openrouter": { - "$ref": "#/definitions/PaseoConfigV1/properties/providers/properties/openai" + "local": { + "type": "object", + "properties": { + "modelsDir": { + "type": "string", + "minLength": 1 + }, + "autoDownload": { + "type": "boolean" + } + }, + "additionalProperties": false } }, "additionalProperties": false @@ -104,7 +114,8 @@ "provider": { "type": "string", "enum": [ - "openai" + "openai", + "local" ] }, "model": { @@ -129,7 +140,9 @@ "provider": { "type": "string", "enum": [ - "openrouter" + "claude", + "codex", + "opencode" ] }, "model": { @@ -145,7 +158,8 @@ "provider": { "type": "string", "enum": [ - "openai" + "openai", + "local" ] }, "model": { @@ -161,15 +175,13 @@ "provider": { "type": "string", "enum": [ - "openai" + "openai", + "local" ] }, "model": { "type": "string", - "enum": [ - "tts-1", - "tts-1-hd" - ] + "minLength": 1 }, "voice": { "type": "string", diff --git a/packages/website/src/routeTree.gen.ts b/packages/website/src/routeTree.gen.ts index 6c8178098..f02250c52 100644 --- a/packages/website/src/routeTree.gen.ts +++ b/packages/website/src/routeTree.gen.ts @@ -13,6 +13,7 @@ import { Route as DocsRouteImport } from './routes/docs' import { Route as IndexRouteImport } from './routes/index' import { Route as DocsIndexRouteImport } from './routes/docs/index' import { Route as DocsWorktreesRouteImport } from './routes/docs/worktrees' +import { Route as DocsVoiceRouteImport } from './routes/docs/voice' import { Route as DocsSecurityRouteImport } from './routes/docs/security' import { Route as DocsConfigurationRouteImport } from './routes/docs/configuration' import { Route as DocsCliRouteImport } from './routes/docs/cli' @@ -38,6 +39,11 @@ const DocsWorktreesRoute = DocsWorktreesRouteImport.update({ path: '/worktrees', getParentRoute: () => DocsRoute, } as any) +const DocsVoiceRoute = DocsVoiceRouteImport.update({ + id: '/voice', + path: '/voice', + getParentRoute: () => DocsRoute, +} as any) const DocsSecurityRoute = DocsSecurityRouteImport.update({ id: '/security', path: '/security', @@ -66,6 +72,7 @@ export interface FileRoutesByFullPath { '/docs/cli': typeof DocsCliRoute '/docs/configuration': typeof DocsConfigurationRoute '/docs/security': typeof DocsSecurityRoute + '/docs/voice': typeof DocsVoiceRoute '/docs/worktrees': typeof DocsWorktreesRoute '/docs/': typeof DocsIndexRoute } @@ -75,6 +82,7 @@ export interface FileRoutesByTo { '/docs/cli': typeof DocsCliRoute '/docs/configuration': typeof DocsConfigurationRoute '/docs/security': typeof DocsSecurityRoute + '/docs/voice': typeof DocsVoiceRoute '/docs/worktrees': typeof DocsWorktreesRoute '/docs': typeof DocsIndexRoute } @@ -86,6 +94,7 @@ export interface FileRoutesById { '/docs/cli': typeof DocsCliRoute '/docs/configuration': typeof DocsConfigurationRoute '/docs/security': typeof DocsSecurityRoute + '/docs/voice': typeof DocsVoiceRoute '/docs/worktrees': typeof DocsWorktreesRoute '/docs/': typeof DocsIndexRoute } @@ -98,6 +107,7 @@ export interface FileRouteTypes { | '/docs/cli' | '/docs/configuration' | '/docs/security' + | '/docs/voice' | '/docs/worktrees' | '/docs/' fileRoutesByTo: FileRoutesByTo @@ -107,6 +117,7 @@ export interface FileRouteTypes { | '/docs/cli' | '/docs/configuration' | '/docs/security' + | '/docs/voice' | '/docs/worktrees' | '/docs' id: @@ -117,6 +128,7 @@ export interface FileRouteTypes { | '/docs/cli' | '/docs/configuration' | '/docs/security' + | '/docs/voice' | '/docs/worktrees' | '/docs/' fileRoutesById: FileRoutesById @@ -156,6 +168,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof DocsWorktreesRouteImport parentRoute: typeof DocsRoute } + '/docs/voice': { + id: '/docs/voice' + path: '/voice' + fullPath: '/docs/voice' + preLoaderRoute: typeof DocsVoiceRouteImport + parentRoute: typeof DocsRoute + } '/docs/security': { id: '/docs/security' path: '/security' @@ -192,6 +211,7 @@ interface DocsRouteChildren { DocsCliRoute: typeof DocsCliRoute DocsConfigurationRoute: typeof DocsConfigurationRoute DocsSecurityRoute: typeof DocsSecurityRoute + DocsVoiceRoute: typeof DocsVoiceRoute DocsWorktreesRoute: typeof DocsWorktreesRoute DocsIndexRoute: typeof DocsIndexRoute } @@ -201,6 +221,7 @@ const DocsRouteChildren: DocsRouteChildren = { DocsCliRoute: DocsCliRoute, DocsConfigurationRoute: DocsConfigurationRoute, DocsSecurityRoute: DocsSecurityRoute, + DocsVoiceRoute: DocsVoiceRoute, DocsWorktreesRoute: DocsWorktreesRoute, DocsIndexRoute: DocsIndexRoute, } diff --git a/packages/website/src/routes/docs.tsx b/packages/website/src/routes/docs.tsx index d393104b2..dfd820c29 100644 --- a/packages/website/src/routes/docs.tsx +++ b/packages/website/src/routes/docs.tsx @@ -7,6 +7,7 @@ export const Route = createFileRoute('/docs')({ const navigation = [ { name: 'Getting started', href: '/docs' }, + { name: 'Voice', href: '/docs/voice' }, { name: 'Git worktrees', href: '/docs/worktrees' }, { name: 'CLI', href: '/docs/cli' }, { name: 'Configuration', href: '/docs/configuration' }, diff --git a/packages/website/src/routes/docs/configuration.tsx b/packages/website/src/routes/docs/configuration.tsx index fe7522fc9..d303acfbe 100644 --- a/packages/website/src/routes/docs/configuration.tsx +++ b/packages/website/src/routes/docs/configuration.tsx @@ -67,8 +67,7 @@ function Configuration() { "$schema": "https://paseo.sh/schemas/paseo.config.v1.json", "version": 1, "providers": { - "openai": { "apiKey": "..." }, - "openrouter": { "apiKey": "..." } + "openai": { "apiKey": "..." } }, "daemon": { "listen": "127.0.0.1:6767", @@ -79,13 +78,32 @@ function Configuration() { +
+

Voice

+

+ Voice is configured through features.dictation and{' '} + features.voiceMode, with provider credentials under{' '} + providers. +

+

+ For voice philosophy, architecture, and complete local/OpenAI setup examples, see{' '} + Voice docs. +

+
+

Common env vars

  • PASEO_HOME — set Paseo home directory
  • PASEO_LISTEN — override daemon.listen
  • PASEO_ALLOWED_HOSTS — override/extend daemon.allowedHosts
    • -
  • OPENAI_API_KEY and OPENROUTER_API_KEY — override provider keys
    • +
  • OPENAI_API_KEY — override OpenAI provider key
    • +
  • PASEO_VOICE_LLM_PROVIDER — override voice LLM provider (claude, codex, opencode)
    • +
  • PASEO_DICTATION_STT_PROVIDER, PASEO_VOICE_STT_PROVIDER, PASEO_VOICE_TTS_PROVIDER — override voice provider selection (local or openai)
    • +
  • PASEO_LOCAL_MODELS_DIR and PASEO_LOCAL_AUTO_DOWNLOAD — control local model directory and download behavior
    • +
  • PASEO_DICTATION_LOCAL_STT_MODEL — override local dictation STT model
    • +
  • PASEO_VOICE_LOCAL_STT_MODEL, PASEO_VOICE_LOCAL_TTS_MODEL — override local voice STT/TTS models
    • +
  • PASEO_VOICE_LOCAL_TTS_SPEAKER_ID, PASEO_VOICE_LOCAL_TTS_SPEED — optional local voice TTS tuning
@@ -101,4 +119,3 @@ function Configuration() { ) } - diff --git a/packages/website/src/routes/docs/index.tsx b/packages/website/src/routes/docs/index.tsx index 0ba254e3c..69dcfe5ec 100644 --- a/packages/website/src/routes/docs/index.tsx +++ b/packages/website/src/routes/docs/index.tsx @@ -87,20 +87,25 @@ function GettingStarted() {

Voice Setup

- Voice features currently require an OpenAI API key. Set it as an environment variable before running the server: + Paseo includes first-class voice support with a local-first architecture and configurable speech + providers.

-
- $ - export OPENAI_API_KEY=your-key-here -

- Local voice support is coming soon. + For architecture, local model behavior, and provider configuration, see the Voice docs page.

+ + Voice docs +

Next