Voice MCP policy: required prompt, cwd override, and restricted tool path

This commit is contained in:
Mohamed Boudra
2026-02-06 15:55:34 +07:00
parent 72e2fdae2b
commit 34ba33402a
12 changed files with 970 additions and 84 deletions

View File

@@ -58,6 +58,7 @@ describe("create_agent MCP tool", () => {
const missingTitle = await tool.inputSchema.safeParseAsync({
cwd: "/tmp/repo",
initialMode: "default",
initialPrompt: "test",
});
expect(missingTitle.success).toBe(false);
expect(missingTitle.error.issues[0].path).toEqual(["title"]);
@@ -66,6 +67,7 @@ describe("create_agent MCP tool", () => {
cwd: "/tmp/repo",
initialMode: "default",
title: "x".repeat(61),
initialPrompt: "test",
});
expect(tooLong.success).toBe(false);
expect(tooLong.error.issues[0].path).toEqual(["title"]);
@@ -74,10 +76,24 @@ describe("create_agent MCP tool", () => {
cwd: "/tmp/repo",
initialMode: "default",
title: "Short title",
initialPrompt: "test",
});
expect(ok.success).toBe(true);
});
it("requires initialPrompt", async () => {
const { agentManager, agentStorage } = createTestDeps();
const server = await createAgentMcpServer({ agentManager, agentStorage, logger });
const tool = (server as any)._registeredTools["create_agent"];
const parsed = await tool.inputSchema.safeParseAsync({
cwd: "/tmp/repo",
initialMode: "default",
title: "Short title",
});
expect(parsed.success).toBe(false);
expect(parsed.error.issues.some((issue: { path: string[] }) => issue.path[0] === "initialPrompt")).toBe(true);
});
it("passes caller-provided titles directly into createAgent", async () => {
const { agentManager, agentStorage, spies } = createTestDeps();
spies.agentManager.createAgent.mockResolvedValue({
@@ -86,6 +102,7 @@ describe("create_agent MCP tool", () => {
lifecycle: "idle",
currentModeId: null,
availableModes: [],
config: { title: "Fix auth bug" },
} as ManagedAgent);
const server = await createAgentMcpServer({ agentManager, agentStorage, logger });
@@ -93,13 +110,16 @@ describe("create_agent MCP tool", () => {
await tool.callback({
cwd: "/tmp/repo",
title: " Fix auth bug ",
initialPrompt: "Do work",
});
expect(spies.agentManager.createAgent).toHaveBeenCalledWith(
expect.objectContaining({
cwd: "/tmp/repo",
title: "Fix auth bug",
})
}),
undefined,
undefined
);
});
@@ -111,6 +131,7 @@ describe("create_agent MCP tool", () => {
lifecycle: "idle",
currentModeId: null,
availableModes: [],
config: { title: "Fix auth" },
} as ManagedAgent);
const server = await createAgentMcpServer({ agentManager, agentStorage, logger });
@@ -118,12 +139,115 @@ describe("create_agent MCP tool", () => {
await tool.callback({
cwd: "/tmp/repo",
title: " Fix auth ",
initialPrompt: "Do work",
});
expect(spies.agentManager.createAgent).toHaveBeenCalledWith(
expect.objectContaining({
title: "Fix auth",
})
}),
undefined,
undefined
);
});
it("allows caller agents to override cwd and applies caller context labels", async () => {
const { agentManager, agentStorage, spies } = createTestDeps();
spies.agentManager.getAgent.mockReturnValue({
id: "voice-agent",
cwd: "/tmp/voice",
provider: "codex",
currentModeId: "full-access",
} as ManagedAgent);
spies.agentManager.createAgent.mockResolvedValue({
id: "child-agent",
cwd: "/tmp/voice/subdir",
lifecycle: "idle",
currentModeId: null,
availableModes: [],
config: { title: "Child" },
} as ManagedAgent);
const server = await createAgentMcpServer({
agentManager,
agentStorage,
callerAgentId: "voice-agent",
resolveCallerContext: () => ({
childAgentDefaultLabels: { ui: "true" },
allowCustomCwd: true,
}),
logger,
});
const tool = (server as any)._registeredTools["create_agent"];
await tool.callback({
cwd: "subdir",
title: "Child",
agentType: "codex",
initialPrompt: "Do work",
});
expect(spies.agentManager.createAgent).toHaveBeenCalledWith(
expect.objectContaining({
cwd: "/tmp/voice/subdir",
}),
undefined,
{ labels: { ui: "true" } }
);
});
});
describe("speak MCP tool", () => {
const logger = createTestLogger();
it("invokes registered speak handler for caller agent", async () => {
const { agentManager, agentStorage } = createTestDeps();
const speak = vi.fn().mockResolvedValue(undefined);
const server = await createAgentMcpServer({
agentManager,
agentStorage,
callerAgentId: "voice-agent-1",
enableVoiceTools: true,
resolveSpeakHandler: () => speak,
logger,
});
const tool = (server as any)._registeredTools["speak"];
expect(tool).toBeDefined();
await tool.callback({ text: "Hello from voice agent." });
expect(speak).toHaveBeenCalledWith(
expect.objectContaining({
text: "Hello from voice agent.",
callerAgentId: "voice-agent-1",
})
);
});
it("fails when no speak handler exists", async () => {
const { agentManager, agentStorage } = createTestDeps();
const server = await createAgentMcpServer({
agentManager,
agentStorage,
callerAgentId: "voice-agent-2",
enableVoiceTools: true,
resolveSpeakHandler: () => null,
logger,
});
const tool = (server as any)._registeredTools["speak"];
await expect(tool.callback({ text: "Hello." })).rejects.toThrow(
"No speak handler registered for caller agent"
);
});
it("does not register speak tool unless voice tools are enabled", async () => {
const { agentManager, agentStorage } = createTestDeps();
const server = await createAgentMcpServer({
agentManager,
agentStorage,
callerAgentId: "agent-no-voice",
logger,
});
const tool = (server as any)._registeredTools["speak"];
expect(tool).toBeUndefined();
});
});

View File

@@ -38,6 +38,22 @@ export interface AgentMcpServerOptions {
* Used for cwd/mode inheritance when agents spawn child agents.
*/
callerAgentId?: string;
/**
* Optional resolver for session-bound speak handlers.
* Used by hidden voice agents to narrate through daemon-managed TTS.
*/
resolveSpeakHandler?: (
callerAgentId: string
) => ((params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>) | null;
resolveCallerContext?: (
callerAgentId: string
) => {
childAgentDefaultLabels?: Record<string, string>;
lockedCwd?: string;
allowCustomCwd?: boolean;
enableVoiceTools?: boolean;
} | null;
enableVoiceTools?: boolean;
logger: Logger;
}
@@ -252,9 +268,17 @@ async function serializeSnapshotWithMetadata(
export async function createAgentMcpServer(
options: AgentMcpServerOptions
): Promise<McpServer> {
const { agentManager, agentStorage, callerAgentId, logger } = options;
const {
agentManager,
agentStorage,
callerAgentId,
resolveSpeakHandler,
resolveCallerContext,
logger,
} = options;
const childLogger = logger.child({ module: "agent", component: "mcp-server" });
const waitTracker = new WaitForAgentTracker(logger);
const callerContext = callerAgentId ? resolveCallerContext?.(callerAgentId) ?? null : null;
const server = new McpServer({
name: "agent-mcp",
@@ -262,6 +286,12 @@ export async function createAgentMcpServer(
});
const agentToAgentInputSchema = {
cwd: z
.string()
.optional()
.describe(
"Optional working directory. Defaults to the caller agent working directory."
),
title: z
.string()
.trim()
@@ -275,9 +305,10 @@ export async function createAgentMcpServer(
),
initialPrompt: z
.string()
.optional()
.trim()
.min(1, "initialPrompt is required")
.describe(
"Optional task to start immediately after creation (non-blocking)."
"Required first task to run immediately after creation."
),
background: z
.boolean()
@@ -307,9 +338,10 @@ export async function createAgentMcpServer(
),
initialPrompt: z
.string()
.optional()
.trim()
.min(1, "initialPrompt is required")
.describe(
"Optional task to start immediately after creation (non-blocking)."
"Required first task to run immediately after creation."
),
initialMode: z
.string()
@@ -339,6 +371,45 @@ export async function createAgentMcpServer(
? agentToAgentInputSchema
: topLevelInputSchema;
if (options.enableVoiceTools || callerContext?.enableVoiceTools) {
server.registerTool(
"speak",
{
title: "Speak",
description:
"Speak text to the user via daemon-managed voice output. Blocks until playback completes.",
inputSchema: {
text: z
.string()
.trim()
.min(1, "text is required")
.max(4000, "text must be 4000 characters or fewer"),
},
outputSchema: {
ok: z.boolean(),
},
},
async (args, context) => {
if (!callerAgentId) {
throw new Error("speak is only available to agent-scoped MCP sessions");
}
const handler = resolveSpeakHandler?.(callerAgentId) ?? null;
if (!handler) {
throw new Error(`No speak handler registered for caller agent '${callerAgentId}'`);
}
await handler({
text: args.text,
callerAgentId,
signal: (context as { signal?: AbortSignal } | undefined)?.signal,
});
return {
content: [],
structuredContent: ensureValidJson({ ok: true }),
};
}
);
}
server.registerTool(
"create_agent",
{
@@ -363,7 +434,7 @@ export async function createAgentMcpServer(
permission: AgentPermissionRequestPayloadSchema.nullable().optional(),
},
},
async (args) => {
async (args: unknown) => {
const {
agentType,
initialPrompt,
@@ -372,7 +443,7 @@ export async function createAgentMcpServer(
} = args as {
cwd?: string;
agentType?: AgentProvider;
initialPrompt?: string;
initialPrompt: string;
initialMode?: string;
worktreeName?: string;
background?: boolean;
@@ -387,7 +458,19 @@ export async function createAgentMcpServer(
if (!parentAgent) {
throw new Error(`Parent agent ${callerAgentId} not found`);
}
resolvedCwd = parentAgent.cwd;
const callerArgs = args as unknown as { cwd?: string };
const requestedCwd = callerArgs.cwd?.trim();
const lockedCwd = callerContext?.lockedCwd?.trim();
if (lockedCwd) {
resolvedCwd = expandPath(lockedCwd);
} else if (requestedCwd && (callerContext?.allowCustomCwd ?? true)) {
resolvedCwd =
requestedCwd.startsWith("/") || requestedCwd.startsWith("~")
? expandPath(requestedCwd)
: resolve(parentAgent.cwd, requestedCwd);
} else {
resolvedCwd = parentAgent.cwd;
}
const provider: AgentProvider = agentType ?? "claude";
const parentMode = parentAgent.currentModeId;
@@ -433,73 +516,78 @@ export async function createAgentMcpServer(
const provider: AgentProvider = agentType ?? "claude";
const normalizedTitle = title?.trim() ?? null;
const snapshot = await agentManager.createAgent({
provider,
cwd: resolvedCwd,
modeId: resolvedMode,
title: normalizedTitle ?? undefined,
const childAgentDefaultLabels =
callerAgentId && callerContext?.childAgentDefaultLabels
? callerContext.childAgentDefaultLabels
: undefined;
const snapshot = await agentManager.createAgent(
{
provider,
cwd: resolvedCwd,
modeId: resolvedMode,
title: normalizedTitle ?? undefined,
},
undefined,
childAgentDefaultLabels ? { labels: childAgentDefaultLabels } : undefined
);
const trimmedPrompt = initialPrompt.trim();
scheduleAgentMetadataGeneration({
agentManager,
agentId: snapshot.id,
cwd: snapshot.cwd,
initialPrompt: trimmedPrompt,
explicitTitle: snapshot.config.title,
paseoHome: options.paseoHome,
logger: childLogger,
});
const trimmedPrompt = initialPrompt?.trim();
if (trimmedPrompt) {
scheduleAgentMetadataGeneration({
agentManager,
agentId: snapshot.id,
cwd: snapshot.cwd,
initialPrompt: trimmedPrompt,
explicitTitle: snapshot.config.title,
paseoHome: options.paseoHome,
logger: childLogger,
});
try {
agentManager.recordUserMessage(snapshot.id, trimmedPrompt);
} catch (error) {
childLogger.error(
{ err: error, agentId: snapshot.id },
"Failed to record initial prompt"
);
}
try {
startAgentRun(agentManager, snapshot.id, trimmedPrompt, childLogger);
// If not running in background, wait for completion
if (!background) {
const result = await waitForAgentWithTimeout(
agentManager,
snapshot.id,
{ waitForActive: true }
);
const responseData = {
agentId: snapshot.id,
type: provider,
status: result.status,
cwd: snapshot.cwd,
currentModeId: snapshot.currentModeId,
availableModes: snapshot.availableModes,
lastMessage: result.lastMessage,
permission: sanitizePermissionRequest(result.permission),
};
const validJson = ensureValidJson(responseData);
const response = {
content: [],
structuredContent: validJson,
};
return response;
}
} catch (error) {
childLogger.error(
{ err: error, agentId: snapshot.id },
"Failed to run initial prompt"
);
}
} else {
try {
agentManager.recordUserMessage(snapshot.id, trimmedPrompt);
} catch (error) {
childLogger.error(
{ err: error, agentId: snapshot.id },
"Failed to record initial prompt"
);
}
// Return immediately if background=true or no initialPrompt
try {
startAgentRun(agentManager, snapshot.id, trimmedPrompt, childLogger);
// If not running in background, wait for completion
if (!background) {
const result = await waitForAgentWithTimeout(
agentManager,
snapshot.id,
{ waitForActive: true }
);
const responseData = {
agentId: snapshot.id,
type: provider,
status: result.status,
cwd: snapshot.cwd,
currentModeId: snapshot.currentModeId,
availableModes: snapshot.availableModes,
lastMessage: result.lastMessage,
permission: sanitizePermissionRequest(result.permission),
};
const validJson = ensureValidJson(responseData);
const response = {
content: [],
structuredContent: validJson,
};
return response;
}
} catch (error) {
childLogger.error(
{ err: error, agentId: snapshot.id },
"Failed to run initial prompt"
);
}
// Return immediately if background=true
const response = {
content: [],
structuredContent: ensureValidJson({

View File

@@ -9,8 +9,15 @@ import { createTestPaseoDaemon } from "./test-utils/paseo-daemon.js";
import { createTestAgentClients } from "./test-utils/fake-agent-client.js";
describe("paseo daemon bootstrap", () => {
test("starts and serves health endpoint", async () => {
const daemonHandle = await createTestPaseoDaemon();
test.runIf(Boolean(process.env.OPENAI_API_KEY))("starts and serves health endpoint", async () => {
const daemonHandle = await createTestPaseoDaemon({
openai: { apiKey: process.env.OPENAI_API_KEY! },
speech: {
dictationSttProvider: "openai",
voiceSttProvider: "openai",
voiceTtsProvider: "openai",
},
});
try {
const response = await fetch(
`http://127.0.0.1:${daemonHandle.port}/api/health`,

View File

@@ -77,6 +77,8 @@ import { acquirePidLock, releasePidLock } from "./pid-lock.js";
import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js";
type AgentMcpTransportMap = Map<string, StreamableHTTPServerTransport>;
type VoiceAgentProvider = "claude" | "codex" | "opencode";
const VOICE_AGENT_FALLBACK_ORDER: VoiceAgentProvider[] = ["claude", "codex", "opencode"];
export type PaseoOpenAIConfig = {
apiKey?: string;
@@ -121,6 +123,8 @@ export type PaseoDaemonConfig = {
openai?: PaseoOpenAIConfig;
speech?: PaseoSpeechConfig;
openrouterApiKey?: string | null;
voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
dictationFinalTimeoutMs?: number;
downloadTokenTtlMs?: number;
@@ -282,12 +286,88 @@ export async function createPaseoDaemon(
`Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand`
);
const requestedVoiceLlmProvider = config.voiceLlmProvider ?? null;
const voiceLlmProviderExplicit = config.voiceLlmProviderExplicit ?? false;
logger.info(
{
requestedVoiceLlmProvider,
voiceLlmProviderExplicit,
},
"Voice LLM provider reconciliation started"
);
const providerClients = createAllClients(logger);
const voiceLlmAvailability: Record<VoiceAgentProvider, boolean> = {
claude: false,
codex: false,
opencode: false,
};
for (const provider of VOICE_AGENT_FALLBACK_ORDER) {
try {
voiceLlmAvailability[provider] = await providerClients[provider].isAvailable();
} catch (error) {
logger.warn({ err: error, provider }, "Voice LLM provider availability check failed");
voiceLlmAvailability[provider] = false;
}
}
const voiceLlmDefaultProvider =
VOICE_AGENT_FALLBACK_ORDER.find((provider) => voiceLlmAvailability[provider]) ?? null;
if (requestedVoiceLlmProvider === "openrouter") {
const openrouterApiKey =
config.openrouterApiKey ?? process.env.OPENROUTER_API_KEY ?? null;
if (!openrouterApiKey) {
logger.error("voiceMode.llm.provider is openrouter but no OpenRouter API key is configured");
throw new Error("Missing OpenRouter API key for voiceMode.llm.provider=openrouter");
}
} else if (
requestedVoiceLlmProvider === "claude" ||
requestedVoiceLlmProvider === "codex" ||
requestedVoiceLlmProvider === "opencode"
) {
if (!voiceLlmAvailability[requestedVoiceLlmProvider]) {
logger.error(
{ provider: requestedVoiceLlmProvider, voiceLlmAvailability },
"Configured voice LLM provider is unavailable"
);
throw new Error(`Configured voice LLM provider '${requestedVoiceLlmProvider}' is unavailable`);
}
} else if (!voiceLlmDefaultProvider) {
logger.error(
{ requestedVoiceLlmProvider, voiceLlmAvailability },
"No local voice LLM provider available for fallback"
);
throw new Error("No local voice LLM provider available (claude/codex/opencode)");
}
logger.info(
{
requestedVoiceLlmProvider,
voiceLlmProviderExplicit,
voiceLlmAvailability,
voiceLlmDefaultProvider,
},
"Voice LLM provider reconciliation completed"
);
if (listenTarget.type !== "tcp" && requestedVoiceLlmProvider !== "openrouter") {
logger.error(
{ listen: config.listen, requestedVoiceLlmProvider },
"Local voice agent mode requires TCP listen target for HTTP MCP bridge"
);
throw new Error("Local voice agent mode requires TCP listen target");
}
let wsServer: VoiceAssistantWebSocketServer | null = null;
// Create in-memory transport for Session's Agent MCP client (voice assistant tools)
const createInMemoryAgentMcpTransport = async (): Promise<InMemoryTransport> => {
const agentMcpServer = await createAgentMcpServer({
agentManager,
agentStorage,
paseoHome: config.paseoHome,
enableVoiceTools: false,
resolveSpeakHandler: (callerAgentId) => wsServer?.resolveVoiceSpeakHandler(callerAgentId) ?? null,
resolveCallerContext: (callerAgentId) => wsServer?.resolveVoiceCallerContext(callerAgentId) ?? null,
logger,
});
@@ -309,6 +389,9 @@ export async function createPaseoDaemon(
agentStorage,
paseoHome: config.paseoHome,
callerAgentId,
enableVoiceTools: false,
resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null,
resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null,
logger,
});
@@ -792,7 +875,12 @@ export async function createPaseoDaemon(
);
}
const wsServer = new VoiceAssistantWebSocketServer(
const voiceAgentMcpUrl =
listenTarget.type === "tcp"
? `http://127.0.0.1:${listenTarget.port}/mcp/agents`
: null;
wsServer = new VoiceAssistantWebSocketServer(
httpServer,
logger,
serverId,
@@ -806,7 +894,12 @@ export async function createPaseoDaemon(
terminalManager,
{
openrouterApiKey: config.openrouterApiKey ?? null,
voiceLlmProvider: config.voiceLlmProvider ?? null,
voiceLlmProviderExplicit,
voiceLlmDefaultProvider,
voiceLlmModel: config.voiceLlmModel ?? null,
voiceLlmAvailability,
voiceAgentMcpUrl,
},
{
finalTimeoutMs: config.dictationFinalTimeoutMs,
@@ -870,7 +963,12 @@ export async function createPaseoDaemon(
relayTransport?.stop().catch(() => undefined);
relayTransport = startRelayTransport({
logger,
attachSocket: (ws) => wsServer.attachExternalSocket(ws),
attachSocket: (ws) => {
if (!wsServer) {
throw new Error("WebSocket server not initialized");
}
return wsServer.attachExternalSocket(ws);
},
relayEndpoint,
serverId,
daemonKeyPair: daemonKeyPair.keyPair,
@@ -911,7 +1009,9 @@ export async function createPaseoDaemon(
sherpaOnline?.free();
sherpaOffline?.free();
await relayTransport?.stop().catch(() => undefined);
await wsServer.close();
if (wsServer) {
await wsServer.close();
}
await new Promise<void>((resolve) => {
httpServer.close(() => resolve());
});

View File

@@ -13,6 +13,14 @@ import {
const DEFAULT_PORT = 6767;
const DEFAULT_RELAY_ENDPOINT = "relay.paseo.sh:443";
const DEFAULT_APP_BASE_URL = "https://app.paseo.sh";
const VOICE_LLM_PROVIDER_IDS = [
"openrouter",
"local-agent",
"claude",
"codex",
"opencode",
] as const;
type VoiceLlmProviderId = (typeof VOICE_LLM_PROVIDER_IDS)[number];
function getDefaultListen(): string {
// Main HTTP server defaults to TCP
@@ -89,6 +97,19 @@ function parseSpeechProviderId(value: unknown): "openai" | "local" | null {
return null;
}
function parseVoiceLlmProviderId(value: unknown): VoiceLlmProviderId | null {
if (typeof value !== "string") {
return null;
}
const normalized = value.trim().toLowerCase();
if (!normalized) {
return null;
}
return (VOICE_LLM_PROVIDER_IDS as readonly string[]).includes(normalized)
? (normalized as VoiceLlmProviderId)
: null;
}
function normalizeSherpaSttPreset(value: string): string {
const raw = value.trim();
const normalized = raw.toLowerCase();
@@ -246,6 +267,13 @@ export function loadConfig(
const openrouterApiKey =
env.OPENROUTER_API_KEY ?? persisted.providers?.openrouter?.apiKey ?? null;
const envVoiceLlmProvider = parseVoiceLlmProviderId(env.PASEO_VOICE_LLM_PROVIDER);
const persistedVoiceLlmProvider = parseVoiceLlmProviderId(
persisted.features?.voiceMode?.llm?.provider
);
const voiceLlmProvider = envVoiceLlmProvider ?? persistedVoiceLlmProvider ?? null;
const voiceLlmProviderExplicit =
envVoiceLlmProvider !== null || persistedVoiceLlmProvider !== null;
const voiceLlmModel = persisted.features?.voiceMode?.llm?.model ?? null;
return {
@@ -272,6 +300,8 @@ export function loadConfig(
...(sherpaOnnx ? { sherpaOnnx } : {}),
},
openrouterApiKey,
voiceLlmProvider,
voiceLlmProviderExplicit,
voiceLlmModel,
};
}

View File

@@ -74,7 +74,9 @@ const FeatureVoiceModeSchema = z
.object({
llm: z
.object({
provider: z.enum(["openrouter"]).optional(),
provider: z
.enum(["openrouter", "local-agent", "claude", "codex", "opencode"])
.optional(),
model: z.string().min(1).optional(),
})
.strict()

View File

@@ -49,6 +49,15 @@ import { experimental_createMCPClient } from "ai";
import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
export type AgentMcpTransportFactory = () => Promise<Transport>;
type VoiceLlmProvider = "openrouter" | "local-agent" | "claude" | "codex" | "opencode";
type VoiceAgentProvider = Exclude<VoiceLlmProvider, "openrouter" | "local-agent">;
type VoiceSpeakHandler = (params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>;
type VoiceCallerContext = {
childAgentDefaultLabels?: Record<string, string>;
lockedCwd?: string;
allowCustomCwd?: boolean;
enableVoiceTools?: boolean;
};
import { buildProviderRegistry } from "./agent/provider-registry.js";
import { AgentManager } from "./agent/agent-manager.js";
import type { ManagedAgent } from "./agent/agent-manager.js";
@@ -59,6 +68,7 @@ import {
generateStructuredAgentResponse,
} from "./agent/agent-response-loop.js";
import type {
AgentPermissionRequest,
AgentPermissionResponse,
AgentPromptContentBlock,
AgentPromptInput,
@@ -130,6 +140,24 @@ const RESTART_EXIT_DELAY_MS = 250;
* Uses Claude Haiku for speed and cost efficiency.
*/
const AUTO_GEN_MODEL = "haiku";
const VOICE_AGENT_FALLBACK_ORDER: VoiceAgentProvider[] = ["claude", "codex", "opencode"];
const VOICE_AGENT_DEFAULT_MODE: Record<VoiceAgentProvider, string> = {
claude: "default",
codex: "read-only",
opencode: "default",
};
const VOICE_AGENT_DEFAULT_MODEL: Partial<Record<VoiceAgentProvider, string>> = {
claude: "haiku",
codex: "gpt-5.2-mini",
};
const VOICE_AGENT_SYSTEM_INSTRUCTION = [
"You are the Paseo voice assistant.",
"The user cannot see your chat messages or tool calls.",
"Always narrate everything through the speak tool.",
"Use concise plain language suitable for speech output.",
"Never use bash, file-edit, or web tools directly.",
"Only use the paseo MCP tools.",
].join(" ");
type ProcessingPhase = "idle" | "transcribing" | "llm";
@@ -320,7 +348,23 @@ export class Session {
private readonly terminalManager: TerminalManager | null;
private terminalSubscriptions: Map<string, () => void> = new Map();
private readonly openrouterApiKey: string | null;
private readonly voiceLlmProvider: VoiceLlmProvider | null;
private readonly voiceLlmProviderExplicit: boolean;
private readonly voiceLlmDefaultProvider: VoiceAgentProvider | null;
private readonly voiceLlmModel: string | null;
private readonly voiceLlmAvailability: Record<VoiceAgentProvider, boolean> | null;
private readonly voiceAgentMcpUrl: string | null;
private readonly registerVoiceSpeakHandler?: (
agentId: string,
handler: VoiceSpeakHandler
) => void;
private readonly unregisterVoiceSpeakHandler?: (agentId: string) => void;
private readonly registerVoiceCallerContext?: (
agentId: string,
context: VoiceCallerContext
) => void;
private readonly unregisterVoiceCallerContext?: (agentId: string) => void;
private voiceAssistantAgentId: string | null = null;
constructor(
clientId: string,
@@ -338,7 +382,18 @@ export class Session {
voiceConversationStore: VoiceConversationStore,
voice?: {
openrouterApiKey?: string | null;
voiceLlmProvider?: VoiceLlmProvider | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmDefaultProvider?: VoiceAgentProvider | null;
voiceLlmModel?: string | null;
voiceLlmAvailability?: Record<VoiceAgentProvider, boolean> | null;
voiceAgentMcpUrl?: string | null;
},
voiceBridge?: {
registerVoiceSpeakHandler?: (agentId: string, handler: VoiceSpeakHandler) => void;
unregisterVoiceSpeakHandler?: (agentId: string) => void;
registerVoiceCallerContext?: (agentId: string, context: VoiceCallerContext) => void;
unregisterVoiceCallerContext?: (agentId: string) => void;
},
dictation?: {
finalTimeoutMs?: number;
@@ -357,7 +412,16 @@ export class Session {
this.terminalManager = terminalManager;
this.voiceConversationStore = voiceConversationStore;
this.openrouterApiKey = voice?.openrouterApiKey ?? null;
this.voiceLlmProvider = voice?.voiceLlmProvider ?? null;
this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false;
this.voiceLlmDefaultProvider = voice?.voiceLlmDefaultProvider ?? null;
this.voiceLlmModel = voice?.voiceLlmModel ?? null;
this.voiceLlmAvailability = voice?.voiceLlmAvailability ?? null;
this.voiceAgentMcpUrl = voice?.voiceAgentMcpUrl ?? null;
this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler;
this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler;
this.registerVoiceCallerContext = voiceBridge?.registerVoiceCallerContext;
this.unregisterVoiceCallerContext = voiceBridge?.unregisterVoiceCallerContext;
this.abortController = new AbortController();
this.sessionLogger = logger.child({
module: "session",
@@ -4241,7 +4305,7 @@ export class Session {
this.messages.push({ role: "user", content: text });
// Process through LLM (TTS enabled in voice mode for voice conversations)
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode);
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, text);
await this.currentStreamPromise;
}
@@ -4518,7 +4582,7 @@ export class Session {
// Set phase to LLM and process (TTS enabled in voice mode for voice conversations)
this.clearSpeechInProgress("transcription complete");
this.setPhase("llm");
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode);
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, result.text);
await this.currentStreamPromise;
this.setPhase("idle");
} catch (error: any) {
@@ -4537,10 +4601,249 @@ export class Session {
}
}
/**
* Resolve the effective voice LLM provider.
* - explicit provider => strict
* - local-agent / unset => fallback order
*/
private resolveVoiceAgentProvider(): VoiceAgentProvider {
const configured = this.voiceLlmProvider;
const availability = this.voiceLlmAvailability ?? {
claude: true,
codex: true,
opencode: true,
};
if (configured === "openrouter") {
throw new Error("voiceLlmProvider=openrouter cannot be used in local-agent flow");
}
if (configured === "claude" || configured === "codex" || configured === "opencode") {
if (!availability[configured]) {
throw new Error(`Configured voice LLM provider '${configured}' is unavailable`);
}
return configured;
}
const fallbackOrder =
this.voiceLlmDefaultProvider && availability[this.voiceLlmDefaultProvider]
? [this.voiceLlmDefaultProvider, ...VOICE_AGENT_FALLBACK_ORDER.filter((id) => id !== this.voiceLlmDefaultProvider)]
: VOICE_AGENT_FALLBACK_ORDER;
for (const provider of fallbackOrder) {
if (availability[provider]) {
return provider;
}
}
throw new Error("No local voice LLM provider is available (claude/codex/opencode)");
}
private getVoiceAgentModel(provider: VoiceAgentProvider): string | undefined {
const configured = this.voiceLlmModel?.trim();
if (configured) {
return configured;
}
return VOICE_AGENT_DEFAULT_MODEL[provider];
}
private async ensureVoiceAssistantAgent(): Promise<string> {
if (this.voiceAssistantAgentId) {
const existing = this.agentManager.getAgent(this.voiceAssistantAgentId);
if (existing) {
return existing.id;
}
this.voiceAssistantAgentId = null;
}
const provider = this.resolveVoiceAgentProvider();
const voiceAgentId = `voice-${uuidv4()}`;
const cwd = join(this.paseoHome, "voice-agent-workspace");
await mkdir(cwd, { recursive: true });
const mcpUrl = this.voiceAgentMcpUrl;
if (!mcpUrl) {
throw new Error("Voice MCP URL is not configured");
}
const model = this.getVoiceAgentModel(provider);
const config: AgentSessionConfig = {
provider,
cwd,
modeId: VOICE_AGENT_DEFAULT_MODE[provider],
...(model ? { model } : {}),
internal: true,
mcpServers: {
paseo: {
type: "http",
url: `${mcpUrl}?callerAgentId=${encodeURIComponent(voiceAgentId)}`,
},
},
};
const created = await this.agentManager.createAgent(config, voiceAgentId, {
labels: {
surface: "voice",
ui: "false",
},
});
this.voiceAssistantAgentId = created.id;
this.registerVoiceSpeakHandler?.(created.id, async ({ text, signal }) => {
const abortSignal = signal ?? this.abortController.signal;
await this.ttsManager.generateAndWaitForPlayback(
text,
(msg) => this.emit(msg),
abortSignal,
true
);
this.emit({
type: "activity_log",
payload: {
id: uuidv4(),
timestamp: new Date(),
type: "assistant",
content: text,
},
});
});
this.registerVoiceCallerContext?.(created.id, {
childAgentDefaultLabels: { ui: "true" },
allowCustomCwd: true,
enableVoiceTools: true,
});
this.sessionLogger.info(
{
voiceAssistantAgentId: created.id,
provider,
model: model ?? null,
providerExplicit: this.voiceLlmProviderExplicit,
},
"Voice assistant agent initialized"
);
return created.id;
}
private buildVoiceAgentPrompt(userText: string): string {
return [
VOICE_AGENT_SYSTEM_INSTRUCTION,
"",
`User said: ${userText.trim()}`,
].join("\n");
}
private shouldAllowVoicePermission(request: AgentPermissionRequest): boolean {
const name = request.name.toLowerCase();
if (name.includes("mcp") || name.includes("paseo") || name.includes("speak")) {
return true;
}
if (name === "codextool") {
const metadata = request.metadata ?? {};
const rawQuestions = metadata.questions;
if (Array.isArray(rawQuestions)) {
const text = JSON.stringify(rawQuestions).toLowerCase();
return text.includes("mcp") || text.includes("paseo") || text.includes("speak");
}
return false;
}
return false;
}
private async processWithVoiceAgent(userText: string): Promise<void> {
const agentId = await this.ensureVoiceAssistantAgent();
await this.interruptAgentIfRunning(agentId);
const prompt = this.buildVoiceAgentPrompt(userText);
this.agentManager.recordUserMessage(agentId, userText);
let sawSpeakToolCall = false;
const assistantTextChunks: string[] = [];
const iterator = this.agentManager.streamAgent(agentId, prompt);
for await (const event of iterator) {
if (event.type === "turn_failed") {
throw new Error(event.error);
}
if (event.type === "timeline") {
if (event.item.type === "tool_call" && typeof event.item.name === "string") {
if (event.item.name.toLowerCase().includes("speak")) {
sawSpeakToolCall = true;
}
}
if (event.item.type === "assistant_message" && event.item.text.trim().length > 0) {
assistantTextChunks.push(event.item.text.trim());
}
}
if (event.type === "permission_requested") {
if (this.shouldAllowVoicePermission(event.request)) {
await this.agentManager.respondToPermission(agentId, event.request.id, {
behavior: "allow",
});
} else {
await this.agentManager.respondToPermission(agentId, event.request.id, {
behavior: "deny",
message: "Voice assistant policy only allows MCP paseo tools.",
interrupt: true,
});
throw new Error(
`Voice assistant denied non-MCP tool request: ${event.request.name}`
);
}
}
}
if (!sawSpeakToolCall && assistantTextChunks.length > 0) {
const fallbackText = assistantTextChunks.join(" ").trim();
await this.ttsManager.generateAndWaitForPlayback(
fallbackText,
(msg) => this.emit(msg),
this.abortController.signal,
true
);
this.emit({
type: "activity_log",
payload: {
id: uuidv4(),
timestamp: new Date(),
type: "assistant",
content: fallbackText,
},
});
this.sessionLogger.warn(
{ voiceAssistantAgentId: agentId },
"Voice agent responded without speak tool; used fallback TTS from assistant text"
);
}
}
/**
* Process user message through LLM with streaming and tool execution
*/
private async processWithLLM(enableTTS: boolean): Promise<void> {
private async processWithLLM(enableTTS: boolean, latestUserText?: string): Promise<void> {
if (enableTTS && this.voiceLlmProvider !== "openrouter") {
const text =
typeof latestUserText === "string" && latestUserText.trim().length > 0
? latestUserText
: (() => {
const lastUser = [...this.messages]
.reverse()
.find((message) => message.role === "user");
if (!lastUser) {
return "";
}
return typeof lastUser.content === "string"
? lastUser.content
: JSON.stringify(lastUser.content);
})();
const normalized = text.trim();
if (!normalized) {
return;
}
await this.processWithVoiceAgent(normalized);
return;
}
let assistantResponse = "";
let pendingTTS: Promise<void> | null = null;
let textBuffer = "";
@@ -5180,6 +5483,21 @@ export class Session {
this.agentTools = null;
}
if (this.voiceAssistantAgentId) {
try {
await this.agentManager.closeAgent(this.voiceAssistantAgentId);
} catch (error) {
this.sessionLogger.warn(
{ err: error, voiceAssistantAgentId: this.voiceAssistantAgentId },
"Failed to close voice assistant agent"
);
} finally {
this.unregisterVoiceSpeakHandler?.(this.voiceAssistantAgentId);
this.unregisterVoiceCallerContext?.(this.voiceAssistantAgentId);
this.voiceAssistantAgentId = null;
}
}
// Unsubscribe from all terminals
for (const unsubscribe of this.terminalSubscriptions.values()) {
unsubscribe();

View File

@@ -1,4 +1,6 @@
import { createRequire } from "node:module";
import os from "node:os";
import path from "node:path";
export type SherpaOnnxModule = {
createOnlineRecognizer: (config: any) => any;
@@ -8,11 +10,39 @@ export type SherpaOnnxModule = {
let cached: SherpaOnnxModule | null = null;
function ensureSherpaNativeLibraryPath(requireFn: NodeRequire): void {
const platform = os.platform();
if (platform !== "darwin" && platform !== "linux") {
return;
}
const platformArch = `${platform}-${os.arch()}`;
const packageName = `sherpa-onnx-${platformArch}`;
let nativeDir: string;
try {
const binaryPath = requireFn.resolve(`${packageName}/sherpa-onnx.node`);
nativeDir = path.dirname(binaryPath);
} catch {
return;
}
const envKey = platform === "darwin" ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH";
const current = process.env[envKey]?.trim() ?? "";
const entries = current.length > 0 ? current.split(":").filter(Boolean) : [];
if (entries.includes(nativeDir)) {
return;
}
process.env[envKey] = entries.length > 0 ? `${nativeDir}:${entries.join(":")}` : nativeDir;
}
export function loadSherpaOnnx(): SherpaOnnxModule {
if (cached) {
return cached;
}
const require = createRequire(import.meta.url);
ensureSherpaNativeLibraryPath(require);
cached = require("sherpa-onnx") as SherpaOnnxModule;
return cached;
}

View File

@@ -21,6 +21,10 @@ type TestPaseoDaemonOptions = {
cleanup?: boolean;
openai?: PaseoOpenAIConfig;
speech?: PaseoSpeechConfig;
openrouterApiKey?: string | null;
voiceLlmProvider?: PaseoDaemonConfig["voiceLlmProvider"];
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
dictationFinalTimeoutMs?: number;
};
@@ -78,7 +82,10 @@ export async function createTestPaseoDaemon(
appBaseUrl: "https://app.paseo.sh",
openai: options.openai,
speech: options.speech,
openrouterApiKey: null,
openrouterApiKey: options.openrouterApiKey ?? null,
voiceLlmProvider: options.voiceLlmProvider ?? null,
voiceLlmProviderExplicit: options.voiceLlmProviderExplicit ?? false,
voiceLlmModel: options.voiceLlmModel ?? null,
dictationFinalTimeoutMs: options.dictationFinalTimeoutMs,
downloadTokenTtlMs: options.downloadTokenTtlMs,
};

View File

@@ -0,0 +1,124 @@
import { afterAll, beforeAll, describe, expect, test } from "vitest";
import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js";
const openaiApiKey = process.env.OPENAI_API_KEY ?? null;
const shouldRun =
process.env.PASEO_VOICE_LOCAL_AGENT_E2E === "1" &&
Boolean(openaiApiKey) &&
!process.env.CI;
function waitForSignal<T>(
timeoutMs: number,
setup: (
resolve: (value: T) => void,
reject: (error: Error) => void
) => () => void
): Promise<T> {
return new Promise((resolve, reject) => {
let cleanup: (() => void) | null = null;
const timeout = setTimeout(() => {
cleanup?.();
reject(new Error(`Timeout waiting for event after ${timeoutMs}ms`));
}, timeoutMs);
cleanup = setup(
(value) => {
clearTimeout(timeout);
cleanup?.();
resolve(value);
},
(error) => {
clearTimeout(timeout);
cleanup?.();
reject(error);
}
);
});
}
(shouldRun ? describe : describe.skip)(
"voice local-agent e2e",
() => {
let ctx: DaemonTestContext;
beforeAll(async () => {
ctx = await createDaemonTestContext({
agentClients: {},
openai: { apiKey: openaiApiKey! },
speech: {
dictationSttProvider: "openai",
voiceSttProvider: "openai",
voiceTtsProvider: "openai",
},
voiceLlmProvider: "codex",
voiceLlmProviderExplicit: true,
voiceLlmModel: "gpt-5.2-mini",
});
}, 120000);
afterAll(async () => {
await ctx.cleanup();
}, 60000);
test(
"routes voice turns through local agent speak tool",
async () => {
await ctx.client.setVoiceConversation(true, `voice-local-agent-${Date.now()}`);
const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => {
const offAudio = ctx.client.on("audio_output", (msg) => {
if (msg.type !== "audio_output") return;
resolve({ chunkId: msg.payload.id });
});
const offError = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "error") return;
reject(new Error(String(msg.payload.content)));
});
return () => {
offAudio();
offError();
};
});
const assistantLogPromise = waitForSignal<string>(120000, (resolve, reject) => {
const offLog = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "assistant") return;
const content = String(msg.payload.content ?? "");
if (!content.trim()) return;
resolve(content);
});
const offError = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "error") return;
reject(new Error(String(msg.payload.content)));
});
return () => {
offLog();
offError();
};
});
ctx.client.sendUserMessage(
"Use the speak tool and say exactly: local voice agent path is working."
);
const [{ chunkId }, assistantText] = await Promise.all([
audioPromise,
assistantLogPromise,
]);
expect(chunkId.length).toBeGreaterThan(0);
expect(assistantText.toLowerCase()).toContain("local voice agent path is working");
const agents = await ctx.client.fetchAgents();
expect(
agents.some((agent) => String(agent.labels?.surface ?? "") === "voice")
).toBe(false);
},
180000
);
}
);

View File

@@ -23,6 +23,7 @@ import { VoiceConversationStore } from "./voice-conversation-store.js";
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
export type AgentMcpTransportFactory = () => Promise<Transport>;
type VoiceAgentProvider = "claude" | "codex" | "opencode";
type WebSocketServerConfig = {
allowedOrigins: Set<string>;
@@ -76,8 +77,26 @@ export class VoiceAssistantWebSocketServer {
} | null;
private readonly voice: {
openrouterApiKey?: string | null;
voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmDefaultProvider?: VoiceAgentProvider | null;
voiceLlmModel?: string | null;
voiceLlmAvailability?: Record<VoiceAgentProvider, boolean> | null;
voiceAgentMcpUrl?: string | null;
} | null;
private readonly voiceSpeakHandlers = new Map<
string,
(params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>
>();
private readonly voiceCallerContexts = new Map<
string,
{
childAgentDefaultLabels?: Record<string, string>;
lockedCwd?: string;
allowCustomCwd?: boolean;
enableVoiceTools?: boolean;
}
>();
constructor(
server: HTTPServer,
@@ -93,7 +112,12 @@ export class VoiceAssistantWebSocketServer {
terminalManager?: TerminalManager | null,
voice?: {
openrouterApiKey?: string | null;
voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmDefaultProvider?: VoiceAgentProvider | null;
voiceLlmModel?: string | null;
voiceLlmAvailability?: Record<VoiceAgentProvider, boolean> | null;
voiceAgentMcpUrl?: string | null;
},
dictation?: {
finalTimeoutMs?: number;
@@ -225,6 +249,20 @@ export class VoiceAssistantWebSocketServer {
this.terminalManager,
this.voiceConversationStore,
this.voice ?? undefined,
{
registerVoiceSpeakHandler: (agentId, handler) => {
this.voiceSpeakHandlers.set(agentId, handler);
},
unregisterVoiceSpeakHandler: (agentId) => {
this.voiceSpeakHandlers.delete(agentId);
},
registerVoiceCallerContext: (agentId, context) => {
this.voiceCallerContexts.set(agentId, context);
},
unregisterVoiceCallerContext: (agentId) => {
this.voiceCallerContexts.delete(agentId);
},
},
this.dictation ?? undefined
);
@@ -263,6 +301,23 @@ export class VoiceAssistantWebSocketServer {
});
}
public resolveVoiceSpeakHandler(
callerAgentId: string
): ((params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>) | null {
return this.voiceSpeakHandlers.get(callerAgentId) ?? null;
}
public resolveVoiceCallerContext(
callerAgentId: string
): {
childAgentDefaultLabels?: Record<string, string>;
lockedCwd?: string;
allowCustomCwd?: boolean;
enableVoiceTools?: boolean;
} | null {
return this.voiceCallerContexts.get(callerAgentId) ?? null;
}
private async detachSocket(
ws: WebSocketLike,
connectionLogger: pino.Logger,

View File

@@ -1,7 +1,8 @@
import path from "node:path";
import dotenv from "dotenv";
// Load repo-root .env for integration/E2E tests (OpenAI, etc.)
// Load package-local .env.test first for integration/E2E credentials, then repo-root .env fallback.
dotenv.config({ path: path.resolve(process.cwd(), ".env.test"), override: true });
dotenv.config({ path: path.resolve(process.cwd(), "../.env") });
process.env.GIT_TERMINAL_PROMPT = "0";