mirror of
https://github.com/getpaseo/paseo.git
synced 2026-07-29 12:01:31 +00:00
Voice MCP policy: required prompt, cwd override, and restricted tool path
This commit is contained in:
@@ -58,6 +58,7 @@ describe("create_agent MCP tool", () => {
|
||||
const missingTitle = await tool.inputSchema.safeParseAsync({
|
||||
cwd: "/tmp/repo",
|
||||
initialMode: "default",
|
||||
initialPrompt: "test",
|
||||
});
|
||||
expect(missingTitle.success).toBe(false);
|
||||
expect(missingTitle.error.issues[0].path).toEqual(["title"]);
|
||||
@@ -66,6 +67,7 @@ describe("create_agent MCP tool", () => {
|
||||
cwd: "/tmp/repo",
|
||||
initialMode: "default",
|
||||
title: "x".repeat(61),
|
||||
initialPrompt: "test",
|
||||
});
|
||||
expect(tooLong.success).toBe(false);
|
||||
expect(tooLong.error.issues[0].path).toEqual(["title"]);
|
||||
@@ -74,10 +76,24 @@ describe("create_agent MCP tool", () => {
|
||||
cwd: "/tmp/repo",
|
||||
initialMode: "default",
|
||||
title: "Short title",
|
||||
initialPrompt: "test",
|
||||
});
|
||||
expect(ok.success).toBe(true);
|
||||
});
|
||||
|
||||
it("requires initialPrompt", async () => {
|
||||
const { agentManager, agentStorage } = createTestDeps();
|
||||
const server = await createAgentMcpServer({ agentManager, agentStorage, logger });
|
||||
const tool = (server as any)._registeredTools["create_agent"];
|
||||
const parsed = await tool.inputSchema.safeParseAsync({
|
||||
cwd: "/tmp/repo",
|
||||
initialMode: "default",
|
||||
title: "Short title",
|
||||
});
|
||||
expect(parsed.success).toBe(false);
|
||||
expect(parsed.error.issues.some((issue: { path: string[] }) => issue.path[0] === "initialPrompt")).toBe(true);
|
||||
});
|
||||
|
||||
it("passes caller-provided titles directly into createAgent", async () => {
|
||||
const { agentManager, agentStorage, spies } = createTestDeps();
|
||||
spies.agentManager.createAgent.mockResolvedValue({
|
||||
@@ -86,6 +102,7 @@ describe("create_agent MCP tool", () => {
|
||||
lifecycle: "idle",
|
||||
currentModeId: null,
|
||||
availableModes: [],
|
||||
config: { title: "Fix auth bug" },
|
||||
} as ManagedAgent);
|
||||
|
||||
const server = await createAgentMcpServer({ agentManager, agentStorage, logger });
|
||||
@@ -93,13 +110,16 @@ describe("create_agent MCP tool", () => {
|
||||
await tool.callback({
|
||||
cwd: "/tmp/repo",
|
||||
title: " Fix auth bug ",
|
||||
initialPrompt: "Do work",
|
||||
});
|
||||
|
||||
expect(spies.agentManager.createAgent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
cwd: "/tmp/repo",
|
||||
title: "Fix auth bug",
|
||||
})
|
||||
}),
|
||||
undefined,
|
||||
undefined
|
||||
);
|
||||
});
|
||||
|
||||
@@ -111,6 +131,7 @@ describe("create_agent MCP tool", () => {
|
||||
lifecycle: "idle",
|
||||
currentModeId: null,
|
||||
availableModes: [],
|
||||
config: { title: "Fix auth" },
|
||||
} as ManagedAgent);
|
||||
|
||||
const server = await createAgentMcpServer({ agentManager, agentStorage, logger });
|
||||
@@ -118,12 +139,115 @@ describe("create_agent MCP tool", () => {
|
||||
await tool.callback({
|
||||
cwd: "/tmp/repo",
|
||||
title: " Fix auth ",
|
||||
initialPrompt: "Do work",
|
||||
});
|
||||
|
||||
expect(spies.agentManager.createAgent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
title: "Fix auth",
|
||||
})
|
||||
}),
|
||||
undefined,
|
||||
undefined
|
||||
);
|
||||
});
|
||||
|
||||
it("allows caller agents to override cwd and applies caller context labels", async () => {
|
||||
const { agentManager, agentStorage, spies } = createTestDeps();
|
||||
spies.agentManager.getAgent.mockReturnValue({
|
||||
id: "voice-agent",
|
||||
cwd: "/tmp/voice",
|
||||
provider: "codex",
|
||||
currentModeId: "full-access",
|
||||
} as ManagedAgent);
|
||||
spies.agentManager.createAgent.mockResolvedValue({
|
||||
id: "child-agent",
|
||||
cwd: "/tmp/voice/subdir",
|
||||
lifecycle: "idle",
|
||||
currentModeId: null,
|
||||
availableModes: [],
|
||||
config: { title: "Child" },
|
||||
} as ManagedAgent);
|
||||
|
||||
const server = await createAgentMcpServer({
|
||||
agentManager,
|
||||
agentStorage,
|
||||
callerAgentId: "voice-agent",
|
||||
resolveCallerContext: () => ({
|
||||
childAgentDefaultLabels: { ui: "true" },
|
||||
allowCustomCwd: true,
|
||||
}),
|
||||
logger,
|
||||
});
|
||||
|
||||
const tool = (server as any)._registeredTools["create_agent"];
|
||||
await tool.callback({
|
||||
cwd: "subdir",
|
||||
title: "Child",
|
||||
agentType: "codex",
|
||||
initialPrompt: "Do work",
|
||||
});
|
||||
|
||||
expect(spies.agentManager.createAgent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
cwd: "/tmp/voice/subdir",
|
||||
}),
|
||||
undefined,
|
||||
{ labels: { ui: "true" } }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("speak MCP tool", () => {
|
||||
const logger = createTestLogger();
|
||||
|
||||
it("invokes registered speak handler for caller agent", async () => {
|
||||
const { agentManager, agentStorage } = createTestDeps();
|
||||
const speak = vi.fn().mockResolvedValue(undefined);
|
||||
const server = await createAgentMcpServer({
|
||||
agentManager,
|
||||
agentStorage,
|
||||
callerAgentId: "voice-agent-1",
|
||||
enableVoiceTools: true,
|
||||
resolveSpeakHandler: () => speak,
|
||||
logger,
|
||||
});
|
||||
const tool = (server as any)._registeredTools["speak"];
|
||||
expect(tool).toBeDefined();
|
||||
|
||||
await tool.callback({ text: "Hello from voice agent." });
|
||||
expect(speak).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
text: "Hello from voice agent.",
|
||||
callerAgentId: "voice-agent-1",
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("fails when no speak handler exists", async () => {
|
||||
const { agentManager, agentStorage } = createTestDeps();
|
||||
const server = await createAgentMcpServer({
|
||||
agentManager,
|
||||
agentStorage,
|
||||
callerAgentId: "voice-agent-2",
|
||||
enableVoiceTools: true,
|
||||
resolveSpeakHandler: () => null,
|
||||
logger,
|
||||
});
|
||||
const tool = (server as any)._registeredTools["speak"];
|
||||
await expect(tool.callback({ text: "Hello." })).rejects.toThrow(
|
||||
"No speak handler registered for caller agent"
|
||||
);
|
||||
});
|
||||
|
||||
it("does not register speak tool unless voice tools are enabled", async () => {
|
||||
const { agentManager, agentStorage } = createTestDeps();
|
||||
const server = await createAgentMcpServer({
|
||||
agentManager,
|
||||
agentStorage,
|
||||
callerAgentId: "agent-no-voice",
|
||||
logger,
|
||||
});
|
||||
const tool = (server as any)._registeredTools["speak"];
|
||||
expect(tool).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -38,6 +38,22 @@ export interface AgentMcpServerOptions {
|
||||
* Used for cwd/mode inheritance when agents spawn child agents.
|
||||
*/
|
||||
callerAgentId?: string;
|
||||
/**
|
||||
* Optional resolver for session-bound speak handlers.
|
||||
* Used by hidden voice agents to narrate through daemon-managed TTS.
|
||||
*/
|
||||
resolveSpeakHandler?: (
|
||||
callerAgentId: string
|
||||
) => ((params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>) | null;
|
||||
resolveCallerContext?: (
|
||||
callerAgentId: string
|
||||
) => {
|
||||
childAgentDefaultLabels?: Record<string, string>;
|
||||
lockedCwd?: string;
|
||||
allowCustomCwd?: boolean;
|
||||
enableVoiceTools?: boolean;
|
||||
} | null;
|
||||
enableVoiceTools?: boolean;
|
||||
logger: Logger;
|
||||
}
|
||||
|
||||
@@ -252,9 +268,17 @@ async function serializeSnapshotWithMetadata(
|
||||
export async function createAgentMcpServer(
|
||||
options: AgentMcpServerOptions
|
||||
): Promise<McpServer> {
|
||||
const { agentManager, agentStorage, callerAgentId, logger } = options;
|
||||
const {
|
||||
agentManager,
|
||||
agentStorage,
|
||||
callerAgentId,
|
||||
resolveSpeakHandler,
|
||||
resolveCallerContext,
|
||||
logger,
|
||||
} = options;
|
||||
const childLogger = logger.child({ module: "agent", component: "mcp-server" });
|
||||
const waitTracker = new WaitForAgentTracker(logger);
|
||||
const callerContext = callerAgentId ? resolveCallerContext?.(callerAgentId) ?? null : null;
|
||||
|
||||
const server = new McpServer({
|
||||
name: "agent-mcp",
|
||||
@@ -262,6 +286,12 @@ export async function createAgentMcpServer(
|
||||
});
|
||||
|
||||
const agentToAgentInputSchema = {
|
||||
cwd: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe(
|
||||
"Optional working directory. Defaults to the caller agent working directory."
|
||||
),
|
||||
title: z
|
||||
.string()
|
||||
.trim()
|
||||
@@ -275,9 +305,10 @@ export async function createAgentMcpServer(
|
||||
),
|
||||
initialPrompt: z
|
||||
.string()
|
||||
.optional()
|
||||
.trim()
|
||||
.min(1, "initialPrompt is required")
|
||||
.describe(
|
||||
"Optional task to start immediately after creation (non-blocking)."
|
||||
"Required first task to run immediately after creation."
|
||||
),
|
||||
background: z
|
||||
.boolean()
|
||||
@@ -307,9 +338,10 @@ export async function createAgentMcpServer(
|
||||
),
|
||||
initialPrompt: z
|
||||
.string()
|
||||
.optional()
|
||||
.trim()
|
||||
.min(1, "initialPrompt is required")
|
||||
.describe(
|
||||
"Optional task to start immediately after creation (non-blocking)."
|
||||
"Required first task to run immediately after creation."
|
||||
),
|
||||
initialMode: z
|
||||
.string()
|
||||
@@ -339,6 +371,45 @@ export async function createAgentMcpServer(
|
||||
? agentToAgentInputSchema
|
||||
: topLevelInputSchema;
|
||||
|
||||
if (options.enableVoiceTools || callerContext?.enableVoiceTools) {
|
||||
server.registerTool(
|
||||
"speak",
|
||||
{
|
||||
title: "Speak",
|
||||
description:
|
||||
"Speak text to the user via daemon-managed voice output. Blocks until playback completes.",
|
||||
inputSchema: {
|
||||
text: z
|
||||
.string()
|
||||
.trim()
|
||||
.min(1, "text is required")
|
||||
.max(4000, "text must be 4000 characters or fewer"),
|
||||
},
|
||||
outputSchema: {
|
||||
ok: z.boolean(),
|
||||
},
|
||||
},
|
||||
async (args, context) => {
|
||||
if (!callerAgentId) {
|
||||
throw new Error("speak is only available to agent-scoped MCP sessions");
|
||||
}
|
||||
const handler = resolveSpeakHandler?.(callerAgentId) ?? null;
|
||||
if (!handler) {
|
||||
throw new Error(`No speak handler registered for caller agent '${callerAgentId}'`);
|
||||
}
|
||||
await handler({
|
||||
text: args.text,
|
||||
callerAgentId,
|
||||
signal: (context as { signal?: AbortSignal } | undefined)?.signal,
|
||||
});
|
||||
return {
|
||||
content: [],
|
||||
structuredContent: ensureValidJson({ ok: true }),
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
server.registerTool(
|
||||
"create_agent",
|
||||
{
|
||||
@@ -363,7 +434,7 @@ export async function createAgentMcpServer(
|
||||
permission: AgentPermissionRequestPayloadSchema.nullable().optional(),
|
||||
},
|
||||
},
|
||||
async (args) => {
|
||||
async (args: unknown) => {
|
||||
const {
|
||||
agentType,
|
||||
initialPrompt,
|
||||
@@ -372,7 +443,7 @@ export async function createAgentMcpServer(
|
||||
} = args as {
|
||||
cwd?: string;
|
||||
agentType?: AgentProvider;
|
||||
initialPrompt?: string;
|
||||
initialPrompt: string;
|
||||
initialMode?: string;
|
||||
worktreeName?: string;
|
||||
background?: boolean;
|
||||
@@ -387,7 +458,19 @@ export async function createAgentMcpServer(
|
||||
if (!parentAgent) {
|
||||
throw new Error(`Parent agent ${callerAgentId} not found`);
|
||||
}
|
||||
resolvedCwd = parentAgent.cwd;
|
||||
const callerArgs = args as unknown as { cwd?: string };
|
||||
const requestedCwd = callerArgs.cwd?.trim();
|
||||
const lockedCwd = callerContext?.lockedCwd?.trim();
|
||||
if (lockedCwd) {
|
||||
resolvedCwd = expandPath(lockedCwd);
|
||||
} else if (requestedCwd && (callerContext?.allowCustomCwd ?? true)) {
|
||||
resolvedCwd =
|
||||
requestedCwd.startsWith("/") || requestedCwd.startsWith("~")
|
||||
? expandPath(requestedCwd)
|
||||
: resolve(parentAgent.cwd, requestedCwd);
|
||||
} else {
|
||||
resolvedCwd = parentAgent.cwd;
|
||||
}
|
||||
|
||||
const provider: AgentProvider = agentType ?? "claude";
|
||||
const parentMode = parentAgent.currentModeId;
|
||||
@@ -433,73 +516,78 @@ export async function createAgentMcpServer(
|
||||
|
||||
const provider: AgentProvider = agentType ?? "claude";
|
||||
const normalizedTitle = title?.trim() ?? null;
|
||||
const snapshot = await agentManager.createAgent({
|
||||
provider,
|
||||
cwd: resolvedCwd,
|
||||
modeId: resolvedMode,
|
||||
title: normalizedTitle ?? undefined,
|
||||
const childAgentDefaultLabels =
|
||||
callerAgentId && callerContext?.childAgentDefaultLabels
|
||||
? callerContext.childAgentDefaultLabels
|
||||
: undefined;
|
||||
const snapshot = await agentManager.createAgent(
|
||||
{
|
||||
provider,
|
||||
cwd: resolvedCwd,
|
||||
modeId: resolvedMode,
|
||||
title: normalizedTitle ?? undefined,
|
||||
},
|
||||
undefined,
|
||||
childAgentDefaultLabels ? { labels: childAgentDefaultLabels } : undefined
|
||||
);
|
||||
|
||||
const trimmedPrompt = initialPrompt.trim();
|
||||
scheduleAgentMetadataGeneration({
|
||||
agentManager,
|
||||
agentId: snapshot.id,
|
||||
cwd: snapshot.cwd,
|
||||
initialPrompt: trimmedPrompt,
|
||||
explicitTitle: snapshot.config.title,
|
||||
paseoHome: options.paseoHome,
|
||||
logger: childLogger,
|
||||
});
|
||||
|
||||
const trimmedPrompt = initialPrompt?.trim();
|
||||
if (trimmedPrompt) {
|
||||
scheduleAgentMetadataGeneration({
|
||||
agentManager,
|
||||
agentId: snapshot.id,
|
||||
cwd: snapshot.cwd,
|
||||
initialPrompt: trimmedPrompt,
|
||||
explicitTitle: snapshot.config.title,
|
||||
paseoHome: options.paseoHome,
|
||||
logger: childLogger,
|
||||
});
|
||||
|
||||
try {
|
||||
agentManager.recordUserMessage(snapshot.id, trimmedPrompt);
|
||||
} catch (error) {
|
||||
childLogger.error(
|
||||
{ err: error, agentId: snapshot.id },
|
||||
"Failed to record initial prompt"
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
startAgentRun(agentManager, snapshot.id, trimmedPrompt, childLogger);
|
||||
|
||||
// If not running in background, wait for completion
|
||||
if (!background) {
|
||||
const result = await waitForAgentWithTimeout(
|
||||
agentManager,
|
||||
snapshot.id,
|
||||
{ waitForActive: true }
|
||||
);
|
||||
|
||||
const responseData = {
|
||||
agentId: snapshot.id,
|
||||
type: provider,
|
||||
status: result.status,
|
||||
cwd: snapshot.cwd,
|
||||
currentModeId: snapshot.currentModeId,
|
||||
availableModes: snapshot.availableModes,
|
||||
lastMessage: result.lastMessage,
|
||||
permission: sanitizePermissionRequest(result.permission),
|
||||
};
|
||||
const validJson = ensureValidJson(responseData);
|
||||
|
||||
const response = {
|
||||
content: [],
|
||||
structuredContent: validJson,
|
||||
};
|
||||
return response;
|
||||
}
|
||||
} catch (error) {
|
||||
childLogger.error(
|
||||
{ err: error, agentId: snapshot.id },
|
||||
"Failed to run initial prompt"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
agentManager.recordUserMessage(snapshot.id, trimmedPrompt);
|
||||
} catch (error) {
|
||||
childLogger.error(
|
||||
{ err: error, agentId: snapshot.id },
|
||||
"Failed to record initial prompt"
|
||||
);
|
||||
}
|
||||
|
||||
// Return immediately if background=true or no initialPrompt
|
||||
try {
|
||||
startAgentRun(agentManager, snapshot.id, trimmedPrompt, childLogger);
|
||||
|
||||
// If not running in background, wait for completion
|
||||
if (!background) {
|
||||
const result = await waitForAgentWithTimeout(
|
||||
agentManager,
|
||||
snapshot.id,
|
||||
{ waitForActive: true }
|
||||
);
|
||||
|
||||
const responseData = {
|
||||
agentId: snapshot.id,
|
||||
type: provider,
|
||||
status: result.status,
|
||||
cwd: snapshot.cwd,
|
||||
currentModeId: snapshot.currentModeId,
|
||||
availableModes: snapshot.availableModes,
|
||||
lastMessage: result.lastMessage,
|
||||
permission: sanitizePermissionRequest(result.permission),
|
||||
};
|
||||
const validJson = ensureValidJson(responseData);
|
||||
|
||||
const response = {
|
||||
content: [],
|
||||
structuredContent: validJson,
|
||||
};
|
||||
return response;
|
||||
}
|
||||
} catch (error) {
|
||||
childLogger.error(
|
||||
{ err: error, agentId: snapshot.id },
|
||||
"Failed to run initial prompt"
|
||||
);
|
||||
}
|
||||
|
||||
// Return immediately if background=true
|
||||
const response = {
|
||||
content: [],
|
||||
structuredContent: ensureValidJson({
|
||||
|
||||
@@ -9,8 +9,15 @@ import { createTestPaseoDaemon } from "./test-utils/paseo-daemon.js";
|
||||
import { createTestAgentClients } from "./test-utils/fake-agent-client.js";
|
||||
|
||||
describe("paseo daemon bootstrap", () => {
|
||||
test("starts and serves health endpoint", async () => {
|
||||
const daemonHandle = await createTestPaseoDaemon();
|
||||
test.runIf(Boolean(process.env.OPENAI_API_KEY))("starts and serves health endpoint", async () => {
|
||||
const daemonHandle = await createTestPaseoDaemon({
|
||||
openai: { apiKey: process.env.OPENAI_API_KEY! },
|
||||
speech: {
|
||||
dictationSttProvider: "openai",
|
||||
voiceSttProvider: "openai",
|
||||
voiceTtsProvider: "openai",
|
||||
},
|
||||
});
|
||||
try {
|
||||
const response = await fetch(
|
||||
`http://127.0.0.1:${daemonHandle.port}/api/health`,
|
||||
|
||||
@@ -77,6 +77,8 @@ import { acquirePidLock, releasePidLock } from "./pid-lock.js";
|
||||
import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js";
|
||||
|
||||
type AgentMcpTransportMap = Map<string, StreamableHTTPServerTransport>;
|
||||
type VoiceAgentProvider = "claude" | "codex" | "opencode";
|
||||
const VOICE_AGENT_FALLBACK_ORDER: VoiceAgentProvider[] = ["claude", "codex", "opencode"];
|
||||
|
||||
export type PaseoOpenAIConfig = {
|
||||
apiKey?: string;
|
||||
@@ -121,6 +123,8 @@ export type PaseoDaemonConfig = {
|
||||
openai?: PaseoOpenAIConfig;
|
||||
speech?: PaseoSpeechConfig;
|
||||
openrouterApiKey?: string | null;
|
||||
voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null;
|
||||
voiceLlmProviderExplicit?: boolean;
|
||||
voiceLlmModel?: string | null;
|
||||
dictationFinalTimeoutMs?: number;
|
||||
downloadTokenTtlMs?: number;
|
||||
@@ -282,12 +286,88 @@ export async function createPaseoDaemon(
|
||||
`Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand`
|
||||
);
|
||||
|
||||
const requestedVoiceLlmProvider = config.voiceLlmProvider ?? null;
|
||||
const voiceLlmProviderExplicit = config.voiceLlmProviderExplicit ?? false;
|
||||
logger.info(
|
||||
{
|
||||
requestedVoiceLlmProvider,
|
||||
voiceLlmProviderExplicit,
|
||||
},
|
||||
"Voice LLM provider reconciliation started"
|
||||
);
|
||||
|
||||
const providerClients = createAllClients(logger);
|
||||
const voiceLlmAvailability: Record<VoiceAgentProvider, boolean> = {
|
||||
claude: false,
|
||||
codex: false,
|
||||
opencode: false,
|
||||
};
|
||||
for (const provider of VOICE_AGENT_FALLBACK_ORDER) {
|
||||
try {
|
||||
voiceLlmAvailability[provider] = await providerClients[provider].isAvailable();
|
||||
} catch (error) {
|
||||
logger.warn({ err: error, provider }, "Voice LLM provider availability check failed");
|
||||
voiceLlmAvailability[provider] = false;
|
||||
}
|
||||
}
|
||||
|
||||
const voiceLlmDefaultProvider =
|
||||
VOICE_AGENT_FALLBACK_ORDER.find((provider) => voiceLlmAvailability[provider]) ?? null;
|
||||
|
||||
if (requestedVoiceLlmProvider === "openrouter") {
|
||||
const openrouterApiKey =
|
||||
config.openrouterApiKey ?? process.env.OPENROUTER_API_KEY ?? null;
|
||||
if (!openrouterApiKey) {
|
||||
logger.error("voiceMode.llm.provider is openrouter but no OpenRouter API key is configured");
|
||||
throw new Error("Missing OpenRouter API key for voiceMode.llm.provider=openrouter");
|
||||
}
|
||||
} else if (
|
||||
requestedVoiceLlmProvider === "claude" ||
|
||||
requestedVoiceLlmProvider === "codex" ||
|
||||
requestedVoiceLlmProvider === "opencode"
|
||||
) {
|
||||
if (!voiceLlmAvailability[requestedVoiceLlmProvider]) {
|
||||
logger.error(
|
||||
{ provider: requestedVoiceLlmProvider, voiceLlmAvailability },
|
||||
"Configured voice LLM provider is unavailable"
|
||||
);
|
||||
throw new Error(`Configured voice LLM provider '${requestedVoiceLlmProvider}' is unavailable`);
|
||||
}
|
||||
} else if (!voiceLlmDefaultProvider) {
|
||||
logger.error(
|
||||
{ requestedVoiceLlmProvider, voiceLlmAvailability },
|
||||
"No local voice LLM provider available for fallback"
|
||||
);
|
||||
throw new Error("No local voice LLM provider available (claude/codex/opencode)");
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{
|
||||
requestedVoiceLlmProvider,
|
||||
voiceLlmProviderExplicit,
|
||||
voiceLlmAvailability,
|
||||
voiceLlmDefaultProvider,
|
||||
},
|
||||
"Voice LLM provider reconciliation completed"
|
||||
);
|
||||
if (listenTarget.type !== "tcp" && requestedVoiceLlmProvider !== "openrouter") {
|
||||
logger.error(
|
||||
{ listen: config.listen, requestedVoiceLlmProvider },
|
||||
"Local voice agent mode requires TCP listen target for HTTP MCP bridge"
|
||||
);
|
||||
throw new Error("Local voice agent mode requires TCP listen target");
|
||||
}
|
||||
let wsServer: VoiceAssistantWebSocketServer | null = null;
|
||||
|
||||
// Create in-memory transport for Session's Agent MCP client (voice assistant tools)
|
||||
const createInMemoryAgentMcpTransport = async (): Promise<InMemoryTransport> => {
|
||||
const agentMcpServer = await createAgentMcpServer({
|
||||
agentManager,
|
||||
agentStorage,
|
||||
paseoHome: config.paseoHome,
|
||||
enableVoiceTools: false,
|
||||
resolveSpeakHandler: (callerAgentId) => wsServer?.resolveVoiceSpeakHandler(callerAgentId) ?? null,
|
||||
resolveCallerContext: (callerAgentId) => wsServer?.resolveVoiceCallerContext(callerAgentId) ?? null,
|
||||
logger,
|
||||
});
|
||||
|
||||
@@ -309,6 +389,9 @@ export async function createPaseoDaemon(
|
||||
agentStorage,
|
||||
paseoHome: config.paseoHome,
|
||||
callerAgentId,
|
||||
enableVoiceTools: false,
|
||||
resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null,
|
||||
resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null,
|
||||
logger,
|
||||
});
|
||||
|
||||
@@ -792,7 +875,12 @@ export async function createPaseoDaemon(
|
||||
);
|
||||
}
|
||||
|
||||
const wsServer = new VoiceAssistantWebSocketServer(
|
||||
const voiceAgentMcpUrl =
|
||||
listenTarget.type === "tcp"
|
||||
? `http://127.0.0.1:${listenTarget.port}/mcp/agents`
|
||||
: null;
|
||||
|
||||
wsServer = new VoiceAssistantWebSocketServer(
|
||||
httpServer,
|
||||
logger,
|
||||
serverId,
|
||||
@@ -806,7 +894,12 @@ export async function createPaseoDaemon(
|
||||
terminalManager,
|
||||
{
|
||||
openrouterApiKey: config.openrouterApiKey ?? null,
|
||||
voiceLlmProvider: config.voiceLlmProvider ?? null,
|
||||
voiceLlmProviderExplicit,
|
||||
voiceLlmDefaultProvider,
|
||||
voiceLlmModel: config.voiceLlmModel ?? null,
|
||||
voiceLlmAvailability,
|
||||
voiceAgentMcpUrl,
|
||||
},
|
||||
{
|
||||
finalTimeoutMs: config.dictationFinalTimeoutMs,
|
||||
@@ -870,7 +963,12 @@ export async function createPaseoDaemon(
|
||||
relayTransport?.stop().catch(() => undefined);
|
||||
relayTransport = startRelayTransport({
|
||||
logger,
|
||||
attachSocket: (ws) => wsServer.attachExternalSocket(ws),
|
||||
attachSocket: (ws) => {
|
||||
if (!wsServer) {
|
||||
throw new Error("WebSocket server not initialized");
|
||||
}
|
||||
return wsServer.attachExternalSocket(ws);
|
||||
},
|
||||
relayEndpoint,
|
||||
serverId,
|
||||
daemonKeyPair: daemonKeyPair.keyPair,
|
||||
@@ -911,7 +1009,9 @@ export async function createPaseoDaemon(
|
||||
sherpaOnline?.free();
|
||||
sherpaOffline?.free();
|
||||
await relayTransport?.stop().catch(() => undefined);
|
||||
await wsServer.close();
|
||||
if (wsServer) {
|
||||
await wsServer.close();
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
httpServer.close(() => resolve());
|
||||
});
|
||||
|
||||
@@ -13,6 +13,14 @@ import {
|
||||
const DEFAULT_PORT = 6767;
|
||||
const DEFAULT_RELAY_ENDPOINT = "relay.paseo.sh:443";
|
||||
const DEFAULT_APP_BASE_URL = "https://app.paseo.sh";
|
||||
const VOICE_LLM_PROVIDER_IDS = [
|
||||
"openrouter",
|
||||
"local-agent",
|
||||
"claude",
|
||||
"codex",
|
||||
"opencode",
|
||||
] as const;
|
||||
type VoiceLlmProviderId = (typeof VOICE_LLM_PROVIDER_IDS)[number];
|
||||
|
||||
function getDefaultListen(): string {
|
||||
// Main HTTP server defaults to TCP
|
||||
@@ -89,6 +97,19 @@ function parseSpeechProviderId(value: unknown): "openai" | "local" | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseVoiceLlmProviderId(value: unknown): VoiceLlmProviderId | null {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
return (VOICE_LLM_PROVIDER_IDS as readonly string[]).includes(normalized)
|
||||
? (normalized as VoiceLlmProviderId)
|
||||
: null;
|
||||
}
|
||||
|
||||
function normalizeSherpaSttPreset(value: string): string {
|
||||
const raw = value.trim();
|
||||
const normalized = raw.toLowerCase();
|
||||
@@ -246,6 +267,13 @@ export function loadConfig(
|
||||
|
||||
const openrouterApiKey =
|
||||
env.OPENROUTER_API_KEY ?? persisted.providers?.openrouter?.apiKey ?? null;
|
||||
const envVoiceLlmProvider = parseVoiceLlmProviderId(env.PASEO_VOICE_LLM_PROVIDER);
|
||||
const persistedVoiceLlmProvider = parseVoiceLlmProviderId(
|
||||
persisted.features?.voiceMode?.llm?.provider
|
||||
);
|
||||
const voiceLlmProvider = envVoiceLlmProvider ?? persistedVoiceLlmProvider ?? null;
|
||||
const voiceLlmProviderExplicit =
|
||||
envVoiceLlmProvider !== null || persistedVoiceLlmProvider !== null;
|
||||
const voiceLlmModel = persisted.features?.voiceMode?.llm?.model ?? null;
|
||||
|
||||
return {
|
||||
@@ -272,6 +300,8 @@ export function loadConfig(
|
||||
...(sherpaOnnx ? { sherpaOnnx } : {}),
|
||||
},
|
||||
openrouterApiKey,
|
||||
voiceLlmProvider,
|
||||
voiceLlmProviderExplicit,
|
||||
voiceLlmModel,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -74,7 +74,9 @@ const FeatureVoiceModeSchema = z
|
||||
.object({
|
||||
llm: z
|
||||
.object({
|
||||
provider: z.enum(["openrouter"]).optional(),
|
||||
provider: z
|
||||
.enum(["openrouter", "local-agent", "claude", "codex", "opencode"])
|
||||
.optional(),
|
||||
model: z.string().min(1).optional(),
|
||||
})
|
||||
.strict()
|
||||
|
||||
@@ -49,6 +49,15 @@ import { experimental_createMCPClient } from "ai";
|
||||
import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
|
||||
|
||||
export type AgentMcpTransportFactory = () => Promise<Transport>;
|
||||
type VoiceLlmProvider = "openrouter" | "local-agent" | "claude" | "codex" | "opencode";
|
||||
type VoiceAgentProvider = Exclude<VoiceLlmProvider, "openrouter" | "local-agent">;
|
||||
type VoiceSpeakHandler = (params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>;
|
||||
type VoiceCallerContext = {
|
||||
childAgentDefaultLabels?: Record<string, string>;
|
||||
lockedCwd?: string;
|
||||
allowCustomCwd?: boolean;
|
||||
enableVoiceTools?: boolean;
|
||||
};
|
||||
import { buildProviderRegistry } from "./agent/provider-registry.js";
|
||||
import { AgentManager } from "./agent/agent-manager.js";
|
||||
import type { ManagedAgent } from "./agent/agent-manager.js";
|
||||
@@ -59,6 +68,7 @@ import {
|
||||
generateStructuredAgentResponse,
|
||||
} from "./agent/agent-response-loop.js";
|
||||
import type {
|
||||
AgentPermissionRequest,
|
||||
AgentPermissionResponse,
|
||||
AgentPromptContentBlock,
|
||||
AgentPromptInput,
|
||||
@@ -130,6 +140,24 @@ const RESTART_EXIT_DELAY_MS = 250;
|
||||
* Uses Claude Haiku for speed and cost efficiency.
|
||||
*/
|
||||
const AUTO_GEN_MODEL = "haiku";
|
||||
const VOICE_AGENT_FALLBACK_ORDER: VoiceAgentProvider[] = ["claude", "codex", "opencode"];
|
||||
const VOICE_AGENT_DEFAULT_MODE: Record<VoiceAgentProvider, string> = {
|
||||
claude: "default",
|
||||
codex: "read-only",
|
||||
opencode: "default",
|
||||
};
|
||||
const VOICE_AGENT_DEFAULT_MODEL: Partial<Record<VoiceAgentProvider, string>> = {
|
||||
claude: "haiku",
|
||||
codex: "gpt-5.2-mini",
|
||||
};
|
||||
const VOICE_AGENT_SYSTEM_INSTRUCTION = [
|
||||
"You are the Paseo voice assistant.",
|
||||
"The user cannot see your chat messages or tool calls.",
|
||||
"Always narrate everything through the speak tool.",
|
||||
"Use concise plain language suitable for speech output.",
|
||||
"Never use bash, file-edit, or web tools directly.",
|
||||
"Only use the paseo MCP tools.",
|
||||
].join(" ");
|
||||
|
||||
type ProcessingPhase = "idle" | "transcribing" | "llm";
|
||||
|
||||
@@ -320,7 +348,23 @@ export class Session {
|
||||
private readonly terminalManager: TerminalManager | null;
|
||||
private terminalSubscriptions: Map<string, () => void> = new Map();
|
||||
private readonly openrouterApiKey: string | null;
|
||||
private readonly voiceLlmProvider: VoiceLlmProvider | null;
|
||||
private readonly voiceLlmProviderExplicit: boolean;
|
||||
private readonly voiceLlmDefaultProvider: VoiceAgentProvider | null;
|
||||
private readonly voiceLlmModel: string | null;
|
||||
private readonly voiceLlmAvailability: Record<VoiceAgentProvider, boolean> | null;
|
||||
private readonly voiceAgentMcpUrl: string | null;
|
||||
private readonly registerVoiceSpeakHandler?: (
|
||||
agentId: string,
|
||||
handler: VoiceSpeakHandler
|
||||
) => void;
|
||||
private readonly unregisterVoiceSpeakHandler?: (agentId: string) => void;
|
||||
private readonly registerVoiceCallerContext?: (
|
||||
agentId: string,
|
||||
context: VoiceCallerContext
|
||||
) => void;
|
||||
private readonly unregisterVoiceCallerContext?: (agentId: string) => void;
|
||||
private voiceAssistantAgentId: string | null = null;
|
||||
|
||||
constructor(
|
||||
clientId: string,
|
||||
@@ -338,7 +382,18 @@ export class Session {
|
||||
voiceConversationStore: VoiceConversationStore,
|
||||
voice?: {
|
||||
openrouterApiKey?: string | null;
|
||||
voiceLlmProvider?: VoiceLlmProvider | null;
|
||||
voiceLlmProviderExplicit?: boolean;
|
||||
voiceLlmDefaultProvider?: VoiceAgentProvider | null;
|
||||
voiceLlmModel?: string | null;
|
||||
voiceLlmAvailability?: Record<VoiceAgentProvider, boolean> | null;
|
||||
voiceAgentMcpUrl?: string | null;
|
||||
},
|
||||
voiceBridge?: {
|
||||
registerVoiceSpeakHandler?: (agentId: string, handler: VoiceSpeakHandler) => void;
|
||||
unregisterVoiceSpeakHandler?: (agentId: string) => void;
|
||||
registerVoiceCallerContext?: (agentId: string, context: VoiceCallerContext) => void;
|
||||
unregisterVoiceCallerContext?: (agentId: string) => void;
|
||||
},
|
||||
dictation?: {
|
||||
finalTimeoutMs?: number;
|
||||
@@ -357,7 +412,16 @@ export class Session {
|
||||
this.terminalManager = terminalManager;
|
||||
this.voiceConversationStore = voiceConversationStore;
|
||||
this.openrouterApiKey = voice?.openrouterApiKey ?? null;
|
||||
this.voiceLlmProvider = voice?.voiceLlmProvider ?? null;
|
||||
this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false;
|
||||
this.voiceLlmDefaultProvider = voice?.voiceLlmDefaultProvider ?? null;
|
||||
this.voiceLlmModel = voice?.voiceLlmModel ?? null;
|
||||
this.voiceLlmAvailability = voice?.voiceLlmAvailability ?? null;
|
||||
this.voiceAgentMcpUrl = voice?.voiceAgentMcpUrl ?? null;
|
||||
this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler;
|
||||
this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler;
|
||||
this.registerVoiceCallerContext = voiceBridge?.registerVoiceCallerContext;
|
||||
this.unregisterVoiceCallerContext = voiceBridge?.unregisterVoiceCallerContext;
|
||||
this.abortController = new AbortController();
|
||||
this.sessionLogger = logger.child({
|
||||
module: "session",
|
||||
@@ -4241,7 +4305,7 @@ export class Session {
|
||||
this.messages.push({ role: "user", content: text });
|
||||
|
||||
// Process through LLM (TTS enabled in voice mode for voice conversations)
|
||||
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode);
|
||||
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, text);
|
||||
await this.currentStreamPromise;
|
||||
}
|
||||
|
||||
@@ -4518,7 +4582,7 @@ export class Session {
|
||||
// Set phase to LLM and process (TTS enabled in voice mode for voice conversations)
|
||||
this.clearSpeechInProgress("transcription complete");
|
||||
this.setPhase("llm");
|
||||
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode);
|
||||
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, result.text);
|
||||
await this.currentStreamPromise;
|
||||
this.setPhase("idle");
|
||||
} catch (error: any) {
|
||||
@@ -4537,10 +4601,249 @@ export class Session {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the effective voice LLM provider.
|
||||
* - explicit provider => strict
|
||||
* - local-agent / unset => fallback order
|
||||
*/
|
||||
private resolveVoiceAgentProvider(): VoiceAgentProvider {
|
||||
const configured = this.voiceLlmProvider;
|
||||
const availability = this.voiceLlmAvailability ?? {
|
||||
claude: true,
|
||||
codex: true,
|
||||
opencode: true,
|
||||
};
|
||||
|
||||
if (configured === "openrouter") {
|
||||
throw new Error("voiceLlmProvider=openrouter cannot be used in local-agent flow");
|
||||
}
|
||||
|
||||
if (configured === "claude" || configured === "codex" || configured === "opencode") {
|
||||
if (!availability[configured]) {
|
||||
throw new Error(`Configured voice LLM provider '${configured}' is unavailable`);
|
||||
}
|
||||
return configured;
|
||||
}
|
||||
|
||||
const fallbackOrder =
|
||||
this.voiceLlmDefaultProvider && availability[this.voiceLlmDefaultProvider]
|
||||
? [this.voiceLlmDefaultProvider, ...VOICE_AGENT_FALLBACK_ORDER.filter((id) => id !== this.voiceLlmDefaultProvider)]
|
||||
: VOICE_AGENT_FALLBACK_ORDER;
|
||||
|
||||
for (const provider of fallbackOrder) {
|
||||
if (availability[provider]) {
|
||||
return provider;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error("No local voice LLM provider is available (claude/codex/opencode)");
|
||||
}
|
||||
|
||||
private getVoiceAgentModel(provider: VoiceAgentProvider): string | undefined {
|
||||
const configured = this.voiceLlmModel?.trim();
|
||||
if (configured) {
|
||||
return configured;
|
||||
}
|
||||
return VOICE_AGENT_DEFAULT_MODEL[provider];
|
||||
}
|
||||
|
||||
private async ensureVoiceAssistantAgent(): Promise<string> {
|
||||
if (this.voiceAssistantAgentId) {
|
||||
const existing = this.agentManager.getAgent(this.voiceAssistantAgentId);
|
||||
if (existing) {
|
||||
return existing.id;
|
||||
}
|
||||
this.voiceAssistantAgentId = null;
|
||||
}
|
||||
|
||||
const provider = this.resolveVoiceAgentProvider();
|
||||
const voiceAgentId = `voice-${uuidv4()}`;
|
||||
const cwd = join(this.paseoHome, "voice-agent-workspace");
|
||||
await mkdir(cwd, { recursive: true });
|
||||
|
||||
const mcpUrl = this.voiceAgentMcpUrl;
|
||||
if (!mcpUrl) {
|
||||
throw new Error("Voice MCP URL is not configured");
|
||||
}
|
||||
|
||||
const model = this.getVoiceAgentModel(provider);
|
||||
const config: AgentSessionConfig = {
|
||||
provider,
|
||||
cwd,
|
||||
modeId: VOICE_AGENT_DEFAULT_MODE[provider],
|
||||
...(model ? { model } : {}),
|
||||
internal: true,
|
||||
mcpServers: {
|
||||
paseo: {
|
||||
type: "http",
|
||||
url: `${mcpUrl}?callerAgentId=${encodeURIComponent(voiceAgentId)}`,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const created = await this.agentManager.createAgent(config, voiceAgentId, {
|
||||
labels: {
|
||||
surface: "voice",
|
||||
ui: "false",
|
||||
},
|
||||
});
|
||||
this.voiceAssistantAgentId = created.id;
|
||||
|
||||
this.registerVoiceSpeakHandler?.(created.id, async ({ text, signal }) => {
|
||||
const abortSignal = signal ?? this.abortController.signal;
|
||||
await this.ttsManager.generateAndWaitForPlayback(
|
||||
text,
|
||||
(msg) => this.emit(msg),
|
||||
abortSignal,
|
||||
true
|
||||
);
|
||||
this.emit({
|
||||
type: "activity_log",
|
||||
payload: {
|
||||
id: uuidv4(),
|
||||
timestamp: new Date(),
|
||||
type: "assistant",
|
||||
content: text,
|
||||
},
|
||||
});
|
||||
});
|
||||
this.registerVoiceCallerContext?.(created.id, {
|
||||
childAgentDefaultLabels: { ui: "true" },
|
||||
allowCustomCwd: true,
|
||||
enableVoiceTools: true,
|
||||
});
|
||||
|
||||
this.sessionLogger.info(
|
||||
{
|
||||
voiceAssistantAgentId: created.id,
|
||||
provider,
|
||||
model: model ?? null,
|
||||
providerExplicit: this.voiceLlmProviderExplicit,
|
||||
},
|
||||
"Voice assistant agent initialized"
|
||||
);
|
||||
return created.id;
|
||||
}
|
||||
|
||||
private buildVoiceAgentPrompt(userText: string): string {
|
||||
return [
|
||||
VOICE_AGENT_SYSTEM_INSTRUCTION,
|
||||
"",
|
||||
`User said: ${userText.trim()}`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
private shouldAllowVoicePermission(request: AgentPermissionRequest): boolean {
|
||||
const name = request.name.toLowerCase();
|
||||
if (name.includes("mcp") || name.includes("paseo") || name.includes("speak")) {
|
||||
return true;
|
||||
}
|
||||
if (name === "codextool") {
|
||||
const metadata = request.metadata ?? {};
|
||||
const rawQuestions = metadata.questions;
|
||||
if (Array.isArray(rawQuestions)) {
|
||||
const text = JSON.stringify(rawQuestions).toLowerCase();
|
||||
return text.includes("mcp") || text.includes("paseo") || text.includes("speak");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private async processWithVoiceAgent(userText: string): Promise<void> {
|
||||
const agentId = await this.ensureVoiceAssistantAgent();
|
||||
|
||||
await this.interruptAgentIfRunning(agentId);
|
||||
|
||||
const prompt = this.buildVoiceAgentPrompt(userText);
|
||||
this.agentManager.recordUserMessage(agentId, userText);
|
||||
|
||||
let sawSpeakToolCall = false;
|
||||
const assistantTextChunks: string[] = [];
|
||||
const iterator = this.agentManager.streamAgent(agentId, prompt);
|
||||
for await (const event of iterator) {
|
||||
if (event.type === "turn_failed") {
|
||||
throw new Error(event.error);
|
||||
}
|
||||
if (event.type === "timeline") {
|
||||
if (event.item.type === "tool_call" && typeof event.item.name === "string") {
|
||||
if (event.item.name.toLowerCase().includes("speak")) {
|
||||
sawSpeakToolCall = true;
|
||||
}
|
||||
}
|
||||
if (event.item.type === "assistant_message" && event.item.text.trim().length > 0) {
|
||||
assistantTextChunks.push(event.item.text.trim());
|
||||
}
|
||||
}
|
||||
if (event.type === "permission_requested") {
|
||||
if (this.shouldAllowVoicePermission(event.request)) {
|
||||
await this.agentManager.respondToPermission(agentId, event.request.id, {
|
||||
behavior: "allow",
|
||||
});
|
||||
} else {
|
||||
await this.agentManager.respondToPermission(agentId, event.request.id, {
|
||||
behavior: "deny",
|
||||
message: "Voice assistant policy only allows MCP paseo tools.",
|
||||
interrupt: true,
|
||||
});
|
||||
throw new Error(
|
||||
`Voice assistant denied non-MCP tool request: ${event.request.name}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!sawSpeakToolCall && assistantTextChunks.length > 0) {
|
||||
const fallbackText = assistantTextChunks.join(" ").trim();
|
||||
await this.ttsManager.generateAndWaitForPlayback(
|
||||
fallbackText,
|
||||
(msg) => this.emit(msg),
|
||||
this.abortController.signal,
|
||||
true
|
||||
);
|
||||
this.emit({
|
||||
type: "activity_log",
|
||||
payload: {
|
||||
id: uuidv4(),
|
||||
timestamp: new Date(),
|
||||
type: "assistant",
|
||||
content: fallbackText,
|
||||
},
|
||||
});
|
||||
this.sessionLogger.warn(
|
||||
{ voiceAssistantAgentId: agentId },
|
||||
"Voice agent responded without speak tool; used fallback TTS from assistant text"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process user message through LLM with streaming and tool execution
|
||||
*/
|
||||
private async processWithLLM(enableTTS: boolean): Promise<void> {
|
||||
private async processWithLLM(enableTTS: boolean, latestUserText?: string): Promise<void> {
|
||||
if (enableTTS && this.voiceLlmProvider !== "openrouter") {
|
||||
const text =
|
||||
typeof latestUserText === "string" && latestUserText.trim().length > 0
|
||||
? latestUserText
|
||||
: (() => {
|
||||
const lastUser = [...this.messages]
|
||||
.reverse()
|
||||
.find((message) => message.role === "user");
|
||||
if (!lastUser) {
|
||||
return "";
|
||||
}
|
||||
return typeof lastUser.content === "string"
|
||||
? lastUser.content
|
||||
: JSON.stringify(lastUser.content);
|
||||
})();
|
||||
const normalized = text.trim();
|
||||
if (!normalized) {
|
||||
return;
|
||||
}
|
||||
await this.processWithVoiceAgent(normalized);
|
||||
return;
|
||||
}
|
||||
|
||||
let assistantResponse = "";
|
||||
let pendingTTS: Promise<void> | null = null;
|
||||
let textBuffer = "";
|
||||
@@ -5180,6 +5483,21 @@ export class Session {
|
||||
this.agentTools = null;
|
||||
}
|
||||
|
||||
if (this.voiceAssistantAgentId) {
|
||||
try {
|
||||
await this.agentManager.closeAgent(this.voiceAssistantAgentId);
|
||||
} catch (error) {
|
||||
this.sessionLogger.warn(
|
||||
{ err: error, voiceAssistantAgentId: this.voiceAssistantAgentId },
|
||||
"Failed to close voice assistant agent"
|
||||
);
|
||||
} finally {
|
||||
this.unregisterVoiceSpeakHandler?.(this.voiceAssistantAgentId);
|
||||
this.unregisterVoiceCallerContext?.(this.voiceAssistantAgentId);
|
||||
this.voiceAssistantAgentId = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Unsubscribe from all terminals
|
||||
for (const unsubscribe of this.terminalSubscriptions.values()) {
|
||||
unsubscribe();
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { createRequire } from "node:module";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
export type SherpaOnnxModule = {
|
||||
createOnlineRecognizer: (config: any) => any;
|
||||
@@ -8,11 +10,39 @@ export type SherpaOnnxModule = {
|
||||
|
||||
let cached: SherpaOnnxModule | null = null;
|
||||
|
||||
function ensureSherpaNativeLibraryPath(requireFn: NodeRequire): void {
|
||||
const platform = os.platform();
|
||||
if (platform !== "darwin" && platform !== "linux") {
|
||||
return;
|
||||
}
|
||||
|
||||
const platformArch = `${platform}-${os.arch()}`;
|
||||
const packageName = `sherpa-onnx-${platformArch}`;
|
||||
|
||||
let nativeDir: string;
|
||||
try {
|
||||
const binaryPath = requireFn.resolve(`${packageName}/sherpa-onnx.node`);
|
||||
nativeDir = path.dirname(binaryPath);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
const envKey = platform === "darwin" ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH";
|
||||
const current = process.env[envKey]?.trim() ?? "";
|
||||
const entries = current.length > 0 ? current.split(":").filter(Boolean) : [];
|
||||
if (entries.includes(nativeDir)) {
|
||||
return;
|
||||
}
|
||||
|
||||
process.env[envKey] = entries.length > 0 ? `${nativeDir}:${entries.join(":")}` : nativeDir;
|
||||
}
|
||||
|
||||
export function loadSherpaOnnx(): SherpaOnnxModule {
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
const require = createRequire(import.meta.url);
|
||||
ensureSherpaNativeLibraryPath(require);
|
||||
cached = require("sherpa-onnx") as SherpaOnnxModule;
|
||||
return cached;
|
||||
}
|
||||
|
||||
@@ -21,6 +21,10 @@ type TestPaseoDaemonOptions = {
|
||||
cleanup?: boolean;
|
||||
openai?: PaseoOpenAIConfig;
|
||||
speech?: PaseoSpeechConfig;
|
||||
openrouterApiKey?: string | null;
|
||||
voiceLlmProvider?: PaseoDaemonConfig["voiceLlmProvider"];
|
||||
voiceLlmProviderExplicit?: boolean;
|
||||
voiceLlmModel?: string | null;
|
||||
dictationFinalTimeoutMs?: number;
|
||||
};
|
||||
|
||||
@@ -78,7 +82,10 @@ export async function createTestPaseoDaemon(
|
||||
appBaseUrl: "https://app.paseo.sh",
|
||||
openai: options.openai,
|
||||
speech: options.speech,
|
||||
openrouterApiKey: null,
|
||||
openrouterApiKey: options.openrouterApiKey ?? null,
|
||||
voiceLlmProvider: options.voiceLlmProvider ?? null,
|
||||
voiceLlmProviderExplicit: options.voiceLlmProviderExplicit ?? false,
|
||||
voiceLlmModel: options.voiceLlmModel ?? null,
|
||||
dictationFinalTimeoutMs: options.dictationFinalTimeoutMs,
|
||||
downloadTokenTtlMs: options.downloadTokenTtlMs,
|
||||
};
|
||||
|
||||
124
packages/server/src/server/voice-local-agent.e2e.test.ts
Normal file
124
packages/server/src/server/voice-local-agent.e2e.test.ts
Normal file
@@ -0,0 +1,124 @@
|
||||
import { afterAll, beforeAll, describe, expect, test } from "vitest";
|
||||
|
||||
import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js";
|
||||
|
||||
const openaiApiKey = process.env.OPENAI_API_KEY ?? null;
|
||||
const shouldRun =
|
||||
process.env.PASEO_VOICE_LOCAL_AGENT_E2E === "1" &&
|
||||
Boolean(openaiApiKey) &&
|
||||
!process.env.CI;
|
||||
|
||||
function waitForSignal<T>(
|
||||
timeoutMs: number,
|
||||
setup: (
|
||||
resolve: (value: T) => void,
|
||||
reject: (error: Error) => void
|
||||
) => () => void
|
||||
): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let cleanup: (() => void) | null = null;
|
||||
const timeout = setTimeout(() => {
|
||||
cleanup?.();
|
||||
reject(new Error(`Timeout waiting for event after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
|
||||
cleanup = setup(
|
||||
(value) => {
|
||||
clearTimeout(timeout);
|
||||
cleanup?.();
|
||||
resolve(value);
|
||||
},
|
||||
(error) => {
|
||||
clearTimeout(timeout);
|
||||
cleanup?.();
|
||||
reject(error);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
(shouldRun ? describe : describe.skip)(
|
||||
"voice local-agent e2e",
|
||||
() => {
|
||||
let ctx: DaemonTestContext;
|
||||
|
||||
beforeAll(async () => {
|
||||
ctx = await createDaemonTestContext({
|
||||
agentClients: {},
|
||||
openai: { apiKey: openaiApiKey! },
|
||||
speech: {
|
||||
dictationSttProvider: "openai",
|
||||
voiceSttProvider: "openai",
|
||||
voiceTtsProvider: "openai",
|
||||
},
|
||||
voiceLlmProvider: "codex",
|
||||
voiceLlmProviderExplicit: true,
|
||||
voiceLlmModel: "gpt-5.2-mini",
|
||||
});
|
||||
}, 120000);
|
||||
|
||||
afterAll(async () => {
|
||||
await ctx.cleanup();
|
||||
}, 60000);
|
||||
|
||||
test(
|
||||
"routes voice turns through local agent speak tool",
|
||||
async () => {
|
||||
await ctx.client.setVoiceConversation(true, `voice-local-agent-${Date.now()}`);
|
||||
|
||||
const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => {
|
||||
const offAudio = ctx.client.on("audio_output", (msg) => {
|
||||
if (msg.type !== "audio_output") return;
|
||||
resolve({ chunkId: msg.payload.id });
|
||||
});
|
||||
const offError = ctx.client.on("activity_log", (msg) => {
|
||||
if (msg.type !== "activity_log") return;
|
||||
if (msg.payload.type !== "error") return;
|
||||
reject(new Error(String(msg.payload.content)));
|
||||
});
|
||||
return () => {
|
||||
offAudio();
|
||||
offError();
|
||||
};
|
||||
});
|
||||
|
||||
const assistantLogPromise = waitForSignal<string>(120000, (resolve, reject) => {
|
||||
const offLog = ctx.client.on("activity_log", (msg) => {
|
||||
if (msg.type !== "activity_log") return;
|
||||
if (msg.payload.type !== "assistant") return;
|
||||
const content = String(msg.payload.content ?? "");
|
||||
if (!content.trim()) return;
|
||||
resolve(content);
|
||||
});
|
||||
const offError = ctx.client.on("activity_log", (msg) => {
|
||||
if (msg.type !== "activity_log") return;
|
||||
if (msg.payload.type !== "error") return;
|
||||
reject(new Error(String(msg.payload.content)));
|
||||
});
|
||||
return () => {
|
||||
offLog();
|
||||
offError();
|
||||
};
|
||||
});
|
||||
|
||||
ctx.client.sendUserMessage(
|
||||
"Use the speak tool and say exactly: local voice agent path is working."
|
||||
);
|
||||
|
||||
const [{ chunkId }, assistantText] = await Promise.all([
|
||||
audioPromise,
|
||||
assistantLogPromise,
|
||||
]);
|
||||
|
||||
expect(chunkId.length).toBeGreaterThan(0);
|
||||
expect(assistantText.toLowerCase()).toContain("local voice agent path is working");
|
||||
|
||||
const agents = await ctx.client.fetchAgents();
|
||||
expect(
|
||||
agents.some((agent) => String(agent.labels?.surface ?? "") === "voice")
|
||||
).toBe(false);
|
||||
},
|
||||
180000
|
||||
);
|
||||
}
|
||||
);
|
||||
@@ -23,6 +23,7 @@ import { VoiceConversationStore } from "./voice-conversation-store.js";
|
||||
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
|
||||
|
||||
export type AgentMcpTransportFactory = () => Promise<Transport>;
|
||||
type VoiceAgentProvider = "claude" | "codex" | "opencode";
|
||||
|
||||
type WebSocketServerConfig = {
|
||||
allowedOrigins: Set<string>;
|
||||
@@ -76,8 +77,26 @@ export class VoiceAssistantWebSocketServer {
|
||||
} | null;
|
||||
private readonly voice: {
|
||||
openrouterApiKey?: string | null;
|
||||
voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null;
|
||||
voiceLlmProviderExplicit?: boolean;
|
||||
voiceLlmDefaultProvider?: VoiceAgentProvider | null;
|
||||
voiceLlmModel?: string | null;
|
||||
voiceLlmAvailability?: Record<VoiceAgentProvider, boolean> | null;
|
||||
voiceAgentMcpUrl?: string | null;
|
||||
} | null;
|
||||
private readonly voiceSpeakHandlers = new Map<
|
||||
string,
|
||||
(params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>
|
||||
>();
|
||||
private readonly voiceCallerContexts = new Map<
|
||||
string,
|
||||
{
|
||||
childAgentDefaultLabels?: Record<string, string>;
|
||||
lockedCwd?: string;
|
||||
allowCustomCwd?: boolean;
|
||||
enableVoiceTools?: boolean;
|
||||
}
|
||||
>();
|
||||
|
||||
constructor(
|
||||
server: HTTPServer,
|
||||
@@ -93,7 +112,12 @@ export class VoiceAssistantWebSocketServer {
|
||||
terminalManager?: TerminalManager | null,
|
||||
voice?: {
|
||||
openrouterApiKey?: string | null;
|
||||
voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null;
|
||||
voiceLlmProviderExplicit?: boolean;
|
||||
voiceLlmDefaultProvider?: VoiceAgentProvider | null;
|
||||
voiceLlmModel?: string | null;
|
||||
voiceLlmAvailability?: Record<VoiceAgentProvider, boolean> | null;
|
||||
voiceAgentMcpUrl?: string | null;
|
||||
},
|
||||
dictation?: {
|
||||
finalTimeoutMs?: number;
|
||||
@@ -225,6 +249,20 @@ export class VoiceAssistantWebSocketServer {
|
||||
this.terminalManager,
|
||||
this.voiceConversationStore,
|
||||
this.voice ?? undefined,
|
||||
{
|
||||
registerVoiceSpeakHandler: (agentId, handler) => {
|
||||
this.voiceSpeakHandlers.set(agentId, handler);
|
||||
},
|
||||
unregisterVoiceSpeakHandler: (agentId) => {
|
||||
this.voiceSpeakHandlers.delete(agentId);
|
||||
},
|
||||
registerVoiceCallerContext: (agentId, context) => {
|
||||
this.voiceCallerContexts.set(agentId, context);
|
||||
},
|
||||
unregisterVoiceCallerContext: (agentId) => {
|
||||
this.voiceCallerContexts.delete(agentId);
|
||||
},
|
||||
},
|
||||
this.dictation ?? undefined
|
||||
);
|
||||
|
||||
@@ -263,6 +301,23 @@ export class VoiceAssistantWebSocketServer {
|
||||
});
|
||||
}
|
||||
|
||||
public resolveVoiceSpeakHandler(
|
||||
callerAgentId: string
|
||||
): ((params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>) | null {
|
||||
return this.voiceSpeakHandlers.get(callerAgentId) ?? null;
|
||||
}
|
||||
|
||||
public resolveVoiceCallerContext(
|
||||
callerAgentId: string
|
||||
): {
|
||||
childAgentDefaultLabels?: Record<string, string>;
|
||||
lockedCwd?: string;
|
||||
allowCustomCwd?: boolean;
|
||||
enableVoiceTools?: boolean;
|
||||
} | null {
|
||||
return this.voiceCallerContexts.get(callerAgentId) ?? null;
|
||||
}
|
||||
|
||||
private async detachSocket(
|
||||
ws: WebSocketLike,
|
||||
connectionLogger: pino.Logger,
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import path from "node:path";
|
||||
import dotenv from "dotenv";
|
||||
|
||||
// Load repo-root .env for integration/E2E tests (OpenAI, etc.)
|
||||
// Load package-local .env.test first for integration/E2E credentials, then repo-root .env fallback.
|
||||
dotenv.config({ path: path.resolve(process.cwd(), ".env.test"), override: true });
|
||||
dotenv.config({ path: path.resolve(process.cwd(), "../.env") });
|
||||
|
||||
process.env.GIT_TERMINAL_PROMPT = "0";
|
||||
|
||||
Reference in New Issue
Block a user