puter/paseo
1
0
Fork 0
mirror of https://github.com/getpaseo/paseo.git synced 2026-07-29 12:01:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #22 from boudra/local-streaming-speech-tts

Browse Source 
voice: harden local voice agent path and UUID validation
This commit is contained in:
Mohamed Boudra
2026-02-07 00:31:30 +07:00
committed by GitHub
parent f26d7f5552 00bbac007c
commit fcb1004f94
69 changed files with 3991 additions and 2096 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
package-lock.json generated
View File

@@ -4688,17 +4688,6 @@
"version": "1.1.12",
"license": "MIT"
},
"node_modules/@openrouter/ai-sdk-provider": {
"version": "1.2.0",
"license": "Apache-2.0",
"engines": {
"node": ">=18"
},
"peerDependencies": {
"ai": "^5.0.0",
"zod": "^3.24.1 || ^v4"
}
},
"node_modules/@opentelemetry/api": {
"version": "1.9.0",
"license": "Apache-2.0",
@@ -20286,7 +20275,6 @@
"@lezer/python": "^1.1.18",
"@modelcontextprotocol/sdk": "^1.20.1",
"@opencode-ai/sdk": "^1.1.12",
"@openrouter/ai-sdk-provider": "^1.2.0",
"@sctg/sentencepiece-js": "^1.1.0",
"@xterm/headless": "^6.0.0",
"ai": "^5.0.76",

32
packages/app/src/contexts/voice-context.tsx
View File

@@ -2,12 +2,13 @@ import { createContext, useContext, useState, ReactNode, useCallback, useEffect,
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
import type { SessionState } from "@/stores/session-store";
import { useSessionStore } from "@/stores/session-store";
import AsyncStorage from "@react-native-async-storage/async-storage";
import { randomUUID } from "expo-crypto";
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
const VOICE_CONVERSATION_ID_STORAGE_KEY = "@paseo:voice-conversation-id";
const KEEP_AWAKE_TAG = "paseo:voice";
const VOICE_VAD_VOLUME_THRESHOLD = 0.18;
const VOICE_VAD_SILENCE_DURATION_MS = 1400;
const VOICE_VAD_SPEECH_CONFIRMATION_MS = 120;
const VOICE_VAD_DETECTION_GRACE_PERIOD_MS = 700;
interface VoiceContextValue {
isVoiceMode: boolean;
@@ -123,10 +124,10 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
console.error("[Voice] Cannot handle error - setMessages not available from SessionState");
}
},
volumeThreshold: 0.3,
silenceDuration: 2000,
speechConfirmationDuration: 300,
detectionGracePeriod: 200,
volumeThreshold: VOICE_VAD_VOLUME_THRESHOLD,
silenceDuration: VOICE_VAD_SILENCE_DURATION_MS,
speechConfirmationDuration: VOICE_VAD_SPEECH_CONFIRMATION_MS,
detectionGracePeriod: VOICE_VAD_DETECTION_GRACE_PERIOD_MS,
});
useEffect(() => {
@@ -166,18 +167,9 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
console.log("[Voice] Mode enabled");
if (session?.client) {
let voiceConversationId =
(await AsyncStorage.getItem(VOICE_CONVERSATION_ID_STORAGE_KEY)) ?? null;
if (!voiceConversationId) {
voiceConversationId = randomUUID();
await AsyncStorage.setItem(
VOICE_CONVERSATION_ID_STORAGE_KEY,
voiceConversationId
);
}
await session.client.setVoiceConversation(true, voiceConversationId);
await session.client.setVoiceMode(true);
} else {
console.warn("[Voice] setVoiceConversation skipped: daemon unavailable");
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
}
} catch (error: any) {
console.error("[Voice] Failed to start:", error);
@@ -200,9 +192,9 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
console.log("[Voice] Mode disabled");
if (session?.client) {
await session.client.setVoiceConversation(false);
await session.client.setVoiceMode(false);
} else {
console.warn("[Voice] setVoiceConversation skipped: daemon unavailable");
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
}
} catch (error: any) {
console.error("[Voice] Failed to stop:", error);

33
packages/app/src/voice/speech-segmenter.test.ts
View File

@@ -88,4 +88,37 @@ describe("SpeechSegmenter", () => {
expect(lastCall.isLast).toBe(true);
expect(lastCall.audioData.length).toBeGreaterThan(0);
});
it("keeps detection alive across a brief pause and confirms short utterances", () => {
const onSpeechStart = vi.fn();
const detectingChanges: boolean[] = [];
const segmenter = new SpeechSegmenter(
{
enableContinuousStreaming: false,
volumeThreshold: 0.18,
silenceDurationMs: 1400,
speechConfirmationMs: 120,
detectionGracePeriodMs: 700,
minChunkDurationMs: 100,
pcmSampleRate: 1000,
},
{
onSpeechStart,
onDetectingChange: (v) => detectingChanges.push(v),
}
);
const t0 = 20_000;
segmenter.pushVolumeLevel(0.4, t0);
segmenter.pushPcmChunk(mkPcmBytes(20));
segmenter.pushVolumeLevel(0.0, t0 + 80);
segmenter.pushPcmChunk(mkPcmBytes(20));
segmenter.pushVolumeLevel(0.4, t0 + 140);
segmenter.pushPcmChunk(mkPcmBytes(20));
expect(detectingChanges).toContain(true);
expect(onSpeechStart).toHaveBeenCalledTimes(1);
});
});

9
packages/cli/src/cli.ts
View File

@@ -14,6 +14,7 @@ import { runInspectCommand } from './commands/agent/inspect.js'
import { runWaitCommand } from './commands/agent/wait.js'
import { runAttachCommand } from './commands/agent/attach.js'
import { withOutput } from './output/index.js'
import { runVoiceMcpBridgeCommand } from './commands/voice-mcp-bridge.js'
const VERSION = '0.1.0'
@@ -141,5 +142,13 @@ export function createCli(): Command {
// Worktree commands
program.addCommand(createWorktreeCommand())
// Internal voice MCP stdio bridge command (hidden).
program
.command('__paseo_voice_mcp_bridge', { hidden: true })
.description('Internal voice MCP bridge command')
.requiredOption('--socket <path>')
.requiredOption('--caller-agent-id <id>')
.action(runVoiceMcpBridgeCommand)
return program
}

17
packages/cli/src/commands/voice-mcp-bridge.ts Normal file
View File

@@ -0,0 +1,17 @@
import { runVoiceMcpBridgeCli } from "@getpaseo/server";
type VoiceBridgeOptions = {
socket: string;
callerAgentId: string;
};
export async function runVoiceMcpBridgeCommand(
options: VoiceBridgeOptions
): Promise<void> {
await runVoiceMcpBridgeCli([
"--socket",
options.socket,
"--caller-agent-id",
options.callerAgentId,
]);
}

5
packages/server/package.json
View File

@@ -36,7 +36,7 @@
"@lezer/python": "^1.1.18",
"@modelcontextprotocol/sdk": "^1.20.1",
"@opencode-ai/sdk": "^1.1.12",
"@openrouter/ai-sdk-provider": "^1.2.0",
"@sctg/sentencepiece-js": "^1.1.0",
"@xterm/headless": "^6.0.0",
"ai": "^5.0.76",
"ajv": "^8.17.1",
@@ -45,13 +45,12 @@
"express-basic-auth": "^1.2.1",
"mnemonic-id": "^3.2.7",
"node-pty": "^1.0.0",
"onnxruntime-node": "^1.23.0",
"openai": "^4.20.0",
"pino": "^10.2.0",
"pino-pretty": "^13.1.3",
"playwright": "^1.56.1",
"qrcode": "^1.5.4",
"@sctg/sentencepiece-js": "^1.1.0",
"onnxruntime-node": "^1.23.0",
"sherpa-onnx": "^1.12.23",
"sherpa-onnx-node": "^1.12.23",
"strip-ansi": "^7.1.2",

22
packages/server/scripts/download-speech-models.ts
View File

@@ -1,12 +1,16 @@
import { resolvePaseoHome } from "../src/server/paseo-home.js";
import { createRootLogger } from "../src/server/logger.js";
import { ensureSherpaOnnxModels } from "../src/server/speech/providers/local/sherpa/model-downloader.js";
import type { SherpaOnnxModelId } from "../src/server/speech/providers/local/sherpa/model-catalog.js";
import {
DEFAULT_LOCAL_STT_MODEL,
DEFAULT_LOCAL_TTS_MODEL,
ensureLocalSpeechModels,
type LocalSpeechModelId,
} from "../src/server/speech/providers/local/models.js";
function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxModelId[] } {
function parseArgs(argv: string[]): { modelsDir: string; modelIds: LocalSpeechModelId[] } {
const home = resolvePaseoHome();
let modelsDir = process.env.PASEO_SHERPA_ONNX_MODELS_DIR || `${home}/models/sherpa-onnx`;
const modelIds: SherpaOnnxModelId[] = [];
let modelsDir = process.env.PASEO_LOCAL_MODELS_DIR || `${home}/models/local-speech`;
const modelIds: LocalSpeechModelId[] = [];
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
@@ -16,7 +20,7 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxMod
continue;
}
if (arg === "--model") {
const id = argv[i + 1] as SherpaOnnxModelId | undefined;
const id = argv[i + 1] as LocalSpeechModelId | undefined;
if (!id) {
throw new Error("--model requires a value");
}
@@ -27,8 +31,8 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxMod
}
if (modelIds.length === 0) {
const stt = (process.env.PASEO_SHERPA_STT_PRESET || "zipformer-bilingual-zh-en-2023-02-20") as SherpaOnnxModelId;
const tts = (process.env.PASEO_SHERPA_TTS_PRESET || "pocket-tts-onnx-int8") as SherpaOnnxModelId;
const stt = (process.env.PASEO_LOCAL_STT_MODEL || DEFAULT_LOCAL_STT_MODEL) as LocalSpeechModelId;
const tts = (process.env.PASEO_LOCAL_TTS_MODEL || DEFAULT_LOCAL_TTS_MODEL) as LocalSpeechModelId;
modelIds.push(stt, tts);
}
@@ -38,5 +42,5 @@ function parseArgs(argv: string[]): { modelsDir: string; modelIds: SherpaOnnxMod
const logger = createRootLogger({ level: "info", format: "pretty" });
const { modelsDir, modelIds } = parseArgs(process.argv.slice(2));
await ensureSherpaOnnxModels({ modelsDir, modelIds, autoDownload: true, logger });
await ensureLocalSpeechModels({ modelsDir, modelIds, autoDownload: true, logger });
logger.info({ modelsDir, modelIds }, "Done downloading speech models");

4
packages/server/scripts/list-speech-models.ts
View File

@@ -1,6 +1,6 @@
import { listSherpaOnnxModels } from "../src/server/speech/providers/local/sherpa/model-catalog.js";
import { listLocalSpeechModels } from "../src/server/speech/providers/local/models.js";
const models = listSherpaOnnxModels()
const models = listLocalSpeechModels()
.slice()
.sort((a, b) => a.kind.localeCompare(b.kind) || a.id.localeCompare(b.id));

95
packages/server/src/client/daemon-client.ts
View File

@@ -12,9 +12,7 @@ import type {
AgentStreamEventPayload,
AgentSnapshotPayload,
AgentPermissionResolvedMessage,
VoiceConversationLoadedMessage,
CreateAgentRequestMessage,
DeleteVoiceConversationResponseMessage,
FileDownloadTokenResponse,
FileExplorerResponse,
GitDiffResponse,
@@ -34,7 +32,6 @@ import type {
ProjectIconResponse,
ListCommandsResponse,
ExecuteCommandResponse,
ListVoiceConversationsResponseMessage,
ListProviderModelsResponseMessage,
SpeechModelsListResponse,
SpeechModelsDownloadResponse,
@@ -184,9 +181,6 @@ export type CreateAgentRequestOptions = {
labels?: Record<string, string>;
} & AgentConfigOverrides;
type VoiceConversationLoadedPayload = VoiceConversationLoadedMessage["payload"];
type ListVoiceConversationsPayload = ListVoiceConversationsResponseMessage["payload"];
type DeleteVoiceConversationPayload = DeleteVoiceConversationResponseMessage["payload"];
type GitDiffPayload = GitDiffResponse["payload"];
type HighlightedDiffPayload = HighlightedDiffResponse["payload"];
type CheckoutStatusPayload = CheckoutStatusResponse["payload"];
@@ -765,10 +759,6 @@ export class DaemonClient {
}
}
sendUserMessage(text: string): void {
this.sendSessionMessage({ type: "user_text", text });
}
clearAgentAttention(agentId: string | string[]): void {
this.sendSessionMessage({ type: "clear_agent_attention", agentId });
}
@@ -926,87 +916,6 @@ export class DaemonClient {
}
}
// ============================================================================
// Voice Conversation RPC
// ============================================================================
async loadVoiceConversation(
voiceConversationId: string,
requestId?: string
): Promise<VoiceConversationLoadedPayload> {
const resolvedRequestId = this.createRequestId(requestId);
const message = SessionInboundMessageSchema.parse({
type: "load_voice_conversation_request",
voiceConversationId,
requestId: resolvedRequestId,
});
return this.sendRequest({
requestId: resolvedRequestId,
message,
timeout: 10000,
options: { skipQueue: true },
select: (msg) => {
if (msg.type !== "voice_conversation_loaded") {
return null;
}
if (msg.payload.requestId !== resolvedRequestId) {
return null;
}
return msg.payload;
},
});
}
async listVoiceConversations(requestId?: string): Promise<ListVoiceConversationsPayload> {
const resolvedRequestId = this.createRequestId(requestId);
const message = SessionInboundMessageSchema.parse({
type: "list_voice_conversations_request",
requestId: resolvedRequestId,
});
return this.sendRequest({
requestId: resolvedRequestId,
message,
timeout: 10000,
options: { skipQueue: true },
select: (msg) => {
if (msg.type !== "list_voice_conversations_response") {
return null;
}
if (msg.payload.requestId !== resolvedRequestId) {
return null;
}
return msg.payload;
},
});
}
async deleteVoiceConversation(
voiceConversationId: string,
requestId?: string
): Promise<DeleteVoiceConversationPayload> {
const resolvedRequestId = this.createRequestId(requestId);
const message = SessionInboundMessageSchema.parse({
type: "delete_voice_conversation_request",
voiceConversationId,
requestId: resolvedRequestId,
});
return this.sendRequest({
requestId: resolvedRequestId,
message,
timeout: 10000,
options: { skipQueue: true },
select: (msg) => {
if (msg.type !== "delete_voice_conversation_response") {
return null;
}
if (msg.payload.requestId !== resolvedRequestId) {
return null;
}
return msg.payload;
},
});
}
// ============================================================================
// Agent Lifecycle
// ============================================================================
@@ -1376,8 +1285,8 @@ export class DaemonClient {
// Audio / Voice
// ============================================================================
async setVoiceConversation(enabled: boolean, voiceConversationId?: string): Promise<void> {
this.sendSessionMessage({ type: "set_voice_conversation", enabled, voiceConversationId });
async setVoiceMode(enabled: boolean, voiceAgentId?: string): Promise<void> {
this.sendSessionMessage({ type: "set_voice_mode", enabled, voiceAgentId });
}
async sendVoiceAudioChunk(

48
packages/server/src/server/agent/activity-curator.test.ts
View File

@@ -66,6 +66,22 @@ describe("curateAgentActivity", () => {
expect(result).toBe("[ListFiles]");
});
test("does not treat generic double-underscore tool names as MCP calls", () => {
const timeline: AgentTimelineItem[] = [
{
type: "tool_call",
callId: "call-1",
name: "custom__tool",
input: {},
status: "completed",
},
];
const result = curateAgentActivity(timeline);
expect(result).toBe("[custom__tool]");
});
test("serializes todo items as [Tasks]", () => {
const timeline: AgentTimelineItem[] = [
{
@@ -328,5 +344,37 @@ describe("curateAgentActivity", () => {
expect(result).toBe("[Grep] TODO");
});
test("shows speak tool text input", () => {
const timeline: AgentTimelineItem[] = [
{
type: "tool_call",
callId: "s1",
name: "speak",
input: { text: "hello from voice" },
status: "completed",
},
];
const result = curateAgentActivity(timeline);
expect(result).toBe('[speak] {"text":"hello from voice"}');
});
test("shows MCP tool input JSON", () => {
const timeline: AgentTimelineItem[] = [
{
type: "tool_call",
callId: "m1",
name: "paseo__create_agent",
input: { cwd: "/tmp/repo", initialPrompt: "do the thing" },
status: "completed",
},
];
const result = curateAgentActivity(timeline);
expect(result).toBe(
'[paseo__create_agent] {"cwd":"/tmp/repo","initialPrompt":"do the thing"}'
);
});
});
});

35
packages/server/src/server/agent/activity-curator.ts
View File

@@ -2,6 +2,7 @@ import type { AgentTimelineItem } from "./agent-sdk-types.js";
import { extractPrincipalParam } from "../../utils/tool-call-parsers.js";
const DEFAULT_MAX_ITEMS = 40;
const MAX_TOOL_INPUT_CHARS = 400;
function appendText(buffer: string, text: string): string {
const normalized = text.trim();
@@ -25,6 +26,35 @@ function flushBuffers(lines: string[], buffers: { message: string; thought: stri
buffers.thought = "";
}
function isLikelyMcpToolCall(name: string): boolean {
const normalized = name.toLowerCase();
return (
normalized === "speak" ||
normalized.startsWith("mcp") ||
normalized.includes("mcp__") ||
normalized.startsWith("paseo") ||
normalized.includes("paseo__")
);
}
function formatToolInputJson(input: unknown): string | null {
if (input === undefined) {
return null;
}
try {
const encoded = JSON.stringify(input);
if (!encoded) {
return null;
}
if (encoded.length <= MAX_TOOL_INPUT_CHARS) {
return encoded;
}
return `${encoded.slice(0, MAX_TOOL_INPUT_CHARS)}...`;
} catch {
return null;
}
}
/**
* Collapse timeline items:
* - Dedupe tool calls by callId (pending/completed -> single)
@@ -127,6 +157,11 @@ export function curateAgentActivity(
break;
case "tool_call": {
flushBuffers(lines, buffers);
const inputJson = formatToolInputJson(item.input);
if (isLikelyMcpToolCall(item.name) && inputJson) {
lines.push(`[${item.name}] ${inputJson}`);
break;
}
const principal = extractPrincipalParam(item.input);
if (principal) {
lines.push(`[${item.name}] ${principal}`);

12
packages/server/src/server/agent/agent-management-mcp.ts
View File

@@ -23,8 +23,6 @@
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import { z } from "zod";
import { homedir } from "node:os";
import { resolve } from "node:path";
import { ensureValidJson } from "../json-utils.js";
import type { Logger } from "pino";
@@ -51,6 +49,7 @@ import { AgentStorage } from "./agent-storage.js";
import { createWorktree } from "../../utils/worktree.js";
import { WaitForAgentTracker } from "./wait-for-agent-tracker.js";
import { scheduleAgentMetadataGeneration } from "./agent-metadata-generator.js";
import { expandUserPath } from "../path-utils.js";
export interface AgentManagementMcpOptions {
agentManager: AgentManager;
@@ -77,13 +76,6 @@ const AgentStatusEnum = z.enum([
// 50 seconds - surface friendly message before SDK tool timeout (~60s)
const AGENT_WAIT_TIMEOUT_MS = 50000;
function expandPath(path: string): string {
if (path.startsWith("~/") || path === "~") {
return resolve(homedir(), path.slice(2));
}
return resolve(path);
}
async function waitForAgentWithTimeout(
agentManager: AgentManager,
agentId: string,
@@ -320,7 +312,7 @@ export async function createAgentManagementMcpServer(
title: string;
};
let resolvedCwd = expandPath(cwd);
let resolvedCwd = expandUserPath(cwd);
if (worktreeName) {
if (!baseBranch) {

131
packages/server/src/server/agent/agent-manager.test.ts
View File

@@ -118,7 +118,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "agent-without-model",
idFactory: () => "00000000-0000-4000-8000-000000000101",
});
const snapshot = await manager.createAgent({
@@ -129,7 +129,7 @@ describe("AgentManager", () => {
expect(snapshot.model).toBeUndefined();
});
test("createAgent persists provided title before returning", async () => {
test("createAgent fails when cwd does not exist", async () => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
@@ -139,7 +139,72 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "agent-with-title",
});
await expect(
manager.createAgent({
provider: "codex",
cwd: join(workdir, "does-not-exist"),
})
).rejects.toThrow("Working directory does not exist");
});
test("createAgent fails when generated agent ID is not a UUID", async () => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
const manager = new AgentManager({
clients: {
codex: new TestAgentClient(),
},
registry: storage,
logger,
idFactory: () => "not-a-uuid",
});
await expect(
manager.createAgent({
provider: "codex",
cwd: workdir,
})
).rejects.toThrow("createAgent: agentId must be a UUID");
});
test("createAgent fails when explicit agent ID is not a UUID", async () => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
const manager = new AgentManager({
clients: {
codex: new TestAgentClient(),
},
registry: storage,
logger,
});
await expect(
manager.createAgent(
{
provider: "codex",
cwd: workdir,
},
"not-a-uuid"
)
).rejects.toThrow("createAgent: agentId must be a UUID");
});
test("createAgent persists provided title before returning", async () => {
const agentId = "00000000-0000-4000-8000-000000000102";
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
const manager = new AgentManager({
clients: {
codex: new TestAgentClient(),
},
registry: storage,
logger,
idFactory: () => agentId,
});
const snapshot = await manager.createAgent({
@@ -148,12 +213,12 @@ describe("AgentManager", () => {
title: "Fix Login Bug",
});
expect(snapshot.id).toBe("agent-with-title");
expect(snapshot.id).toBe(agentId);
expect(snapshot.lifecycle).toBe("idle");
const persisted = await storage.get("agent-with-title");
const persisted = await storage.get(agentId);
expect(persisted?.title).toBe("Fix Login Bug");
expect(persisted?.id).toBe("agent-with-title");
expect(persisted?.id).toBe(agentId);
});
test("createAgent populates runtimeInfo after session creation", async () => {
@@ -166,7 +231,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "agent-with-runtime-info",
idFactory: () => "00000000-0000-4000-8000-000000000103",
});
const snapshot = await manager.createAgent({
@@ -191,7 +256,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "agent-with-run-runtime",
idFactory: () => "00000000-0000-4000-8000-000000000104",
});
const snapshot = await manager.createAgent({
@@ -211,6 +276,10 @@ describe("AgentManager", () => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
const generatedAgentIds = [
"00000000-0000-4000-8000-000000000105",
"00000000-0000-4000-8000-000000000106",
];
let agentCounter = 0;
const manager = new AgentManager({
clients: {
@@ -218,7 +287,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => `agent-${agentCounter++}`,
idFactory: () => generatedAgentIds[agentCounter++] ?? randomUUID(),
});
// Create a normal agent
@@ -242,6 +311,7 @@ describe("AgentManager", () => {
});
test("getAgent returns internal agents by ID", async () => {
const internalAgentId = "00000000-0000-4000-8000-000000000107";
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
@@ -251,7 +321,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "internal-agent",
idFactory: () => internalAgentId,
});
await manager.createAgent({
@@ -261,7 +331,7 @@ describe("AgentManager", () => {
internal: true,
});
const agent = manager.getAgent("internal-agent");
const agent = manager.getAgent(internalAgentId);
expect(agent).not.toBeNull();
expect(agent?.internal).toBe(true);
});
@@ -270,6 +340,10 @@ describe("AgentManager", () => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
const generatedAgentIds = [
"00000000-0000-4000-8000-000000000108",
"00000000-0000-4000-8000-000000000109",
];
let agentCounter = 0;
const manager = new AgentManager({
clients: {
@@ -277,7 +351,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => `agent-${agentCounter++}`,
idFactory: () => generatedAgentIds[agentCounter++] ?? randomUUID(),
});
const receivedEvents: string[] = [];
@@ -303,11 +377,12 @@ describe("AgentManager", () => {
});
// Should only have events from the normal agent
expect(receivedEvents.filter((id) => id === "agent-0").length).toBeGreaterThan(0);
expect(receivedEvents.filter((id) => id === "agent-1").length).toBe(0);
expect(receivedEvents.filter((id) => id === generatedAgentIds[0]).length).toBeGreaterThan(0);
expect(receivedEvents.filter((id) => id === generatedAgentIds[1]).length).toBe(0);
});
test("subscribe emits state events for internal agents when subscribed by agentId", async () => {
const internalAgentId = "00000000-0000-4000-8000-000000000110";
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
@@ -317,7 +392,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "internal-agent",
idFactory: () => internalAgentId,
});
const receivedEvents: string[] = [];
@@ -328,7 +403,7 @@ describe("AgentManager", () => {
receivedEvents.push(event.agent.id);
}
},
{ agentId: "internal-agent", replayState: false }
{ agentId: internalAgentId, replayState: false }
);
await manager.createAgent({
@@ -339,10 +414,26 @@ describe("AgentManager", () => {
});
// Should receive events when subscribed by specific agentId
expect(receivedEvents.filter((id) => id === "internal-agent").length).toBeGreaterThan(0);
expect(receivedEvents.filter((id) => id === internalAgentId).length).toBeGreaterThan(0);
});
test("subscribe fails when filter agentId is not a UUID", () => {
const manager = new AgentManager({
clients: {
codex: new TestAgentClient(),
},
logger,
});
expect(() =>
manager.subscribe(() => {}, {
agentId: "invalid-agent-id",
})
).toThrow("subscribe: agentId must be a UUID");
});
test("onAgentAttention is not called for internal agents", async () => {
const internalAgentId = "00000000-0000-4000-8000-000000000111";
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
const storage = new AgentStorage(storagePath, logger);
@@ -353,7 +444,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "internal-agent",
idFactory: () => internalAgentId,
onAgentAttention: ({ agentId }) => {
attentionCalls.push(agentId);
},
@@ -461,7 +552,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "plan-mode-agent",
idFactory: () => "00000000-0000-4000-8000-000000000112",
});
// Create agent in plan mode
@@ -588,7 +679,7 @@ describe("AgentManager", () => {
},
registry: storage,
logger,
idFactory: () => "close-race-agent",
idFactory: () => "00000000-0000-4000-8000-000000000113",
});
const snapshot = await manager.createAgent({

58
packages/server/src/server/agent/agent-manager.ts
View File

@@ -1,10 +1,12 @@
import { randomUUID } from "node:crypto";
import { resolve } from "node:path";
import { stat } from "node:fs/promises";
import {
AGENT_LIFECYCLE_STATUSES,
type AgentLifecycleStatus,
} from "../../shared/agent-lifecycle.js";
import type { Logger } from "pino";
import { z } from "zod";
import type {
AgentCapabilityFlags,
@@ -182,6 +184,7 @@ const BUSY_STATUSES: AgentLifecycleStatus[] = [
"initializing",
"running",
];
const AgentIdSchema = z.string().uuid();
function isAgentBusy(status: AgentLifecycleStatus): boolean {
return BUSY_STATUSES.includes(status);
@@ -201,6 +204,14 @@ function createAbortError(
return Object.assign(new Error(message), { name: "AbortError" });
}
function validateAgentId(agentId: string, source: string): string {
const result = AgentIdSchema.safeParse(agentId);
if (!result.success) {
throw new Error(`${source}: agentId must be a UUID`);
}
return result.data;
}
export class AgentManager {
private readonly clients = new Map<AgentProvider, AgentClient>();
private readonly agents = new Map<string, ActiveManagedAgent>();
@@ -238,9 +249,13 @@ export class AgentManager {
}
subscribe(callback: AgentSubscriber, options?: SubscribeOptions): () => void {
const targetAgentId =
options?.agentId == null
? null
: validateAgentId(options.agentId, "subscribe");
const record: SubscriptionRecord = {
callback,
agentId: options?.agentId ?? null,
agentId: targetAgentId,
};
this.subscribers.add(record);
@@ -334,7 +349,10 @@ export class AgentManager {
options?: { labels?: Record<string, string> }
): Promise<ManagedAgent> {
// Generate agent ID early so we can use it in MCP config
const resolvedAgentId = agentId ?? this.idFactory();
const resolvedAgentId = validateAgentId(
agentId ?? this.idFactory(),
"createAgent"
);
const normalizedConfig = await this.normalizeConfig(config, {
labels: options?.labels,
agentId: resolvedAgentId,
@@ -364,6 +382,10 @@ export class AgentManager {
labels?: Record<string, string>;
}
): Promise<ManagedAgent> {
const resolvedAgentId = validateAgentId(
agentId ?? this.idFactory(),
"resumeAgent"
);
const metadata = (handle.metadata ?? {}) as Partial<AgentSessionConfig>;
const mergedConfig = {
...metadata,
@@ -380,7 +402,7 @@ export class AgentManager {
return this.registerSession(
session,
normalizedConfig,
agentId ?? this.idFactory(),
resolvedAgentId,
options
);
}
@@ -981,13 +1003,14 @@ export class AgentManager {
labels?: Record<string, string>;
}
): Promise<ManagedAgent> {
if (this.agents.has(agentId)) {
throw new Error(`Agent with id ${agentId} already exists`);
const resolvedAgentId = validateAgentId(agentId, "registerSession");
if (this.agents.has(resolvedAgentId)) {
throw new Error(`Agent with id ${resolvedAgentId} already exists`);
}
const now = new Date();
const managed = {
id: agentId,
id: resolvedAgentId,
provider: config.provider,
cwd: config.cwd,
session,
@@ -1010,9 +1033,9 @@ export class AgentManager {
labels: options?.labels ?? {},
} as ActiveManagedAgent;
this.agents.set(agentId, managed);
this.agents.set(resolvedAgentId, managed);
// Initialize previousStatus to track transitions
this.previousStatuses.set(agentId, managed.lifecycle);
this.previousStatuses.set(resolvedAgentId, managed.lifecycle);
await this.refreshRuntimeInfo(managed);
await this.persistSnapshot(managed, {
title: config.title ?? null,
@@ -1319,6 +1342,20 @@ export class AgentManager {
// Always resolve cwd to absolute path for consistent history file lookup
if (normalized.cwd) {
normalized.cwd = resolve(normalized.cwd);
try {
const cwdStats = await stat(normalized.cwd);
if (!cwdStats.isDirectory()) {
throw new Error(`Working directory is not a directory: ${normalized.cwd}`);
}
} catch (error) {
if (error instanceof Error && "code" in error && (error as NodeJS.ErrnoException).code === "ENOENT") {
throw new Error(`Working directory does not exist: ${normalized.cwd}`);
}
if (error instanceof Error) {
throw error;
}
throw new Error(`Failed to access working directory: ${normalized.cwd}`);
}
}
if (typeof normalized.model === "string") {
@@ -1338,9 +1375,10 @@ export class AgentManager {
}
private requireAgent(id: string): ActiveManagedAgent {
const agent = this.agents.get(id);
const normalizedId = validateAgentId(id, "requireAgent");
const agent = this.agents.get(normalizedId);
if (!agent) {
throw new Error(`Unknown agent '${id}'`);
throw new Error(`Unknown agent '${normalizedId}'`);
}
return agent;
}

1
packages/server/src/server/agent/agent-mcp.e2e.test.ts
View File

@@ -90,7 +90,6 @@ describe("agent MCP end-to-end (offline)", () => {
mcpDebug: false,
agentClients: createTestAgentClients(),
agentStoragePath: path.join(paseoHome, "agents"),
openrouterApiKey: null,
};
const daemon = await createPaseoDaemon(daemonConfig, pino({ level: "silent" }));

165
packages/server/src/server/agent/mcp-server.test.ts
View File

@@ -1,4 +1,7 @@
import { describe, expect, it, vi } from "vitest";
import { mkdtemp, mkdir, rm } from "node:fs/promises";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { createTestLogger } from "../../test-utils/test-logger.js";
import { createAgentMcpServer } from "./mcp-server.js";
@@ -48,6 +51,7 @@ function createTestDeps(): TestDeps {
describe("create_agent MCP tool", () => {
const logger = createTestLogger();
const existingCwd = process.cwd();
it("requires a concise title no longer than 60 characters", async () => {
const { agentManager, agentStorage } = createTestDeps();
@@ -56,26 +60,59 @@ describe("create_agent MCP tool", () => {
expect(tool).toBeDefined();
const missingTitle = await tool.inputSchema.safeParseAsync({
cwd: "/tmp/repo",
cwd: existingCwd,
initialMode: "default",
initialPrompt: "test",
});
expect(missingTitle.success).toBe(false);
expect(missingTitle.error.issues[0].path).toEqual(["title"]);
const tooLong = await tool.inputSchema.safeParseAsync({
cwd: "/tmp/repo",
cwd: existingCwd,
initialMode: "default",
title: "x".repeat(61),
initialPrompt: "test",
});
expect(tooLong.success).toBe(false);
expect(tooLong.error.issues[0].path).toEqual(["title"]);
const ok = await tool.inputSchema.safeParseAsync({
cwd: "/tmp/repo",
cwd: existingCwd,
initialMode: "default",
title: "Short title",
initialPrompt: "test",
});
expect(ok.success).toBe(true);
});
it("requires initialPrompt", async () => {
const { agentManager, agentStorage } = createTestDeps();
const server = await createAgentMcpServer({ agentManager, agentStorage, logger });
const tool = (server as any)._registeredTools["create_agent"];
const parsed = await tool.inputSchema.safeParseAsync({
cwd: existingCwd,
initialMode: "default",
title: "Short title",
});
expect(ok.success).toBe(true);
expect(parsed.success).toBe(false);
expect(parsed.error.issues.some((issue: { path: string[] }) => issue.path[0] === "initialPrompt")).toBe(true);
});
it("surfaces createAgent validation failures", async () => {
const { agentManager, agentStorage, spies } = createTestDeps();
spies.agentManager.createAgent.mockRejectedValue(
new Error("Working directory does not exist: /path/that/does/not/exist")
);
const server = await createAgentMcpServer({ agentManager, agentStorage, logger });
const tool = (server as any)._registeredTools["create_agent"];
await expect(
tool.callback({
cwd: "/path/that/does/not/exist",
title: "Short title",
initialPrompt: "Do work",
})
).rejects.toThrow("Working directory does not exist");
});
it("passes caller-provided titles directly into createAgent", async () => {
@@ -86,20 +123,24 @@ describe("create_agent MCP tool", () => {
lifecycle: "idle",
currentModeId: null,
availableModes: [],
config: { title: "Fix auth bug" },
} as ManagedAgent);
const server = await createAgentMcpServer({ agentManager, agentStorage, logger });
const tool = (server as any)._registeredTools["create_agent"];
await tool.callback({
cwd: "/tmp/repo",
cwd: existingCwd,
title: " Fix auth bug ",
initialPrompt: "Do work",
});
expect(spies.agentManager.createAgent).toHaveBeenCalledWith(
expect.objectContaining({
cwd: "/tmp/repo",
cwd: existingCwd,
title: "Fix auth bug",
})
}),
undefined,
undefined
);
});
@@ -111,19 +152,127 @@ describe("create_agent MCP tool", () => {
lifecycle: "idle",
currentModeId: null,
availableModes: [],
config: { title: "Fix auth" },
} as ManagedAgent);
const server = await createAgentMcpServer({ agentManager, agentStorage, logger });
const tool = (server as any)._registeredTools["create_agent"];
await tool.callback({
cwd: "/tmp/repo",
cwd: existingCwd,
title: " Fix auth ",
initialPrompt: "Do work",
});
expect(spies.agentManager.createAgent).toHaveBeenCalledWith(
expect.objectContaining({
title: "Fix auth",
}),
undefined,
undefined
);
});
it("allows caller agents to override cwd and applies caller context labels", async () => {
const { agentManager, agentStorage, spies } = createTestDeps();
const baseDir = await mkdtemp(join(tmpdir(), "paseo-mcp-test-"));
const subdir = join(baseDir, "subdir");
await mkdir(subdir, { recursive: true });
spies.agentManager.getAgent.mockReturnValue({
id: "voice-agent",
cwd: baseDir,
provider: "codex",
currentModeId: "full-access",
} as ManagedAgent);
spies.agentManager.createAgent.mockResolvedValue({
id: "child-agent",
cwd: subdir,
lifecycle: "idle",
currentModeId: null,
availableModes: [],
config: { title: "Child" },
} as ManagedAgent);
const server = await createAgentMcpServer({
agentManager,
agentStorage,
callerAgentId: "voice-agent",
resolveCallerContext: () => ({
childAgentDefaultLabels: { ui: "true" },
allowCustomCwd: true,
}),
logger,
});
const tool = (server as any)._registeredTools["create_agent"];
await tool.callback({
cwd: "subdir",
title: "Child",
agentType: "codex",
initialPrompt: "Do work",
});
expect(spies.agentManager.createAgent).toHaveBeenCalledWith(
expect.objectContaining({
cwd: subdir,
}),
undefined,
{ labels: { ui: "true" } }
);
await rm(baseDir, { recursive: true, force: true });
});
});
describe("speak MCP tool", () => {
const logger = createTestLogger();
it("invokes registered speak handler for caller agent", async () => {
const { agentManager, agentStorage } = createTestDeps();
const speak = vi.fn().mockResolvedValue(undefined);
const server = await createAgentMcpServer({
agentManager,
agentStorage,
callerAgentId: "voice-agent-1",
enableVoiceTools: true,
resolveSpeakHandler: () => speak,
logger,
});
const tool = (server as any)._registeredTools["speak"];
expect(tool).toBeDefined();
await tool.callback({ text: "Hello from voice agent." });
expect(speak).toHaveBeenCalledWith(
expect.objectContaining({
text: "Hello from voice agent.",
callerAgentId: "voice-agent-1",
})
);
});
it("fails when no speak handler exists", async () => {
const { agentManager, agentStorage } = createTestDeps();
const server = await createAgentMcpServer({
agentManager,
agentStorage,
callerAgentId: "voice-agent-2",
enableVoiceTools: true,
resolveSpeakHandler: () => null,
logger,
});
const tool = (server as any)._registeredTools["speak"];
await expect(tool.callback({ text: "Hello." })).rejects.toThrow(
"No speak handler registered for caller agent"
);
});
it("does not register speak tool unless voice tools are enabled", async () => {
const { agentManager, agentStorage } = createTestDeps();
const server = await createAgentMcpServer({
agentManager,
agentStorage,
callerAgentId: "agent-no-voice",
logger,
});
const tool = (server as any)._registeredTools["speak"];
expect(tool).toBeUndefined();
});
});

296
packages/server/src/server/agent/mcp-server.ts
View File

@@ -1,9 +1,12 @@
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import { z } from "zod";
import { homedir } from "node:os";
import { resolve } from "node:path";
import { ensureValidJson } from "../json-utils.js";
import type { Logger } from "pino";
import type { RequestHandlerExtra } from "@modelcontextprotocol/sdk/shared/protocol.js";
import type {
ServerNotification,
ServerRequest,
} from "@modelcontextprotocol/sdk/types.js";
import type {
AgentPromptInput,
@@ -28,6 +31,11 @@ import { AgentStorage } from "./agent-storage.js";
import { createWorktree } from "../../utils/worktree.js";
import { WaitForAgentTracker } from "./wait-for-agent-tracker.js";
import { scheduleAgentMetadataGeneration } from "./agent-metadata-generator.js";
import type {
VoiceCallerContext,
VoiceSpeakHandler,
} from "../voice-types.js";
import { expandUserPath, resolvePathFromBase } from "../path-utils.js";
export interface AgentMcpServerOptions {
agentManager: AgentManager;
@@ -38,6 +46,17 @@ export interface AgentMcpServerOptions {
* Used for cwd/mode inheritance when agents spawn child agents.
*/
callerAgentId?: string;
/**
* Optional resolver for session-bound speak handlers.
* Used by hidden voice agents to narrate through daemon-managed TTS.
*/
resolveSpeakHandler?: (
callerAgentId: string
) => VoiceSpeakHandler | null;
resolveCallerContext?: (
callerAgentId: string
) => VoiceCallerContext | null;
enableVoiceTools?: boolean;
logger: Logger;
}
@@ -100,11 +119,25 @@ const AgentStatusEnum = z.enum([
// 50 seconds - surface friendly message before SDK tool timeout (~60s)
const AGENT_WAIT_TIMEOUT_MS = 50000;
function expandPath(path: string): string {
if (path.startsWith("~/") || path === "~") {
return resolve(homedir(), path.slice(2));
type McpToolContext = RequestHandlerExtra<ServerRequest, ServerNotification>;
function resolveChildAgentCwd(params: {
parentCwd: string;
requestedCwd?: string;
lockedCwd?: string;
allowCustomCwd: boolean;
}): string {
const lockedCwd = params.lockedCwd?.trim();
if (lockedCwd) {
return expandUserPath(lockedCwd);
}
return resolve(path);
const requestedCwd = params.requestedCwd?.trim();
if (!requestedCwd || !params.allowCustomCwd) {
return params.parentCwd;
}
return resolvePathFromBase(params.parentCwd, requestedCwd);
}
/**
@@ -252,9 +285,17 @@ async function serializeSnapshotWithMetadata(
export async function createAgentMcpServer(
options: AgentMcpServerOptions
): Promise<McpServer> {
const { agentManager, agentStorage, callerAgentId, logger } = options;
const {
agentManager,
agentStorage,
callerAgentId,
resolveSpeakHandler,
resolveCallerContext,
logger,
} = options;
const childLogger = logger.child({ module: "agent", component: "mcp-server" });
const waitTracker = new WaitForAgentTracker(logger);
const callerContext = callerAgentId ? resolveCallerContext?.(callerAgentId) ?? null : null;
const server = new McpServer({
name: "agent-mcp",
@@ -262,6 +303,12 @@ export async function createAgentMcpServer(
});
const agentToAgentInputSchema = {
cwd: z
.string()
.optional()
.describe(
"Optional working directory. Defaults to the caller agent working directory."
),
title: z
.string()
.trim()
@@ -275,9 +322,10 @@ export async function createAgentMcpServer(
),
initialPrompt: z
.string()
.optional()
.trim()
.min(1, "initialPrompt is required")
.describe(
"Optional task to start immediately after creation (non-blocking)."
"Required first task to run immediately after creation."
),
background: z
.boolean()
@@ -307,9 +355,10 @@ export async function createAgentMcpServer(
),
initialPrompt: z
.string()
.optional()
.trim()
.min(1, "initialPrompt is required")
.describe(
"Optional task to start immediately after creation (non-blocking)."
"Required first task to run immediately after creation."
),
initialMode: z
.string()
@@ -338,6 +387,50 @@ export async function createAgentMcpServer(
const createAgentInputSchema = callerAgentId
? agentToAgentInputSchema
: topLevelInputSchema;
const agentToAgentCreateAgentArgsSchema = z.object(agentToAgentInputSchema);
const topLevelCreateAgentArgsSchema = z.object({
...topLevelInputSchema,
initialMode: topLevelInputSchema.initialMode.optional(),
});
if (options.enableVoiceTools || callerContext?.enableVoiceTools) {
server.registerTool(
"speak",
{
title: "Speak",
description:
"Speak text to the user via daemon-managed voice output. Blocks until playback completes.",
inputSchema: {
text: z
.string()
.trim()
.min(1, "text is required")
.max(4000, "text must be 4000 characters or fewer"),
},
outputSchema: {
ok: z.boolean(),
},
},
async (args, context?: McpToolContext) => {
if (!callerAgentId) {
throw new Error("speak is only available to agent-scoped MCP sessions");
}
const handler = resolveSpeakHandler?.(callerAgentId) ?? null;
if (!handler) {
throw new Error(`No speak handler registered for caller agent '${callerAgentId}'`);
}
await handler({
text: args.text,
callerAgentId,
signal: context?.signal,
});
return {
content: [],
structuredContent: ensureValidJson({ ok: true }),
};
}
);
}
server.registerTool(
"create_agent",
@@ -363,33 +456,32 @@ export async function createAgentMcpServer(
permission: AgentPermissionRequestPayloadSchema.nullable().optional(),
},
},
async (args) => {
const {
agentType,
initialPrompt,
background = false,
title,
} = args as {
cwd?: string;
agentType?: AgentProvider;
initialPrompt?: string;
initialMode?: string;
worktreeName?: string;
background?: boolean;
title: string;
};
async (args: unknown) => {
let provider: AgentProvider;
let initialPrompt: string;
let background = false;
let normalizedTitle: string | null;
let resolvedCwd: string;
let resolvedMode: string | undefined;
if (callerAgentId) {
const callerArgs = agentToAgentCreateAgentArgsSchema.parse(args);
provider = callerArgs.agentType ?? "claude";
initialPrompt = callerArgs.initialPrompt;
background = callerArgs.background ?? false;
normalizedTitle = callerArgs.title.trim();
const parentAgent = agentManager.getAgent(callerAgentId);
if (!parentAgent) {
throw new Error(`Parent agent ${callerAgentId} not found`);
}
resolvedCwd = parentAgent.cwd;
const provider: AgentProvider = agentType ?? "claude";
resolvedCwd = resolveChildAgentCwd({
parentCwd: parentAgent.cwd,
requestedCwd: callerArgs.cwd,
lockedCwd: callerContext?.lockedCwd,
allowCustomCwd: callerContext?.allowCustomCwd ?? true,
});
const parentMode = parentAgent.currentModeId;
if (parentMode) {
resolvedMode = mapModeAcrossProviders(
@@ -399,12 +491,11 @@ export async function createAgentMcpServer(
);
}
} else {
const topLevelArgs = args as unknown as {
cwd: string;
initialMode: string;
worktreeName?: string;
baseBranch?: string;
};
const topLevelArgs = topLevelCreateAgentArgsSchema.parse(args);
provider = topLevelArgs.agentType ?? "claude";
initialPrompt = topLevelArgs.initialPrompt;
background = topLevelArgs.background ?? false;
normalizedTitle = topLevelArgs.title.trim();
const {
cwd,
initialMode,
@@ -412,7 +503,7 @@ export async function createAgentMcpServer(
baseBranch,
} = topLevelArgs;
resolvedCwd = expandPath(cwd);
resolvedCwd = expandUserPath(cwd);
if (worktreeName) {
if (!baseBranch) {
@@ -431,75 +522,78 @@ export async function createAgentMcpServer(
resolvedMode = initialMode;
}
const provider: AgentProvider = agentType ?? "claude";
const normalizedTitle = title?.trim() ?? null;
const snapshot = await agentManager.createAgent({
provider,
cwd: resolvedCwd,
modeId: resolvedMode,
title: normalizedTitle ?? undefined,
const childAgentDefaultLabels =
callerAgentId && callerContext?.childAgentDefaultLabels
? callerContext.childAgentDefaultLabels
: undefined;
const snapshot = await agentManager.createAgent(
{
provider,
cwd: resolvedCwd,
modeId: resolvedMode,
title: normalizedTitle ?? undefined,
},
undefined,
childAgentDefaultLabels ? { labels: childAgentDefaultLabels } : undefined
);
const trimmedPrompt = initialPrompt.trim();
scheduleAgentMetadataGeneration({
agentManager,
agentId: snapshot.id,
cwd: snapshot.cwd,
initialPrompt: trimmedPrompt,
explicitTitle: snapshot.config.title,
paseoHome: options.paseoHome,
logger: childLogger,
});
const trimmedPrompt = initialPrompt?.trim();
if (trimmedPrompt) {
scheduleAgentMetadataGeneration({
agentManager,
agentId: snapshot.id,
cwd: snapshot.cwd,
initialPrompt: trimmedPrompt,
explicitTitle: snapshot.config.title,
paseoHome: options.paseoHome,
logger: childLogger,
});
try {
agentManager.recordUserMessage(snapshot.id, trimmedPrompt);
} catch (error) {
childLogger.error(
{ err: error, agentId: snapshot.id },
"Failed to record initial prompt"
);
}
try {
startAgentRun(agentManager, snapshot.id, trimmedPrompt, childLogger);
// If not running in background, wait for completion
if (!background) {
const result = await waitForAgentWithTimeout(
agentManager,
snapshot.id,
{ waitForActive: true }
);
const responseData = {
agentId: snapshot.id,
type: provider,
status: result.status,
cwd: snapshot.cwd,
currentModeId: snapshot.currentModeId,
availableModes: snapshot.availableModes,
lastMessage: result.lastMessage,
permission: sanitizePermissionRequest(result.permission),
};
const validJson = ensureValidJson(responseData);
const response = {
content: [],
structuredContent: validJson,
};
return response;
}
} catch (error) {
childLogger.error(
{ err: error, agentId: snapshot.id },
"Failed to run initial prompt"
);
}
} else {
try {
agentManager.recordUserMessage(snapshot.id, trimmedPrompt);
} catch (error) {
childLogger.error(
{ err: error, agentId: snapshot.id },
"Failed to record initial prompt"
);
}
// Return immediately if background=true or no initialPrompt
try {
startAgentRun(agentManager, snapshot.id, trimmedPrompt, childLogger);
// If not running in background, wait for completion
if (!background) {
const result = await waitForAgentWithTimeout(
agentManager,
snapshot.id,
{ waitForActive: true }
);
const responseData = {
agentId: snapshot.id,
type: provider,
status: result.status,
cwd: snapshot.cwd,
currentModeId: snapshot.currentModeId,
availableModes: snapshot.availableModes,
lastMessage: result.lastMessage,
permission: sanitizePermissionRequest(result.permission),
};
const validJson = ensureValidJson(responseData);
const response = {
content: [],
structuredContent: validJson,
};
return response;
}
} catch (error) {
childLogger.error(
{ err: error, agentId: snapshot.id },
"Failed to run initial prompt"
);
}
// Return immediately if background=true
const response = {
content: [],
structuredContent: ensureValidJson({

19
packages/server/src/server/agent/provider-manifest.ts
View File

@@ -7,6 +7,11 @@ export interface AgentProviderDefinition {
description: string;
defaultModeId: string | null;
modes: AgentMode[];
voice?: {
enabled: boolean;
defaultModeId: string;
defaultModel?: string;
};
}
const CLAUDE_MODES: AgentMode[] = [
@@ -68,6 +73,11 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [
"Anthropic's multi-tool assistant with MCP support, streaming, and deep reasoning",
defaultModeId: "default",
modes: CLAUDE_MODES,
voice: {
enabled: true,
defaultModeId: "default",
defaultModel: "haiku",
},
},
{
id: "codex",
@@ -76,6 +86,11 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [
"OpenAI's Codex workspace agent with sandbox controls and optional network access",
defaultModeId: "auto",
modes: CODEX_MODES,
voice: {
enabled: true,
defaultModeId: "read-only",
defaultModel: "gpt-5.2-mini",
},
},
{
id: "opencode",
@@ -84,6 +99,10 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [
"Open-source coding assistant with multi-provider model support",
defaultModeId: "default",
modes: OPENCODE_MODES,
voice: {
enabled: true,
defaultModeId: "default",
},
},
];

20
packages/server/src/server/bootstrap.smoke.test.ts
View File

@@ -10,7 +10,16 @@ import { createTestAgentClients } from "./test-utils/fake-agent-client.js";
describe("paseo daemon bootstrap", () => {
test("starts and serves health endpoint", async () => {
const daemonHandle = await createTestPaseoDaemon();
const daemonHandle = await createTestPaseoDaemon({
openai: { apiKey: "test-openai-api-key" },
speech: {
providers: {
dictationStt: { provider: "openai", explicit: true },
voiceStt: { provider: "openai", explicit: true },
voiceTts: { provider: "openai", explicit: true },
},
},
});
try {
const response = await fetch(
`http://127.0.0.1:${daemonHandle.port}/api/health`,
@@ -49,11 +58,12 @@ describe("paseo daemon bootstrap", () => {
appBaseUrl: "https://app.paseo.sh",
openai: undefined,
speech: {
dictationSttProvider: "openai",
voiceSttProvider: "openai",
voiceTtsProvider: "openai",
providers: {
dictationStt: { provider: "openai", explicit: true },
voiceStt: { provider: "openai", explicit: true },
voiceTts: { provider: "openai", explicit: true },
},
},
openrouterApiKey: null,
};
try {

625
packages/server/src/server/bootstrap.ts
View File

@@ -3,6 +3,7 @@ import { createServer as createHTTPServer } from "http";
import { createReadStream, unlinkSync, existsSync } from "fs";
import { stat } from "fs/promises";
import { randomUUID } from "node:crypto";
import path from "node:path";
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
@@ -40,20 +41,10 @@ function parseListenString(listen: string): ListenTarget {
import { VoiceAssistantWebSocketServer } from "./websocket-server.js";
import { DownloadTokenStore } from "./file-download/token-store.js";
import { OpenAISTT, type STTConfig } from "./speech/providers/openai/stt.js";
import { OpenAITTS, type TTSConfig } from "./speech/providers/openai/tts.js";
import { OpenAIRealtimeTranscriptionSession } from "./speech/providers/openai/realtime-transcription-session.js";
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
import { SherpaOnlineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-online-recognizer.js";
import { SherpaOfflineRecognizerEngine } from "./speech/providers/local/sherpa/sherpa-offline-recognizer.js";
import { SherpaOnnxSTT } from "./speech/providers/local/sherpa/sherpa-stt.js";
import { SherpaOnnxParakeetSTT } from "./speech/providers/local/sherpa/sherpa-parakeet-stt.js";
import { SherpaOnnxTTS } from "./speech/providers/local/sherpa/sherpa-tts.js";
import { SherpaRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-realtime-session.js";
import { SherpaParakeetRealtimeTranscriptionSession } from "./speech/providers/local/sherpa/sherpa-parakeet-realtime-session.js";
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./speech/providers/local/sherpa/model-downloader.js";
import type { SherpaOnnxModelId } from "./speech/providers/local/sherpa/model-catalog.js";
import { PocketTtsOnnxTTS } from "./speech/providers/local/pocket/pocket-tts-onnx.js";
import type { OpenAiSpeechProviderConfig } from "./speech/providers/openai/config.js";
import type { LocalSpeechProviderConfig } from "./speech/providers/local/config.js";
import type { RequestedSpeechProviders } from "./speech/speech-types.js";
import { initializeSpeechRuntime } from "./speech/speech-runtime.js";
import { AgentManager } from "./agent/agent-manager.js";
import { AgentStorage } from "./agent/agent-storage.js";
import { attachAgentStoragePersistence } from "./persistence-hooks.js";
@@ -73,35 +64,50 @@ import type {
AgentClient,
AgentProvider,
} from "./agent/agent-sdk-types.js";
import { AGENT_PROVIDER_DEFINITIONS } from "./agent/provider-manifest.js";
import { acquirePidLock, releasePidLock } from "./pid-lock.js";
import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js";
import { createVoiceMcpBridgeSocketServer, type VoiceMcpBridgeSocketServer } from "./voice-mcp-bridge.js";
type AgentMcpTransportMap = Map<string, StreamableHTTPServerTransport>;
export type PaseoOpenAIConfig = {
apiKey?: string;
stt?: Partial<STTConfig> & { apiKey?: string };
tts?: Partial<TTSConfig> & { apiKey?: string };
};
function resolveVoiceMcpBridgeCommand(logger: Logger): { command: string; baseArgs: string[] } {
const explicit = process.env.PASEO_BIN_PATH?.trim();
if (explicit) {
const resolved = { command: explicit, baseArgs: ["__paseo_voice_mcp_bridge"] };
logger.info({ source: "PASEO_BIN_PATH", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
return resolved;
}
export type PaseoSherpaOnnxConfig = {
modelsDir: string;
autoDownload?: boolean;
stt?: {
preset?: string;
};
tts?: {
preset?: string;
speakerId?: number;
speed?: number;
};
};
const argv1 = process.argv[1]?.trim();
if (!argv1) {
logger.warn("Could not resolve argv[1] for voice MCP bridge; falling back to 'paseo'");
const resolved = { command: "paseo", baseArgs: ["__paseo_voice_mcp_bridge"] };
logger.info({ source: "fallback", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
return resolved;
}
const base = path.basename(argv1).toLowerCase();
if (base.includes("tsx") && process.argv[2]) {
const resolved = {
command: process.execPath,
baseArgs: [argv1, process.argv[2], "__paseo_voice_mcp_bridge"],
};
logger.info({ source: "tsx", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
return resolved;
}
const resolved = { command: argv1, baseArgs: ["__paseo_voice_mcp_bridge"] };
logger.info({ source: "argv", command: resolved.command, baseArgs: resolved.baseArgs }, "Resolved voice MCP bridge command");
return resolved;
}
export type PaseoOpenAIConfig = OpenAiSpeechProviderConfig;
export type PaseoLocalSpeechConfig = LocalSpeechProviderConfig;
export type PaseoSpeechConfig = {
dictationSttProvider?: "openai" | "local";
voiceSttProvider?: "openai" | "local";
voiceTtsProvider?: "openai" | "local";
sherpaOnnx?: PaseoSherpaOnnxConfig;
providers: RequestedSpeechProviders;
local?: PaseoLocalSpeechConfig;
};
export type PaseoDaemonConfig = {
@@ -120,7 +126,8 @@ export type PaseoDaemonConfig = {
appBaseUrl?: string;
openai?: PaseoOpenAIConfig;
speech?: PaseoSpeechConfig;
openrouterApiKey?: string | null;
voiceLlmProvider?: AgentProvider | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
dictationFinalTimeoutMs?: number;
downloadTokenTtlMs?: number;
@@ -282,12 +289,110 @@ export async function createPaseoDaemon(
`Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand`
);
const requestedVoiceLlmProvider = config.voiceLlmProvider ?? null;
const voiceLlmProviderExplicit = config.voiceLlmProviderExplicit ?? false;
const voiceEnabledProviders = AGENT_PROVIDER_DEFINITIONS
.filter((definition) => definition.voice?.enabled)
.map((definition) => definition.id as AgentProvider);
logger.info(
{
requestedVoiceLlmProvider,
voiceLlmProviderExplicit,
voiceEnabledProviders,
},
"Voice LLM provider reconciliation started"
);
const providerClients = createAllClients(logger);
Object.assign(providerClients, config.agentClients);
const voiceLlmAvailability = Object.fromEntries(
voiceEnabledProviders.map((provider) => [provider, false])
) as Record<AgentProvider, boolean>;
for (const provider of voiceEnabledProviders) {
try {
voiceLlmAvailability[provider] = await providerClients[provider].isAvailable();
} catch (error) {
logger.warn({ err: error, provider }, "Voice LLM provider availability check failed");
voiceLlmAvailability[provider] = false;
}
}
let resolvedVoiceLlmProvider: AgentProvider | null = null;
if (requestedVoiceLlmProvider) {
if (!voiceEnabledProviders.includes(requestedVoiceLlmProvider)) {
logger.error(
{ provider: requestedVoiceLlmProvider, voiceEnabledProviders },
"Configured voice LLM provider does not support voice mode"
);
throw new Error(
`Configured voice LLM provider '${requestedVoiceLlmProvider}' does not support voice mode`
);
}
if (!voiceLlmAvailability[requestedVoiceLlmProvider]) {
logger.error(
{ provider: requestedVoiceLlmProvider, voiceLlmAvailability },
"Configured voice LLM provider is unavailable"
);
throw new Error(`Configured voice LLM provider '${requestedVoiceLlmProvider}' is unavailable`);
}
resolvedVoiceLlmProvider = requestedVoiceLlmProvider;
} else {
resolvedVoiceLlmProvider =
voiceEnabledProviders.find((provider) => voiceLlmAvailability[provider]) ?? null;
}
let resolvedVoiceLlmModeId: string | null = null;
let resolvedVoiceLlmModel: string | null = null;
if (!resolvedVoiceLlmProvider) {
if (voiceLlmProviderExplicit) {
logger.error(
{ requestedVoiceLlmProvider, voiceLlmAvailability },
"No voice LLM provider available"
);
throw new Error("No voice LLM provider available");
}
logger.warn(
{ requestedVoiceLlmProvider, voiceLlmAvailability },
"No default voice LLM provider available; voice mode will be disabled until a provider is configured"
);
} else {
const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find(
(definition) => definition.id === resolvedVoiceLlmProvider
);
if (!resolvedVoiceProviderDefinition?.voice?.enabled) {
throw new Error(
`Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry`
);
}
resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId;
resolvedVoiceLlmModel =
config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null;
}
logger.info(
{
requestedVoiceLlmProvider,
voiceLlmProviderExplicit,
resolvedVoiceLlmProvider,
resolvedVoiceLlmModeId,
resolvedVoiceLlmModel,
voiceLlmAvailability,
},
"Voice LLM provider reconciliation completed"
);
let wsServer: VoiceAssistantWebSocketServer | null = null;
let voiceMcpBridgeServer: VoiceMcpBridgeSocketServer | null = null;
// Create in-memory transport for Session's Agent MCP client (voice assistant tools)
const createInMemoryAgentMcpTransport = async (): Promise<InMemoryTransport> => {
const agentMcpServer = await createAgentMcpServer({
agentManager,
agentStorage,
paseoHome: config.paseoHome,
enableVoiceTools: false,
resolveSpeakHandler: (callerAgentId) => wsServer?.resolveVoiceSpeakHandler(callerAgentId) ?? null,
resolveCallerContext: (callerAgentId) => wsServer?.resolveVoiceCallerContext(callerAgentId) ?? null,
logger,
});
@@ -309,6 +414,9 @@ export async function createPaseoDaemon(
agentStorage,
paseoHome: config.paseoHome,
callerAgentId,
enableVoiceTools: false,
resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null,
resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null,
logger,
});
@@ -414,385 +522,37 @@ export async function createPaseoDaemon(
logger.info("Agent MCP HTTP endpoint disabled");
}
let sttService: SpeechToTextProvider | null = null;
let ttsService: TextToSpeechProvider | null = null;
let dictationSttService: SpeechToTextProvider | null = null;
let sherpaOnline: SherpaOnlineRecognizerEngine | null = null;
let sherpaOffline: SherpaOfflineRecognizerEngine | null = null;
let sherpaTts: TextToSpeechProvider | null = null;
const openaiApiKey = config.openai?.apiKey;
const speechConfig = config.speech ?? null;
const sherpaConfig = speechConfig?.sherpaOnnx ?? null;
const voiceSttProvider = speechConfig?.voiceSttProvider ?? "local";
const voiceTtsProvider = speechConfig?.voiceTtsProvider ?? "local";
const dictationSttProvider = speechConfig?.dictationSttProvider ?? "local";
const wantsLocalDictation = dictationSttProvider === "local";
const wantsLocalVoiceStt = voiceSttProvider === "local";
const wantsLocalVoiceTts = voiceTtsProvider === "local";
const openaiSttApiKey = config.openai?.stt?.apiKey ?? openaiApiKey;
const openaiTtsApiKey = config.openai?.tts?.apiKey ?? openaiApiKey;
const openaiDictationApiKey = openaiApiKey;
const missingOpenAiCredentialsFor: string[] = [];
if (voiceSttProvider === "openai" && !openaiSttApiKey) {
missingOpenAiCredentialsFor.push("voice.stt");
}
if (voiceTtsProvider === "openai" && !openaiTtsApiKey) {
missingOpenAiCredentialsFor.push("voice.tts");
}
if (dictationSttProvider === "openai" && !openaiDictationApiKey) {
missingOpenAiCredentialsFor.push("dictation.stt");
}
if (missingOpenAiCredentialsFor.length > 0) {
logger.error(
{
requestedProviders: {
dictationStt: dictationSttProvider,
voiceStt: voiceSttProvider,
voiceTts: voiceTtsProvider,
},
missingOpenAiCredentialsFor,
},
"Invalid speech configuration: OpenAI provider selected but credentials are missing"
);
throw new Error(
`Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}`
);
}
logger.info(
{
requestedProviders: {
dictationStt: dictationSttProvider,
voiceStt: voiceSttProvider,
voiceTts: voiceTtsProvider,
},
availability: {
openai: {
stt: Boolean(openaiSttApiKey),
tts: Boolean(openaiTtsApiKey),
dictationStt: Boolean(openaiDictationApiKey),
},
local: {
configured: Boolean(sherpaConfig),
modelsDir: sherpaConfig?.modelsDir ?? null,
autoDownload: sherpaConfig?.autoDownload ?? null,
},
},
},
"Speech provider reconciliation started"
);
if ((wantsLocalDictation || wantsLocalVoiceStt || wantsLocalVoiceTts) && sherpaConfig) {
const autoDownload = sherpaConfig.autoDownload ?? (process.env.VITEST ? false : true);
let sttPreset = (sherpaConfig.stt?.preset ?? "parakeet-tdt-0.6b-v3-int8").trim();
if (
sttPreset !== "zipformer-bilingual-zh-en-2023-02-20" &&
sttPreset !== "paraformer-bilingual-zh-en" &&
sttPreset !== "parakeet-tdt-0.6b-v3-int8"
) {
throw new Error(`Unknown local STT preset: ${sttPreset}`);
}
let ttsPreset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim();
if (
ttsPreset !== "kitten-nano-en-v0_1-fp16" &&
ttsPreset !== "kokoro-en-v0_19" &&
ttsPreset !== "pocket-tts-onnx-int8"
) {
throw new Error(`Unknown local TTS preset: ${ttsPreset}`);
}
const modelIds: SherpaOnnxModelId[] = [];
if (wantsLocalDictation || wantsLocalVoiceStt) {
modelIds.push(sttPreset as SherpaOnnxModelId);
}
if (wantsLocalVoiceTts) {
modelIds.push(ttsPreset as SherpaOnnxModelId);
}
try {
logger.info(
{
modelsDir: sherpaConfig.modelsDir,
modelIds,
autoDownload,
},
"Ensuring local speech models"
);
await ensureSherpaOnnxModels({
modelsDir: sherpaConfig.modelsDir,
modelIds,
autoDownload,
const voiceMcpSocketPath = path.join(config.paseoHome, "runtime", "voice-mcp.sock");
const voiceMcpBridgeCommand = resolveVoiceMcpBridgeCommand(logger);
voiceMcpBridgeServer = createVoiceMcpBridgeSocketServer({
socketPath: voiceMcpSocketPath,
logger,
createAgentMcpServerForCaller: async (callerAgentId) => {
return createAgentMcpServer({
agentManager,
agentStorage,
paseoHome: config.paseoHome,
callerAgentId,
enableVoiceTools: false,
resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null,
resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null,
logger,
});
} catch (err) {
logger.error(
{
err,
modelsDir: sherpaConfig.modelsDir,
autoDownload,
hint:
"Run: npm run dev --workspace=@getpaseo/server, then run: " +
"`tsx packages/server/scripts/download-speech-models.ts --models-dir <DIR> --model <MODEL_ID>`",
},
"Failed to ensure local speech models"
);
}
}
},
});
const {
sttService,
ttsService,
dictationSttService,
cleanup: cleanupSpeechRuntime,
localModelConfig,
} = await initializeSpeechRuntime({
logger,
openaiConfig: config.openai,
speechConfig: config.speech,
});
if ((wantsLocalDictation || wantsLocalVoiceStt) && sherpaConfig) {
let preset = (sherpaConfig.stt?.preset ?? "parakeet-tdt-0.6b-v3-int8").trim();
if (
preset !== "zipformer-bilingual-zh-en-2023-02-20" &&
preset !== "paraformer-bilingual-zh-en" &&
preset !== "parakeet-tdt-0.6b-v3-int8"
) {
throw new Error(`Unknown local STT preset: ${preset}`);
}
const base = sherpaConfig.modelsDir;
try {
if (preset === "parakeet-tdt-0.6b-v3-int8") {
const modelDir = getSherpaOnnxModelDir(base, "parakeet-tdt-0.6b-v3-int8");
sherpaOffline = new SherpaOfflineRecognizerEngine(
{
model: {
kind: "nemo_transducer",
encoder: `${modelDir}/encoder.int8.onnx`,
decoder: `${modelDir}/decoder.int8.onnx`,
joiner: `${modelDir}/joiner.int8.onnx`,
tokens: `${modelDir}/tokens.txt`,
},
numThreads: 2,
debug: 0,
},
logger
);
} else {
const model =
preset === "paraformer-bilingual-zh-en"
? {
kind: "paraformer" as const,
encoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/encoder.int8.onnx`,
decoder: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/decoder.int8.onnx`,
tokens: `${base}/sherpa-onnx-streaming-paraformer-bilingual-zh-en/tokens.txt`,
}
: {
kind: "transducer" as const,
encoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx`,
decoder: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx`,
joiner: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx`,
tokens: `${base}/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt`,
modelType: "zipformer",
};
sherpaOnline = new SherpaOnlineRecognizerEngine(
{
model,
numThreads: 1,
debug: 0,
},
logger
);
}
} catch (err) {
logger.error(
{
err,
modelsDir: sherpaConfig.modelsDir,
preset,
hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`,
},
"Failed to initialize Sherpa STT (models missing or invalid)"
);
sherpaOnline = null;
sherpaOffline = null;
}
} else if (wantsLocalDictation || wantsLocalVoiceStt) {
logger.warn(
{ configured: Boolean(sherpaConfig) },
"Local STT selected but local provider config is missing; STT will be unavailable"
);
}
if (wantsLocalVoiceTts && sherpaConfig) {
let preset = (sherpaConfig.tts?.preset ?? "pocket-tts-onnx-int8").trim();
if (
preset !== "kitten-nano-en-v0_1-fp16" &&
preset !== "kokoro-en-v0_19" &&
preset !== "pocket-tts-onnx-int8"
) {
throw new Error(`Unknown local TTS preset: ${preset}`);
}
try {
if (preset === "pocket-tts-onnx-int8") {
const modelDir = getSherpaOnnxModelDir(sherpaConfig.modelsDir, "pocket-tts-onnx-int8");
sherpaTts = await PocketTtsOnnxTTS.create(
{
modelDir,
precision: "int8",
targetChunkMs: 50,
},
logger
);
} else {
const modelDir = `${sherpaConfig.modelsDir}/${preset}`;
sherpaTts = new SherpaOnnxTTS(
{
preset: preset as any,
modelDir,
speakerId: sherpaConfig.tts?.speakerId,
speed: sherpaConfig.tts?.speed,
},
logger
);
}
} catch (err) {
logger.error(
{
err,
preset,
hint: `Run: tsx packages/server/scripts/download-speech-models.ts --models-dir '${sherpaConfig.modelsDir}' --model '${preset}'`,
},
"Failed to initialize Sherpa TTS (models missing or invalid)"
);
sherpaTts = null;
}
} else if (wantsLocalVoiceTts) {
logger.warn(
{ configured: Boolean(sherpaConfig) },
"Local TTS selected but local provider config is missing; TTS will be unavailable"
);
}
if (wantsLocalVoiceStt && sherpaOffline) {
sttService = new SherpaOnnxParakeetSTT({ engine: sherpaOffline }, logger);
} else if (wantsLocalVoiceStt && sherpaOnline) {
sttService = new SherpaOnnxSTT({ engine: sherpaOnline }, logger);
}
if (wantsLocalVoiceTts && sherpaTts) {
ttsService = sherpaTts;
}
if (wantsLocalDictation && sherpaOnline) {
dictationSttService = {
id: "local",
createSession: () => new SherpaRealtimeTranscriptionSession({ engine: sherpaOnline! }),
};
} else if (wantsLocalDictation && sherpaOffline) {
dictationSttService = {
id: "local",
createSession: () =>
new SherpaParakeetRealtimeTranscriptionSession({ engine: sherpaOffline! }),
};
}
const needsOpenAiStt = !sttService && voiceSttProvider === "openai";
const needsOpenAiTts = !ttsService && voiceTtsProvider === "openai";
const needsOpenAiDictation = dictationSttProvider === "openai";
if (
(needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) &&
(openaiSttApiKey || openaiTtsApiKey || openaiDictationApiKey)
) {
logger.info("OpenAI speech provider initialized");
if (needsOpenAiStt) {
if (openaiSttApiKey) {
const { apiKey: _sttApiKey, ...sttConfig } = config.openai?.stt ?? {};
sttService = new OpenAISTT(
{
apiKey: openaiSttApiKey,
...sttConfig,
},
logger
);
}
}
if (needsOpenAiTts) {
if (openaiTtsApiKey) {
const { apiKey: _ttsApiKey, ...ttsConfig } = config.openai?.tts ?? {};
ttsService = new OpenAITTS(
{
apiKey: openaiTtsApiKey,
voice: "alloy",
model: "tts-1",
responseFormat: "pcm",
...ttsConfig,
},
logger
);
}
}
if (needsOpenAiDictation) {
const transcriptionModel =
process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe";
dictationSttService = {
id: "openai",
createSession: ({ logger: sessionLogger, language, prompt }) =>
new OpenAIRealtimeTranscriptionSession({
apiKey: openaiDictationApiKey!,
logger: sessionLogger,
transcriptionModel,
...(language ? { language } : {}),
...(prompt ? { prompt } : {}),
turnDetection: null,
}),
};
}
} else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) {
logger.warn("OPENAI_API_KEY not set - OpenAI STT/TTS/dictation is unavailable");
}
const effectiveProviders = {
dictationStt: dictationSttService?.id ?? "unavailable",
voiceStt: sttService?.id ?? "unavailable",
voiceTts: !ttsService ? "unavailable" : ttsService === sherpaTts ? "local" : "openai",
};
const unavailableFeatures = [
!dictationSttService ? "dictation.stt" : null,
!sttService ? "voice.stt" : null,
!ttsService ? "voice.tts" : null,
].filter((feature): feature is string => feature !== null);
if (unavailableFeatures.length > 0) {
logger.error(
{
requestedProviders: {
dictationStt: dictationSttProvider,
voiceStt: voiceSttProvider,
voiceTts: voiceTtsProvider,
},
effectiveProviders,
unavailableFeatures,
},
"Speech provider reconciliation failed: configured features are unavailable"
);
throw new Error(
`Configured speech features unavailable: ${unavailableFeatures.join(", ")}`
);
} else {
logger.info(
{
effectiveProviders,
},
"Speech provider reconciliation completed"
);
}
const wsServer = new VoiceAssistantWebSocketServer(
wsServer = new VoiceAssistantWebSocketServer(
httpServer,
logger,
serverId,
@@ -805,12 +565,26 @@ export async function createPaseoDaemon(
{ stt: sttService, tts: ttsService },
terminalManager,
{
openrouterApiKey: config.openrouterApiKey ?? null,
voiceLlmModel: config.voiceLlmModel ?? null,
voiceLlmProvider: resolvedVoiceLlmProvider,
voiceLlmModeId: resolvedVoiceLlmModeId,
voiceLlmProviderExplicit,
voiceLlmModel: resolvedVoiceLlmModel,
voiceAgentMcpStdio: {
command: voiceMcpBridgeCommand.command,
baseArgs: [
...voiceMcpBridgeCommand.baseArgs,
"--socket",
voiceMcpSocketPath,
],
env: {
PASEO_HOME: config.paseoHome,
},
},
},
{
finalTimeoutMs: config.dictationFinalTimeoutMs,
stt: dictationSttService,
localModels: localModelConfig ?? undefined,
}
);
@@ -870,7 +644,12 @@ export async function createPaseoDaemon(
relayTransport?.stop().catch(() => undefined);
relayTransport = startRelayTransport({
logger,
attachSocket: (ws) => wsServer.attachExternalSocket(ws),
attachSocket: (ws) => {
if (!wsServer) {
throw new Error("WebSocket server not initialized");
}
return wsServer.attachExternalSocket(ws);
},
relayEndpoint,
serverId,
daemonKeyPair: daemonKeyPair.keyPair,
@@ -896,6 +675,9 @@ export async function createPaseoDaemon(
httpServer.listen(listenTarget.path);
}
});
if (voiceMcpBridgeServer) {
await voiceMcpBridgeServer.start();
}
};
const stop = async () => {
@@ -905,13 +687,14 @@ export async function createPaseoDaemon(
await agentStorage.flush().catch(() => undefined);
await shutdownProviders(logger);
terminalManager.killAll();
if (sherpaTts && typeof (sherpaTts as any).free === "function") {
(sherpaTts as any).free();
}
sherpaOnline?.free();
sherpaOffline?.free();
cleanupSpeechRuntime();
await relayTransport?.stop().catch(() => undefined);
await wsServer.close();
if (wsServer) {
await wsServer.close();
}
if (voiceMcpBridgeServer) {
await voiceMcpBridgeServer.stop().catch(() => undefined);
}
await new Promise<void>((resolve) => {
httpServer.close(() => resolve());
});

198
packages/server/src/server/config.ts
View File

@@ -1,9 +1,11 @@
import path from "node:path";
import { z } from "zod";
import type { PaseoDaemonConfig } from "./bootstrap.js";
import type { STTConfig } from "./speech/providers/openai/stt.js";
import type { TTSConfig } from "./speech/providers/openai/tts.js";
import { loadPersistedConfig } from "./persisted-config.js";
import type { AgentProvider } from "./agent/agent-sdk-types.js";
import { AgentProviderSchema } from "./agent/provider-manifest.js";
import { resolveSpeechConfig } from "./speech/speech-config-resolver.js";
import {
mergeAllowedHosts,
parseAllowedHostsEnv,
@@ -13,7 +15,6 @@ import {
const DEFAULT_PORT = 6767;
const DEFAULT_RELAY_ENDPOINT = "relay.paseo.sh:443";
const DEFAULT_APP_BASE_URL = "https://app.paseo.sh";
function getDefaultListen(): string {
// Main HTTP server defaults to TCP
return `127.0.0.1:${DEFAULT_PORT}`;
@@ -26,97 +27,14 @@ export type CliConfigOverrides = Partial<{
allowedHosts: AllowedHostsConfig;
}>;
function parseOpenAIConfig(
env: NodeJS.ProcessEnv,
configApiKey: string | undefined,
config: {
dictationSttModel?: string;
dictationSttConfidenceThreshold?: number;
voiceSttModel?: string;
voiceTtsVoice?: TTSConfig["voice"];
voiceTtsModel?: TTSConfig["model"];
}
) {
const apiKey = env.OPENAI_API_KEY ?? configApiKey;
if (!apiKey) return undefined;
const OptionalVoiceLlmProviderSchema = z
.union([z.string(), z.null(), z.undefined()])
.transform((value): string | null => (typeof value === "string" ? value.trim().toLowerCase() : null))
.pipe(z.union([AgentProviderSchema, z.null()]));
const sttConfidenceThreshold = env.STT_CONFIDENCE_THRESHOLD
? parseFloat(env.STT_CONFIDENCE_THRESHOLD)
: config.dictationSttConfidenceThreshold;
const sttModel = (
env.STT_MODEL ??
config.voiceSttModel ??
config.dictationSttModel
) as STTConfig["model"] | undefined;
const ttsVoice = (env.TTS_VOICE || "alloy") as
| "alloy"
| "echo"
| "fable"
| "onyx"
| "nova"
| "shimmer";
const ttsModel = (env.TTS_MODEL || config.voiceTtsModel || "tts-1") as
| "tts-1"
| "tts-1-hd";
const configuredVoice = config.voiceTtsVoice;
return {
apiKey,
stt: {
apiKey,
confidenceThreshold: sttConfidenceThreshold,
...(sttModel ? { model: sttModel } : {}),
},
tts: {
apiKey,
voice: configuredVoice ?? ttsVoice,
model: ttsModel,
responseFormat: "pcm" as TTSConfig["responseFormat"],
},
};
}
function parseSpeechProviderId(value: unknown): "openai" | "local" | null {
if (typeof value !== "string") {
return null;
}
const normalized = value.trim().toLowerCase();
if (!normalized) {
return null;
}
if (normalized === "openai") return "openai";
if (normalized === "local") return "local";
return null;
}
function normalizeSherpaSttPreset(value: string): string {
const raw = value.trim();
const normalized = raw.toLowerCase();
if (normalized === "zipformer" || normalized === "zipformer-bilingual") {
return "zipformer-bilingual-zh-en-2023-02-20";
}
if (normalized === "paraformer") {
return "paraformer-bilingual-zh-en";
}
if (normalized === "parakeet" || normalized === "parakeet-v3" || normalized === "parakeet-tdt") {
return "parakeet-tdt-0.6b-v3-int8";
}
return raw;
}
function normalizeSherpaTtsPreset(value: string): string {
const raw = value.trim();
const normalized = raw.toLowerCase();
if (normalized === "pocket" || normalized === "pocket-tts") {
return "pocket-tts-onnx-int8";
}
if (normalized === "kitten") {
return "kitten-nano-en-v0_1-fp16";
}
if (normalized === "kokoro") {
return "kokoro-en-v0_19";
}
return raw;
function parseOptionalVoiceLlmProvider(value: unknown): AgentProvider | null {
const parsed = OptionalVoiceLlmProviderSchema.safeParse(value);
return parsed.success ? parsed.data : null;
}
export function loadConfig(
@@ -171,81 +89,19 @@ export function loadConfig(
const appBaseUrl =
env.PASEO_APP_BASE_URL ?? persisted.app?.baseUrl ?? DEFAULT_APP_BASE_URL;
const openai = parseOpenAIConfig(env, persisted.providers?.openai?.apiKey, {
dictationSttModel: persisted.features?.dictation?.stt?.model,
dictationSttConfidenceThreshold:
persisted.features?.dictation?.stt?.confidenceThreshold,
voiceSttModel: persisted.features?.voiceMode?.stt?.model,
voiceTtsModel: persisted.features?.voiceMode?.tts?.model,
voiceTtsVoice: persisted.features?.voiceMode?.tts?.voice,
const { openai, speech } = resolveSpeechConfig({
paseoHome,
env,
persisted,
});
const dictationSttProvider =
parseSpeechProviderId(env.PASEO_DICTATION_STT_PROVIDER) ??
parseSpeechProviderId(persisted.features?.dictation?.stt?.provider) ??
"local";
const voiceSttProvider =
parseSpeechProviderId(env.PASEO_VOICE_STT_PROVIDER) ??
parseSpeechProviderId(persisted.features?.voiceMode?.stt?.provider) ??
"local";
const voiceTtsProvider =
parseSpeechProviderId(env.PASEO_VOICE_TTS_PROVIDER) ??
parseSpeechProviderId(persisted.features?.voiceMode?.tts?.provider) ??
"local";
const shouldConfigureSherpa =
dictationSttProvider === "local" ||
voiceSttProvider === "local" ||
voiceTtsProvider === "local" ||
typeof env.PASEO_SHERPA_ONNX_MODELS_DIR === "string" ||
Boolean(persisted.providers?.sherpaOnnx);
const sherpaModelsDir =
(env.PASEO_SHERPA_ONNX_MODELS_DIR ?? persisted.providers?.sherpaOnnx?.modelsDir)?.trim() ||
path.join(paseoHome, "models", "sherpa-onnx");
const sherpaOnnx = shouldConfigureSherpa
? {
modelsDir: sherpaModelsDir,
autoDownload:
env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD !== undefined
? env.PASEO_SHERPA_ONNX_AUTO_DOWNLOAD === "1"
: persisted.providers?.sherpaOnnx?.autoDownload ??
// In tests we should never hit the network unexpectedly.
Boolean(env.VITEST) === false,
stt: {
preset: normalizeSherpaSttPreset(
(env.PASEO_SHERPA_STT_PRESET ?? persisted.providers?.sherpaOnnx?.stt?.preset)?.trim() ||
(persisted.features?.voiceMode?.stt?.preset ??
persisted.features?.dictation?.stt?.preset)?.trim() ||
"parakeet-tdt-0.6b-v3-int8"
),
},
tts: {
preset: normalizeSherpaTtsPreset(
(env.PASEO_SHERPA_TTS_PRESET ??
persisted.providers?.sherpaOnnx?.tts?.preset ??
persisted.features?.voiceMode?.tts?.preset)?.trim() ||
(env.VITEST ? "kitten-nano-en-v0_1-fp16" : "pocket-tts-onnx-int8")
),
speakerId:
env.PASEO_SHERPA_TTS_SPEAKER_ID !== undefined
? Number.parseInt(env.PASEO_SHERPA_TTS_SPEAKER_ID, 10)
: persisted.providers?.sherpaOnnx?.tts?.speakerId ??
persisted.features?.voiceMode?.tts?.speakerId,
speed:
env.PASEO_SHERPA_TTS_SPEED !== undefined
? Number.parseFloat(env.PASEO_SHERPA_TTS_SPEED)
: persisted.providers?.sherpaOnnx?.tts?.speed ??
persisted.features?.voiceMode?.tts?.speed,
},
}
: undefined;
const openrouterApiKey =
env.OPENROUTER_API_KEY ?? persisted.providers?.openrouter?.apiKey ?? null;
const envVoiceLlmProvider = parseOptionalVoiceLlmProvider(env.PASEO_VOICE_LLM_PROVIDER);
const persistedVoiceLlmProvider = parseOptionalVoiceLlmProvider(
persisted.features?.voiceMode?.llm?.provider
);
const voiceLlmProvider = envVoiceLlmProvider ?? persistedVoiceLlmProvider ?? null;
const voiceLlmProviderExplicit =
envVoiceLlmProvider !== null || persistedVoiceLlmProvider !== null;
const voiceLlmModel = persisted.features?.voiceMode?.llm?.model ?? null;
return {
@@ -265,13 +121,9 @@ export function loadConfig(
relayPublicEndpoint,
appBaseUrl,
openai,
speech: {
dictationSttProvider,
voiceSttProvider,
voiceTtsProvider,
...(sherpaOnnx ? { sherpaOnnx } : {}),
},
openrouterApiKey,
speech,
voiceLlmProvider,
voiceLlmProviderExplicit,
voiceLlmModel,
};
}

141
packages/server/src/server/daemon-client.e2e.test.ts
View File

@@ -3,6 +3,7 @@ import { mkdtempSync, writeFileSync, rmSync, existsSync } from "node:fs";
import { tmpdir, homedir } from "node:os";
import path from "node:path";
import { execSync } from "node:child_process";
import { randomUUID } from "node:crypto";
import {
createDaemonTestContext,
@@ -18,9 +19,9 @@ import {
const openaiApiKey = process.env.OPENAI_API_KEY ?? null;
const sherpaModelsDir =
process.env.PASEO_SHERPA_ONNX_MODELS_DIR ??
path.join(homedir(), ".paseo", "models", "sherpa-onnx");
const localModelsDir =
process.env.PASEO_LOCAL_MODELS_DIR ??
path.join(homedir(), ".paseo", "models", "local-speech");
function hasSherpaZipformerModels(modelsDir: string): boolean {
return (
@@ -49,7 +50,7 @@ function hasSherpaKittenModels(modelsDir: string): boolean {
);
}
const hasLocalSpeech = hasSherpaZipformerModels(sherpaModelsDir) && hasSherpaKittenModels(sherpaModelsDir);
const hasLocalSpeech = hasSherpaZipformerModels(localModelsDir) && hasSherpaKittenModels(localModelsDir);
const hasAnySpeech = hasLocalSpeech || Boolean(openaiApiKey);
const speechTest = hasAnySpeech ? test : test.skip;
@@ -92,23 +93,42 @@ describe("daemon client E2E", () => {
let ctx: DaemonTestContext;
beforeAll(async () => {
const speechConfig =
openaiApiKey
? {
providers: {
dictationStt: { provider: "openai" as const, explicit: true },
voiceStt: { provider: "openai" as const, explicit: true },
voiceTts: { provider: "openai" as const, explicit: true },
},
}
: hasLocalSpeech
? {
providers: {
dictationStt: { provider: "local" as const, explicit: true },
voiceStt: { provider: "local" as const, explicit: true },
voiceTts: { provider: "local" as const, explicit: true },
},
local: {
modelsDir: localModelsDir,
models: {
dictationStt:
process.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
"zipformer-bilingual-zh-en-2023-02-20",
voiceStt:
process.env.PASEO_VOICE_LOCAL_STT_MODEL ??
"zipformer-bilingual-zh-en-2023-02-20",
voiceTts:
process.env.PASEO_VOICE_LOCAL_TTS_MODEL ?? "kitten-nano-en-v0_1-fp16",
},
},
}
: undefined;
ctx = await createDaemonTestContext({
dictationFinalTimeoutMs: 5000,
...(openaiApiKey ? { openai: { apiKey: openaiApiKey } } : {}),
speech: {
dictationSttProvider: "local",
voiceSttProvider: "local",
voiceTtsProvider: "local",
sherpaOnnx: {
modelsDir: sherpaModelsDir,
stt: {
preset: process.env.PASEO_SHERPA_STT_PRESET ?? "zipformer-bilingual-zh-en-2023-02-20",
},
tts: {
preset: process.env.PASEO_SHERPA_TTS_PRESET ?? "kitten-nano-en-v0_1-fp16",
},
},
},
...(speechConfig ? { speech: speechConfig } : {}),
});
}, 60000);
@@ -119,22 +139,18 @@ describe("daemon client E2E", () => {
test("handles session actions", async () => {
expect(ctx.client.isConnected).toBe(true);
const voiceConversationId = `voice-${Date.now()}`;
const loadResult = await ctx.client.loadVoiceConversation(voiceConversationId);
expect(loadResult.voiceConversationId).toBe(voiceConversationId);
expect(typeof loadResult.messageCount).toBe("number");
const agents = await ctx.client.fetchAgents();
expect(Array.isArray(agents)).toBe(true);
const listResult = await ctx.client.listVoiceConversations();
expect(Array.isArray(listResult.conversations)).toBe(true);
const voiceAgents = await ctx.client.fetchAgents({
filter: { labels: { surface: "voice" } },
});
expect(Array.isArray(voiceAgents)).toBe(true);
const missingId = `missing-${Date.now()}`;
const deleteResult = await ctx.client.deleteVoiceConversation(missingId);
expect(deleteResult.voiceConversationId).toBe(missingId);
expect(deleteResult.success).toBe(false);
expect(deleteResult.error).toBeTruthy();
await expect(ctx.client.setVoiceMode(true)).resolves.toBeUndefined();
await expect(ctx.client.setVoiceMode(false)).resolves.toBeUndefined();
await ctx.client.deleteAgent(randomUUID());
}, 30000);
test("emits server_info on websocket connect", async () => {
@@ -160,19 +176,23 @@ describe("daemon client E2E", () => {
await client.close();
}, 15000);
test("matches request IDs for concurrent session requests", async () => {
const firstRequestId = `list-${Date.now()}-a`;
const secondRequestId = `list-${Date.now()}-b`;
test("handles concurrent filtered agent fetch requests", async () => {
const firstRequestId = `fetch-${Date.now()}-a`;
const secondRequestId = `fetch-${Date.now()}-b`;
const [first, second] = await Promise.all([
ctx.client.listVoiceConversations(firstRequestId),
ctx.client.listVoiceConversations(secondRequestId),
ctx.client.fetchAgents({
requestId: firstRequestId,
filter: { labels: { surface: "voice" } },
}),
ctx.client.fetchAgents({
requestId: secondRequestId,
filter: { labels: { surface: "voice" } },
}),
]);
expect(Array.isArray(first.conversations)).toBe(true);
expect(Array.isArray(second.conversations)).toBe(true);
expect(first.requestId).toBe(firstRequestId);
expect(second.requestId).toBe(secondRequestId);
expect(Array.isArray(first)).toBe(true);
expect(Array.isArray(second)).toBe(true);
}, 15000);
test(
@@ -355,7 +375,7 @@ describe("daemon client E2E", () => {
expect(sawAssistantMessage).toBe(true);
expect(sawRawAssistantMessage).toBe(true);
await ctx.client.setVoiceConversation(false);
await ctx.client.setVoiceMode(false);
await ctx.client.abortRequest();
await ctx.client.audioPlayed("audio-1");
@@ -566,16 +586,16 @@ describe("daemon client E2E", () => {
120000
);
test.runIf(Boolean(process.env.OPENROUTER_API_KEY))(
"streams session activity logs and chunks",
speechTest(
"does not process non-voice audio through the voice agent path",
async () => {
await ctx.client.setVoiceConversation(false);
await ctx.client.setVoiceMode(false);
let sawAssistantChunk = false;
let sawTranscriptLog = false;
let sawAssistantChunk = false;
let sawAssistantLog = false;
const completion = waitForSignal(60000, (resolve) => {
const transcriptSeen = waitForSignal(60000, (resolve) => {
const unsubscribeChunk = ctx.client.on("assistant_chunk", (message) => {
if (message.type !== "assistant_chunk") {
return;
@@ -583,9 +603,6 @@ describe("daemon client E2E", () => {
if (message.payload.chunk.length > 0) {
sawAssistantChunk = true;
}
if (sawAssistantChunk && sawTranscriptLog && sawAssistantLog) {
resolve();
}
});
const unsubscribeActivity = ctx.client.on("activity_log", (message) => {
@@ -594,13 +611,11 @@ describe("daemon client E2E", () => {
}
if (message.payload.type === "transcript") {
sawTranscriptLog = true;
resolve();
}
if (message.payload.type === "assistant") {
sawAssistantLog = true;
}
if (sawAssistantChunk && sawTranscriptLog && sawAssistantLog) {
resolve();
}
});
return () => {
@@ -609,16 +624,30 @@ describe("daemon client E2E", () => {
};
});
await ctx.client.sendUserMessage("Say 'hello' and nothing else");
await completion;
const fixturePath = path.resolve(
process.cwd(),
"..",
"app",
"e2e",
"fixtures",
"recording.wav"
);
const wav = await import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
await ctx.client.sendVoiceAudioChunk(wav.toString("base64"), "audio/wav", true);
await transcriptSeen;
await new Promise((resolve) => setTimeout(resolve, 1500));
expect(sawTranscriptLog).toBe(true);
expect(sawAssistantChunk).toBe(false);
expect(sawAssistantLog).toBe(false);
},
120000
90000
);
speechTest(
"voice mode buffers audio until isLast and emits transcription_result",
async () => {
await ctx.client.setVoiceConversation(true, `voice-${Date.now()}`);
await ctx.client.setVoiceMode(true, randomUUID());
const transcription = waitForSignal(30_000, (resolve) => {
const unsubscribe = ctx.client.on("transcription_result", (message) => {
@@ -713,7 +742,7 @@ describe("daemon client E2E", () => {
}
} finally {
await Promise.allSettled([transcription, errorSignal]);
await ctx.client.setVoiceConversation(false);
await ctx.client.setVoiceMode(false);
}
},
90_000

1
packages/server/src/server/daemon-e2e/connection-offer.e2e.test.ts
View File

@@ -186,6 +186,7 @@ describe("ConnectionOfferV2 (daemon E2E)", () => {
PASEO_HOME: tempHome,
PASEO_LISTEN: `0.0.0.0:${port}`,
OPENAI_API_KEY: "",
PASEO_LOCAL_AUTO_DOWNLOAD: "0",
PASEO_LOG_FORMAT: "json",
};

1
packages/server/src/server/exports.ts
View File

@@ -5,6 +5,7 @@ export { resolvePaseoHome } from "./paseo-home.js";
export { createRootLogger, type LogLevel, type LogFormat } from "./logger.js";
export { loadPersistedConfig, type PersistedConfig } from "./persisted-config.js";
export { DaemonClient, type DaemonClientConfig, type ConnectionState, type DaemonEvent } from "../client/daemon-client.js";
export { runVoiceMcpBridgeCli } from "./voice-mcp-bridge.js";
// Agent SDK types for CLI commands
export type {

7
packages/server/src/server/index.ts
View File

@@ -10,8 +10,15 @@ import { resolvePaseoHome } from "./paseo-home.js";
import { createRootLogger } from "./logger.js";
import { loadPersistedConfig } from "./persisted-config.js";
import { PidLockError } from "./pid-lock.js";
import { runVoiceMcpBridgeCli } from "./voice-mcp-bridge.js";
async function main() {
const bridgeArgIndex = process.argv.findIndex((arg) => arg === "__paseo_voice_mcp_bridge");
if (bridgeArgIndex >= 0) {
await runVoiceMcpBridgeCli(process.argv.slice(bridgeArgIndex + 1));
return;
}
let paseoHome: string;
let logger: ReturnType<typeof createRootLogger>;
let config: ReturnType<typeof loadConfig>;

22
packages/server/src/server/path-utils.ts Normal file
View File

@@ -0,0 +1,22 @@
import { homedir } from "node:os";
import { isAbsolute, resolve } from "node:path";
function hasHomePrefix(value: string): boolean {
return value === "~" || value.startsWith("~/");
}
export function expandUserPath(value: string): string {
const trimmed = value.trim();
if (hasHomePrefix(trimmed)) {
return resolve(homedir(), trimmed.slice(2));
}
return resolve(trimmed);
}
export function resolvePathFromBase(baseCwd: string, requestedPath: string): string {
const trimmed = requestedPath.trim();
if (hasHomePrefix(trimmed) || isAbsolute(trimmed)) {
return expandUserPath(trimmed);
}
return resolve(baseCwd, trimmed);
}

41
packages/server/src/server/persisted-config.ts
View File

@@ -1,6 +1,7 @@
import { existsSync, readFileSync, writeFileSync } from "node:fs";
import path from "node:path";
import { z } from "zod";
import { AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js";
const LogConfigSchema = z
.object({
@@ -17,44 +18,25 @@ const ProviderCredentialsSchema = z
})
.strict();
const SherpaOnnxProviderSchema = z
const LocalSpeechProviderSchema = z
.object({
modelsDir: z.string().min(1).optional(),
autoDownload: z.boolean().optional(),
stt: z
.object({
preset: z.string().min(1).optional(),
})
.strict()
.optional(),
tts: z
.object({
preset: z.string().min(1).optional(),
speakerId: z.number().int().optional(),
speed: z.number().optional(),
})
.strict()
.optional(),
})
.strict();
const ProvidersSchema = z
.object({
openai: ProviderCredentialsSchema.optional(),
openrouter: ProviderCredentialsSchema.optional(),
sherpaOnnx: SherpaOnnxProviderSchema.optional(),
local: LocalSpeechProviderSchema.optional(),
})
.strict();
const SpeechProviderIdSchema = z.preprocess(
(value) => {
if (typeof value !== "string") {
return value;
}
return value.trim().toLowerCase();
},
z.enum(["openai", "local"])
);
const SpeechProviderIdSchema = z
.string()
.trim()
.toLowerCase()
.pipe(z.enum(["openai", "local"]));
const FeatureDictationSchema = z
.object({
@@ -62,7 +44,6 @@ const FeatureDictationSchema = z
.object({
provider: SpeechProviderIdSchema.optional(),
model: z.string().min(1).optional(),
preset: z.string().min(1).optional(),
confidenceThreshold: z.number().optional(),
})
.strict()
@@ -74,7 +55,7 @@ const FeatureVoiceModeSchema = z
.object({
llm: z
.object({
provider: z.enum(["openrouter"]).optional(),
provider: z.enum(AGENT_PROVIDER_IDS as [string, ...string[]]).optional(),
model: z.string().min(1).optional(),
})
.strict()
@@ -83,16 +64,14 @@ const FeatureVoiceModeSchema = z
.object({
provider: SpeechProviderIdSchema.optional(),
model: z.string().min(1).optional(),
preset: z.string().min(1).optional(),
})
.strict()
.optional(),
tts: z
.object({
provider: SpeechProviderIdSchema.optional(),
model: z.enum(["tts-1", "tts-1-hd"]).optional(),
model: z.string().min(1).optional(),
voice: z.enum(["alloy", "echo", "fable", "onyx", "nova", "shimmer"]).optional(),
preset: z.string().min(1).optional(),
speakerId: z.number().int().optional(),
speed: z.number().optional(),
})

1061
packages/server/src/server/session.ts
View File

File diff suppressed because it is too large Load Diff

29
packages/server/src/server/session.voice-mcp-config.test.ts Normal file
View File

@@ -0,0 +1,29 @@
import { describe, expect, test } from "vitest";
import { buildVoiceAgentMcpServerConfig } from "./session.js";
describe("voice MCP stdio config", () => {
test("builds stdio MCP config for voice agent", () => {
const config = buildVoiceAgentMcpServerConfig({
callerAgentId: "voice-agent-123",
command: "/usr/local/bin/paseo",
baseArgs: ["__paseo_voice_mcp_bridge", "--socket", "/tmp/paseo-voice.sock"],
env: {
PASEO_HOME: "/tmp/paseo-home",
},
});
expect(config.type).toBe("stdio");
expect(config.command).toBe("/usr/local/bin/paseo");
expect(config.args).toEqual([
"__paseo_voice_mcp_bridge",
"--socket",
"/tmp/paseo-voice.sock",
"--caller-agent-id",
"voice-agent-123",
]);
expect(config.env).toEqual({
PASEO_HOME: "/tmp/paseo-home",
});
});
});

167
packages/server/src/server/speech/providers/local/config.ts Normal file
View File

@@ -0,0 +1,167 @@
import path from "node:path";
import { z } from "zod";
import type { PersistedConfig } from "../../../persisted-config.js";
import type { RequestedSpeechProviders } from "../../speech-types.js";
import {
DEFAULT_LOCAL_STT_MODEL,
DEFAULT_LOCAL_TTS_MODEL,
LocalSttModelIdSchema,
LocalTtsModelIdSchema,
type LocalSpeechModelId,
type LocalSttModelId,
type LocalTtsModelId,
} from "./models.js";
export type LocalSpeechModelConfig = {
dictationStt: LocalSttModelId;
voiceStt: LocalSttModelId;
voiceTts: LocalTtsModelId;
voiceTtsSpeakerId?: number;
voiceTtsSpeed?: number;
};
export type LocalSpeechProviderConfig = {
modelsDir: string;
autoDownload?: boolean;
models: LocalSpeechModelConfig;
};
export type ResolvedLocalSpeechConfig = {
local: LocalSpeechProviderConfig | undefined;
};
export type { LocalSpeechModelId, LocalSttModelId, LocalTtsModelId };
const DEFAULT_LOCAL_MODELS_SUBDIR = path.join("models", "local-speech");
const BooleanStringSchema = z
.string()
.trim()
.toLowerCase()
.pipe(z.enum(["1", "0", "true", "false", "yes", "no"]))
.transform((value) => value === "1" || value === "true" || value === "yes");
const OptionalBooleanFlagSchema = z
.union([z.boolean(), BooleanStringSchema])
.optional();
const NumberLikeSchema = z.union([
z.number(),
z.string().trim().min(1),
]);
const OptionalFiniteNumberSchema = NumberLikeSchema
.pipe(z.coerce.number().finite())
.optional();
const OptionalIntegerSchema = NumberLikeSchema
.pipe(z.coerce.number().int())
.optional();
const LocalSpeechResolutionSchema = z.object({
includeProviderConfig: z.boolean(),
modelsDir: z.string().trim().min(1),
autoDownload: OptionalBooleanFlagSchema.default(true),
dictationLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL),
voiceLocalSttModel: LocalSttModelIdSchema.default(DEFAULT_LOCAL_STT_MODEL),
voiceLocalTtsModel: LocalTtsModelIdSchema.default(DEFAULT_LOCAL_TTS_MODEL),
voiceLocalTtsSpeakerId: OptionalIntegerSchema,
voiceLocalTtsSpeed: OptionalFiniteNumberSchema,
});
function persistedLocalFeatureModel(
provider: RequestedSpeechProviders[keyof RequestedSpeechProviders]["provider"],
model: string | undefined
): string | undefined {
if (provider !== "local") {
return undefined;
}
return model;
}
function shouldIncludeLocalProviderConfig(params: {
providers: RequestedSpeechProviders;
env: NodeJS.ProcessEnv;
persisted: PersistedConfig;
}): boolean {
const localRequestedByFeature =
params.providers.dictationStt.provider === "local" ||
params.providers.voiceStt.provider === "local" ||
params.providers.voiceTts.provider === "local";
return (
localRequestedByFeature ||
params.env.PASEO_LOCAL_MODELS_DIR !== undefined ||
params.persisted.providers?.local !== undefined
);
}
export function resolveLocalSpeechConfig(params: {
paseoHome: string;
env: NodeJS.ProcessEnv;
persisted: PersistedConfig;
providers: RequestedSpeechProviders;
}): ResolvedLocalSpeechConfig {
const includeProviderConfig = shouldIncludeLocalProviderConfig(params);
const parsed = LocalSpeechResolutionSchema.parse({
includeProviderConfig,
modelsDir:
params.env.PASEO_LOCAL_MODELS_DIR ??
params.persisted.providers?.local?.modelsDir ??
path.join(params.paseoHome, DEFAULT_LOCAL_MODELS_SUBDIR),
autoDownload:
params.env.PASEO_LOCAL_AUTO_DOWNLOAD ??
params.persisted.providers?.local?.autoDownload,
dictationLocalSttModel:
params.env.PASEO_DICTATION_LOCAL_STT_MODEL ??
persistedLocalFeatureModel(
params.providers.dictationStt.provider,
params.persisted.features?.dictation?.stt?.model
) ??
DEFAULT_LOCAL_STT_MODEL,
voiceLocalSttModel:
params.env.PASEO_VOICE_LOCAL_STT_MODEL ??
persistedLocalFeatureModel(
params.providers.voiceStt.provider,
params.persisted.features?.voiceMode?.stt?.model
) ??
DEFAULT_LOCAL_STT_MODEL,
voiceLocalTtsModel:
params.env.PASEO_VOICE_LOCAL_TTS_MODEL ??
persistedLocalFeatureModel(
params.providers.voiceTts.provider,
params.persisted.features?.voiceMode?.tts?.model
) ??
DEFAULT_LOCAL_TTS_MODEL,
voiceLocalTtsSpeakerId:
params.env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID ??
params.persisted.features?.voiceMode?.tts?.speakerId,
voiceLocalTtsSpeed:
params.env.PASEO_VOICE_LOCAL_TTS_SPEED ??
params.persisted.features?.voiceMode?.tts?.speed,
});
return {
local:
parsed.includeProviderConfig
? {
modelsDir: parsed.modelsDir,
autoDownload: parsed.autoDownload,
models: {
dictationStt: parsed.dictationLocalSttModel,
voiceStt: parsed.voiceLocalSttModel,
voiceTts: parsed.voiceLocalTtsModel,
...(parsed.voiceLocalTtsSpeakerId !== undefined
? { voiceTtsSpeakerId: parsed.voiceLocalTtsSpeakerId }
: {}),
...(parsed.voiceLocalTtsSpeed !== undefined
? { voiceTtsSpeed: parsed.voiceLocalTtsSpeed }
: {}),
},
}
: undefined,
};
}

48
packages/server/src/server/speech/providers/local/models.ts Normal file
View File

@@ -0,0 +1,48 @@
import {
ensureSherpaOnnxModels,
getSherpaOnnxModelDir,
} from "./sherpa/model-downloader.js";
import {
DEFAULT_LOCAL_STT_MODEL,
DEFAULT_LOCAL_TTS_MODEL,
LocalSttModelIdSchema,
LocalTtsModelIdSchema,
listSherpaOnnxModels,
type LocalSpeechModelId,
type LocalSttModelId,
type LocalTtsModelId,
} from "./sherpa/model-catalog.js";
export {
DEFAULT_LOCAL_STT_MODEL,
DEFAULT_LOCAL_TTS_MODEL,
LocalSttModelIdSchema,
LocalTtsModelIdSchema,
type LocalSpeechModelId,
type LocalSttModelId,
type LocalTtsModelId,
};
export type LocalSpeechModelSpec = ReturnType<typeof listSherpaOnnxModels>[number];
export function listLocalSpeechModels(): LocalSpeechModelSpec[] {
return listSherpaOnnxModels();
}
export function getLocalSpeechModelDir(modelsDir: string, modelId: LocalSpeechModelId): string {
return getSherpaOnnxModelDir(modelsDir, modelId);
}
export async function ensureLocalSpeechModels(options: {
modelsDir: string;
modelIds: LocalSpeechModelId[];
autoDownload?: boolean;
logger: import("pino").Logger;
}): Promise<Record<LocalSpeechModelId, string>> {
return ensureSherpaOnnxModels({
modelsDir: options.modelsDir,
modelIds: options.modelIds,
autoDownload: options.autoDownload ?? true,
logger: options.logger,
});
}

370
packages/server/src/server/speech/providers/local/runtime.ts Normal file
View File

@@ -0,0 +1,370 @@
import type { Logger } from "pino";
import type { PaseoSpeechConfig } from "../../../bootstrap.js";
import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js";
import type { RequestedSpeechProviders } from "../../speech-types.js";
import { PocketTtsOnnxTTS } from "./pocket/pocket-tts-onnx.js";
import {
ensureLocalSpeechModels,
getLocalSpeechModelDir,
DEFAULT_LOCAL_STT_MODEL,
DEFAULT_LOCAL_TTS_MODEL,
LocalSttModelIdSchema,
LocalTtsModelIdSchema,
type LocalSpeechModelId,
type LocalSttModelId,
type LocalTtsModelId,
} from "./models.js";
import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js";
import { SherpaOnlineRecognizerEngine } from "./sherpa/sherpa-online-recognizer.js";
import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js";
import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js";
import { SherpaRealtimeTranscriptionSession } from "./sherpa/sherpa-realtime-session.js";
import { SherpaOnnxSTT } from "./sherpa/sherpa-stt.js";
import { SherpaOnnxTTS } from "./sherpa/sherpa-tts.js";
type LocalSttEngine =
| { kind: "offline"; engine: SherpaOfflineRecognizerEngine }
| { kind: "online"; engine: SherpaOnlineRecognizerEngine };
type ResolvedLocalModels = {
dictationLocalSttModel: LocalSttModelId;
voiceLocalSttModel: LocalSttModelId;
voiceLocalTtsModel: LocalTtsModelId;
};
type LocalSpeechAvailability = {
configured: boolean;
modelsDir: string | null;
autoDownload: boolean | null;
};
export type InitializedLocalSpeech = {
sttService: SpeechToTextProvider | null;
ttsService: TextToSpeechProvider | null;
dictationSttService: SpeechToTextProvider | null;
localVoiceTtsProvider: TextToSpeechProvider | null;
localModelConfig: {
modelsDir: string;
defaultModelIds: LocalSpeechModelId[];
} | null;
availability: LocalSpeechAvailability;
cleanup: () => void;
};
function buildModelDownloadHint(modelId: LocalSpeechModelId): string {
return `Use 'paseo speech download --model ${modelId}' to download this model.`;
}
function resolveConfiguredLocalModels(
speechConfig: PaseoSpeechConfig | null
): ResolvedLocalModels {
return {
dictationLocalSttModel: LocalSttModelIdSchema.parse(
speechConfig?.local?.models.dictationStt ?? DEFAULT_LOCAL_STT_MODEL
),
voiceLocalSttModel: LocalSttModelIdSchema.parse(
speechConfig?.local?.models.voiceStt ?? DEFAULT_LOCAL_STT_MODEL
),
voiceLocalTtsModel: LocalTtsModelIdSchema.parse(
speechConfig?.local?.models.voiceTts ?? DEFAULT_LOCAL_TTS_MODEL
),
};
}
export function getLocalSpeechAvailability(
speechConfig: PaseoSpeechConfig | null
): LocalSpeechAvailability {
const localConfig = speechConfig?.local ?? null;
return {
configured: Boolean(localConfig),
modelsDir: localConfig?.modelsDir ?? null,
autoDownload: localConfig?.autoDownload ?? null,
};
}
function computeRequiredLocalModelIds(params: {
providers: RequestedSpeechProviders;
models: ResolvedLocalModels;
}): LocalSpeechModelId[] {
const ids = new Set<LocalSpeechModelId>();
if (params.providers.dictationStt.provider === "local") {
ids.add(params.models.dictationLocalSttModel);
}
if (params.providers.voiceStt.provider === "local") {
ids.add(params.models.voiceLocalSttModel);
}
if (params.providers.voiceTts.provider === "local") {
ids.add(params.models.voiceLocalTtsModel);
}
return Array.from(ids);
}
async function createLocalSttEngine(params: {
modelId: LocalSttModelId;
modelsDir: string;
logger: Logger;
}): Promise<LocalSttEngine> {
const { modelId, modelsDir, logger } = params;
if (modelId === "parakeet-tdt-0.6b-v3-int8") {
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
return {
kind: "offline",
engine: new SherpaOfflineRecognizerEngine(
{
model: {
kind: "nemo_transducer",
encoder: `${modelDir}/encoder.int8.onnx`,
decoder: `${modelDir}/decoder.int8.onnx`,
joiner: `${modelDir}/joiner.int8.onnx`,
tokens: `${modelDir}/tokens.txt`,
},
numThreads: 2,
debug: 0,
},
logger
),
};
}
if (modelId === "paraformer-bilingual-zh-en") {
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
return {
kind: "online",
engine: new SherpaOnlineRecognizerEngine(
{
model: {
kind: "paraformer",
encoder: `${modelDir}/encoder.int8.onnx`,
decoder: `${modelDir}/decoder.int8.onnx`,
tokens: `${modelDir}/tokens.txt`,
},
numThreads: 1,
debug: 0,
},
logger
),
};
}
if (modelId === "zipformer-bilingual-zh-en-2023-02-20") {
const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
return {
kind: "online",
engine: new SherpaOnlineRecognizerEngine(
{
model: {
kind: "transducer",
encoder: `${modelDir}/encoder-epoch-99-avg-1.onnx`,
decoder: `${modelDir}/decoder-epoch-99-avg-1.onnx`,
joiner: `${modelDir}/joiner-epoch-99-avg-1.onnx`,
tokens: `${modelDir}/tokens.txt`,
modelType: "zipformer",
},
numThreads: 1,
debug: 0,
},
logger
),
};
}
throw new Error(`Unsupported local STT model '${modelId}'`);
}
export async function initializeLocalSpeechServices(params: {
providers: RequestedSpeechProviders;
speechConfig: PaseoSpeechConfig | null;
logger: Logger;
}): Promise<InitializedLocalSpeech> {
const { providers, logger, speechConfig } = params;
const localConfig = speechConfig?.local ?? null;
const localModels = resolveConfiguredLocalModels(speechConfig);
let sttService: SpeechToTextProvider | null = null;
let ttsService: TextToSpeechProvider | null = null;
let dictationSttService: SpeechToTextProvider | null = null;
let localVoiceTtsProvider: TextToSpeechProvider | null = null;
const requiredLocalModelIds = computeRequiredLocalModelIds({
providers,
models: localModels,
});
if (requiredLocalModelIds.length > 0 && localConfig) {
try {
logger.info(
{
modelsDir: localConfig.modelsDir,
modelIds: requiredLocalModelIds,
autoDownload: localConfig.autoDownload ?? true,
},
"Ensuring local speech models"
);
await ensureLocalSpeechModels({
modelsDir: localConfig.modelsDir,
modelIds: requiredLocalModelIds,
autoDownload: localConfig.autoDownload ?? true,
logger,
});
} catch (err) {
logger.error(
{
err,
modelsDir: localConfig.modelsDir,
modelIds: requiredLocalModelIds,
autoDownload: localConfig.autoDownload ?? true,
hint:
"Use `paseo speech models` to inspect status and " +
"`paseo speech download --model <MODEL_ID>` to fetch missing models.",
},
"Failed to ensure local speech models"
);
}
}
const localSttEngines = new Map<LocalSttModelId, LocalSttEngine>();
const getLocalSttEngine = async (
modelId: LocalSttModelId
): Promise<LocalSttEngine | null> => {
const existing = localSttEngines.get(modelId);
if (existing) {
return existing;
}
if (!localConfig) {
return null;
}
try {
const created = await createLocalSttEngine({
modelId,
modelsDir: localConfig.modelsDir,
logger,
});
localSttEngines.set(modelId, created);
return created;
} catch (err) {
logger.error(
{
err,
modelsDir: localConfig.modelsDir,
modelId,
hint: buildModelDownloadHint(modelId),
},
"Failed to initialize local STT engine (models missing or invalid)"
);
return null;
}
};
if (providers.voiceStt.provider === "local") {
if (!localConfig) {
logger.warn(
{ configured: false },
"Local STT selected for voice but local provider config is missing; STT will be unavailable"
);
} else {
const voiceEngine = await getLocalSttEngine(localModels.voiceLocalSttModel);
if (voiceEngine?.kind === "offline") {
sttService = new SherpaOnnxParakeetSTT({ engine: voiceEngine.engine }, logger);
} else if (voiceEngine?.kind === "online") {
sttService = new SherpaOnnxSTT({ engine: voiceEngine.engine }, logger);
}
}
}
if (providers.dictationStt.provider === "local") {
if (!localConfig) {
logger.warn(
{ configured: false },
"Local STT selected for dictation but local provider config is missing; dictation STT will be unavailable"
);
} else {
const dictationEngine = await getLocalSttEngine(localModels.dictationLocalSttModel);
if (dictationEngine?.kind === "offline") {
dictationSttService = {
id: "local",
createSession: () =>
new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
};
} else if (dictationEngine?.kind === "online") {
dictationSttService = {
id: "local",
createSession: () => new SherpaRealtimeTranscriptionSession({ engine: dictationEngine.engine }),
};
}
}
}
if (providers.voiceTts.provider === "local") {
if (!localConfig) {
logger.warn(
{ configured: false },
"Local TTS selected for voice but local provider config is missing; TTS will be unavailable"
);
} else {
try {
if (localModels.voiceLocalTtsModel === "pocket-tts-onnx-int8") {
const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
localVoiceTtsProvider = await PocketTtsOnnxTTS.create(
{
modelDir,
precision: "int8",
targetChunkMs: 50,
},
logger
);
} else {
const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
localVoiceTtsProvider = new SherpaOnnxTTS(
{
preset: localModels.voiceLocalTtsModel,
modelDir,
speakerId: speechConfig?.local?.models.voiceTtsSpeakerId,
speed: speechConfig?.local?.models.voiceTtsSpeed,
},
logger
);
}
ttsService = localVoiceTtsProvider;
} catch (err) {
logger.error(
{
err,
modelsDir: localConfig.modelsDir,
modelId: localModels.voiceLocalTtsModel,
hint: buildModelDownloadHint(localModels.voiceLocalTtsModel),
},
"Failed to initialize local TTS engine (models missing or invalid)"
);
}
}
}
const cleanup = () => {
const maybeFreeable = localVoiceTtsProvider as unknown as { free?: () => void } | null;
if (typeof maybeFreeable?.free === "function") {
maybeFreeable.free();
}
for (const engine of localSttEngines.values()) {
engine.engine.free();
}
};
return {
sttService,
ttsService,
dictationSttService,
localVoiceTtsProvider,
localModelConfig:
localConfig
? {
modelsDir: localConfig.modelsDir,
defaultModelIds: requiredLocalModelIds,
}
: null,
availability: getLocalSpeechAvailability(speechConfig),
cleanup,
};
}

128
packages/server/src/server/speech/providers/local/sherpa/model-catalog.ts
View File

@@ -1,26 +1,22 @@
import { z } from "zod";
export type SherpaOnnxModelKind = "stt-online" | "stt-offline" | "tts";
export type SherpaOnnxModelId =
| "zipformer-bilingual-zh-en-2023-02-20"
| "paraformer-bilingual-zh-en"
| "parakeet-tdt-0.6b-v3-int8"
| "kitten-nano-en-v0_1-fp16"
| "kokoro-en-v0_19"
| "pocket-tts-onnx-int8";
type DefaultModelRole = "stt" | "tts";
export type SherpaOnnxModelSpec = {
id: SherpaOnnxModelId;
type SherpaOnnxCatalogEntry = {
kind: SherpaOnnxModelKind;
archiveUrl?: string;
downloadFiles?: Array<{ url: string; relPath: string }>;
extractedDir: string;
requiredFiles: string[];
description: string;
aliases?: readonly string[];
defaultFor?: DefaultModelRole;
};
export const SHERPA_ONNX_MODEL_CATALOG: Record<SherpaOnnxModelId, SherpaOnnxModelSpec> = {
export const SHERPA_ONNX_MODEL_CATALOG = {
"zipformer-bilingual-zh-en-2023-02-20": {
id: "zipformer-bilingual-zh-en-2023-02-20",
kind: "stt-online",
archiveUrl:
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2",
@@ -32,44 +28,45 @@ export const SHERPA_ONNX_MODEL_CATALOG: Record<SherpaOnnxModelId, SherpaOnnxMode
"tokens.txt",
],
description: "Streaming Zipformer transducer (fast, good accuracy).",
aliases: ["zipformer", "zipformer-bilingual"],
},
"paraformer-bilingual-zh-en": {
id: "paraformer-bilingual-zh-en",
kind: "stt-online",
archiveUrl:
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2",
extractedDir: "sherpa-onnx-streaming-paraformer-bilingual-zh-en",
requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "tokens.txt"],
description: "Streaming Paraformer (often strong accuracy; heavier).",
aliases: ["paraformer"],
},
"parakeet-tdt-0.6b-v3-int8": {
id: "parakeet-tdt-0.6b-v3-int8",
kind: "stt-offline",
archiveUrl:
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2",
extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8",
requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"],
description: "NVIDIA Parakeet TDT v3 (offline NeMo transducer, multilingual).",
aliases: ["parakeet", "parakeet-v3", "parakeet-tdt"],
defaultFor: "stt",
},
"kitten-nano-en-v0_1-fp16": {
id: "kitten-nano-en-v0_1-fp16",
kind: "tts",
archiveUrl:
"https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2",
extractedDir: "kitten-nano-en-v0_1-fp16",
requiredFiles: ["model.fp16.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
description: "KittenTTS (small, fast English TTS).",
aliases: ["kitten"],
},
"kokoro-en-v0_19": {
id: "kokoro-en-v0_19",
kind: "tts",
archiveUrl: "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2",
extractedDir: "kokoro-en-v0_19",
requiredFiles: ["model.onnx", "voices.bin", "tokens.txt", "espeak-ng-data"],
description: "Kokoro TTS (higher quality; larger).",
aliases: ["kokoro"],
},
"pocket-tts-onnx-int8": {
id: "pocket-tts-onnx-int8",
kind: "tts",
extractedDir: "pocket-tts-onnx-int8",
downloadFiles: [
@@ -112,17 +109,110 @@ export const SHERPA_ONNX_MODEL_CATALOG: Record<SherpaOnnxModelId, SherpaOnnxMode
"reference_sample.wav",
],
description: "Pocket TTS ONNX (INT8) with streaming decode support (via onnxruntime).",
aliases: ["pocket", "pocket-tts"],
defaultFor: "tts",
},
} as const satisfies Record<string, SherpaOnnxCatalogEntry>;
export type SherpaOnnxModelId = keyof typeof SHERPA_ONNX_MODEL_CATALOG;
export type LocalSpeechModelId = SherpaOnnxModelId;
type ModelIdByKind<K extends SherpaOnnxModelKind> = {
[Id in SherpaOnnxModelId]: (typeof SHERPA_ONNX_MODEL_CATALOG)[Id]["kind"] extends K
? Id
: never;
}[SherpaOnnxModelId];
export type LocalSttModelId = ModelIdByKind<"stt-online"> | ModelIdByKind<"stt-offline">;
export type LocalTtsModelId = ModelIdByKind<"tts">;
const ALL_MODEL_IDS = Object.keys(SHERPA_ONNX_MODEL_CATALOG) as SherpaOnnxModelId[];
export const LOCAL_STT_MODEL_IDS = ALL_MODEL_IDS.filter(
(id): id is LocalSttModelId => SHERPA_ONNX_MODEL_CATALOG[id].kind !== "tts"
);
export const LOCAL_TTS_MODEL_IDS = ALL_MODEL_IDS.filter(
(id): id is LocalTtsModelId => SHERPA_ONNX_MODEL_CATALOG[id].kind === "tts"
);
function resolveDefaultModelId(role: "stt"): LocalSttModelId;
function resolveDefaultModelId(role: "tts"): LocalTtsModelId;
function resolveDefaultModelId(role: DefaultModelRole): SherpaOnnxModelId {
const match = ALL_MODEL_IDS.find((id) => {
const entry: SherpaOnnxCatalogEntry = SHERPA_ONNX_MODEL_CATALOG[id];
return entry.defaultFor === role;
});
if (!match) {
throw new Error(`No default model configured for role '${role}'`);
}
return match;
}
export const DEFAULT_LOCAL_STT_MODEL = resolveDefaultModelId("stt");
export const DEFAULT_LOCAL_TTS_MODEL = resolveDefaultModelId("tts");
function buildAliasMap<T extends string>(modelIds: readonly T[]): Record<string, T> {
const aliasMap: Record<string, T> = {};
for (const modelId of modelIds) {
const aliases = SHERPA_ONNX_MODEL_CATALOG[modelId as SherpaOnnxModelId].aliases ?? [];
for (const alias of aliases) {
aliasMap[alias.trim().toLowerCase()] = modelId;
}
}
return aliasMap;
}
function createAliasedModelIdSchema<T extends string>(params: {
modelIds: readonly T[];
aliases: Record<string, T>;
}): z.ZodType<T, z.ZodTypeDef, string> {
const validIds = new Set(params.modelIds);
return z
.string()
.trim()
.toLowerCase()
.refine(
(value): value is T =>
validIds.has(value as T) || Object.prototype.hasOwnProperty.call(params.aliases, value),
{
message: "Invalid model id",
}
)
.transform((value) => params.aliases[value] ?? (value as T));
}
const STT_MODEL_ALIASES = buildAliasMap(LOCAL_STT_MODEL_IDS);
const TTS_MODEL_ALIASES = buildAliasMap(LOCAL_TTS_MODEL_IDS);
export const LocalSttModelIdSchema = createAliasedModelIdSchema({
modelIds: LOCAL_STT_MODEL_IDS,
aliases: STT_MODEL_ALIASES,
});
export const LocalTtsModelIdSchema = createAliasedModelIdSchema({
modelIds: LOCAL_TTS_MODEL_IDS,
aliases: TTS_MODEL_ALIASES,
});
export type SherpaOnnxModelSpec = SherpaOnnxCatalogEntry & {
id: SherpaOnnxModelId;
};
export function listSherpaOnnxModels(): SherpaOnnxModelSpec[] {
return Object.values(SHERPA_ONNX_MODEL_CATALOG);
return ALL_MODEL_IDS.map((id) => ({
id,
...SHERPA_ONNX_MODEL_CATALOG[id],
}));
}
export function getSherpaOnnxModelSpec(id: SherpaOnnxModelId): SherpaOnnxModelSpec {
const spec = SHERPA_ONNX_MODEL_CATALOG[id];
if (!spec) {
throw new Error(`Unknown sherpa-onnx model id: ${id}`);
throw new Error(`Unknown local speech model id: ${id}`);
}
return spec;
return {
id,
...spec,
};
}

6
packages/server/src/server/speech/providers/local/sherpa/model-downloader.ts
View File

@@ -96,7 +96,7 @@ async function isNonEmptyFile(filePath: string): Promise<boolean> {
export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOptions): Promise<string> {
const logger = options.logger.child({
module: "speech",
provider: "sherpa-onnx",
provider: "local",
component: "model-downloader",
modelId: options.modelId,
});
@@ -109,8 +109,8 @@ export async function ensureSherpaOnnxModel(options: EnsureSherpaOnnxModelOption
if (!options.autoDownload) {
throw new Error(
`Missing sherpa-onnx model files for ${options.modelId} in ${modelDir}. ` +
`Set PASEO_SHERPA_ONNX_AUTO_DOWNLOAD=1 to auto-download.`
`Missing local speech model files for ${options.modelId} in ${modelDir}. ` +
`Set PASEO_LOCAL_AUTO_DOWNLOAD=1 to auto-download.`
);
}

2
packages/server/src/server/speech/providers/local/sherpa/sherpa-offline-recognizer.ts
View File

@@ -34,7 +34,7 @@ export class SherpaOfflineRecognizerEngine {
private readonly logger: pino.Logger;
constructor(config: SherpaOfflineRecognizerConfig, logger: pino.Logger) {
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "offline-recognizer" });
this.logger = logger.child({ module: "speech", provider: "local", component: "offline-recognizer" });
assertFileExists(config.model.encoder, "offline encoder");
assertFileExists(config.model.decoder, "offline decoder");

2
packages/server/src/server/speech/providers/local/sherpa/sherpa-online-recognizer.ts
View File

@@ -46,7 +46,7 @@ export class SherpaOnlineRecognizerEngine {
private readonly logger: pino.Logger;
constructor(config: SherpaOnlineRecognizerConfig, logger: pino.Logger) {
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "online-recognizer" });
this.logger = logger.child({ module: "speech", provider: "local", component: "online-recognizer" });
const { model } = config;
if (model.kind === "transducer") {

1
packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-loader.ts
View File

@@ -12,6 +12,7 @@ export function loadSherpaOnnx(): SherpaOnnxModule {
if (cached) {
return cached;
}
const require = createRequire(import.meta.url);
cached = require("sherpa-onnx") as SherpaOnnxModule;
return cached;

14
packages/server/src/server/speech/providers/local/sherpa/sherpa-onnx-node-loader.ts
View File

@@ -14,13 +14,16 @@ function platformArch(): string {
return `${platform}-${process.arch}`;
}
function prependLibraryPath(envKey: "DYLD_LIBRARY_PATH" | "LD_LIBRARY_PATH", dir: string): void {
function prependLibraryPath(
envKey: "DYLD_LIBRARY_PATH" | "LD_LIBRARY_PATH" | "PATH",
dir: string
): void {
const current = process.env[envKey] ?? "";
const parts = current.split(":").filter(Boolean);
const parts = current.split(path.delimiter).filter(Boolean);
if (parts.includes(dir)) {
return;
}
process.env[envKey] = [dir, ...parts].join(":");
process.env[envKey] = [dir, ...parts].join(path.delimiter);
}
export function loadSherpaOnnxNode(): SherpaOnnxNodeModule {
@@ -31,8 +34,7 @@ export function loadSherpaOnnxNode(): SherpaOnnxNodeModule {
const require = createRequire(import.meta.url);
// sherpa-onnx-node depends on a platform-specific package (e.g. sherpa-onnx-darwin-arm64)
// that contains the native addon + its shared libraries. On macOS/Linux we need to ensure
// the appropriate library path env var includes that directory before requiring the addon.
// that contains the native addon and shared libraries. Ensure the OS loader path includes it.
const arch = platformArch();
const pkgName = `sherpa-onnx-${arch}`;
@@ -43,6 +45,8 @@ export function loadSherpaOnnxNode(): SherpaOnnxNodeModule {
prependLibraryPath("DYLD_LIBRARY_PATH", pkgDir);
} else if (process.platform === "linux") {
prependLibraryPath("LD_LIBRARY_PATH", pkgDir);
} else if (process.platform === "win32") {
prependLibraryPath("PATH", pkgDir);
}
} catch {
// Best effort - if the platform package isn't present, require() below will throw a useful error.

2
packages/server/src/server/speech/providers/local/sherpa/sherpa-parakeet-stt.ts
View File

@@ -25,7 +25,7 @@ export class SherpaOnnxParakeetSTT implements SpeechToTextProvider {
constructor(config: SherpaParakeetSttConfig, logger: pino.Logger) {
this.engine = config.engine;
this.silencePeakThreshold = config.silencePeakThreshold ?? 300;
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "parakeet-stt" });
this.logger = logger.child({ module: "speech", provider: "local", component: "parakeet-stt" });
}
public createSession(params: {

2
packages/server/src/server/speech/providers/local/sherpa/sherpa-stt.ts
View File

@@ -28,7 +28,7 @@ export class SherpaOnnxSTT implements SpeechToTextProvider {
this.engine = config.engine;
this.silencePeakThreshold = config.silencePeakThreshold ?? 300;
this.tailPaddingMs = config.tailPaddingMs ?? 500;
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "stt" });
this.logger = logger.child({ module: "speech", provider: "local", component: "stt" });
}
public createSession(params: {

2
packages/server/src/server/speech/providers/local/sherpa/sherpa-tts.ts
View File

@@ -33,7 +33,7 @@ export class SherpaOnnxTTS implements TextToSpeechProvider {
if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") {
throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`);
}
this.logger = logger.child({ module: "speech", provider: "sherpa-onnx", component: "tts" });
this.logger = logger.child({ module: "speech", provider: "local", component: "tts" });
this.speakerId = config.speakerId ?? 0;
this.speed = config.speed ?? 1.0;

31
packages/server/src/server/speech/providers/local/sherpa/speech-download.e2e.test.ts
View File

@@ -2,6 +2,7 @@ import { describe, expect, test } from "vitest";
import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { randomUUID } from "node:crypto";
import pino from "pino";
import { ensureSherpaOnnxModels, getSherpaOnnxModelDir } from "./model-downloader.js";
@@ -96,7 +97,7 @@ describe("speech models (download E2E)", () => {
const set = getModelSet();
const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-speech-download-"));
const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "sherpa-onnx");
const modelsDir = path.join(paseoHomeRoot, ".paseo", "models", "local-speech");
const modelIds: SherpaOnnxModelId[] =
set === "parakeet-pocket"
@@ -114,14 +115,26 @@ describe("speech models (download E2E)", () => {
paseoHomeRoot,
dictationFinalTimeoutMs: 8000,
speech: {
dictationSttProvider: "local",
voiceSttProvider: "local",
voiceTtsProvider: "local",
sherpaOnnx: {
providers: {
dictationStt: { provider: "local", explicit: true },
voiceStt: { provider: "local", explicit: true },
voiceTts: { provider: "local", explicit: true },
},
local: {
modelsDir,
autoDownload: false,
stt: { preset: set === "parakeet-pocket" ? "parakeet-tdt-0.6b-v3-int8" : "zipformer-bilingual-zh-en-2023-02-20" },
tts: { preset: set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16" },
models: {
dictationStt:
set === "parakeet-pocket"
? "parakeet-tdt-0.6b-v3-int8"
: "zipformer-bilingual-zh-en-2023-02-20",
voiceStt:
set === "parakeet-pocket"
? "parakeet-tdt-0.6b-v3-int8"
: "zipformer-bilingual-zh-en-2023-02-20",
voiceTts:
set === "parakeet-pocket" ? "pocket-tts-onnx-int8" : "kitten-nano-en-v0_1-fp16",
},
},
},
});
@@ -178,7 +191,7 @@ describe("speech models (download E2E)", () => {
};
});
await ctx.client.setVoiceConversation(true, `voice-download-${Date.now()}`);
await ctx.client.setVoiceMode(true, randomUUID());
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
const isLast = offset + chunkBytes >= pcm16.length;
@@ -188,7 +201,7 @@ describe("speech models (download E2E)", () => {
if (voiceText.length > 0) {
expect(voiceText).toContain("voice note");
}
await ctx.client.setVoiceConversation(false);
await ctx.client.setVoiceMode(false);
// Streaming TTS: generate locally from downloaded model and validate chunking.
const ttsText = "This is a voice note.";

46
packages/server/src/server/speech/providers/openai/config.test.ts Normal file
View File

@@ -0,0 +1,46 @@
import { describe, expect, test } from "vitest";
import { PersistedConfigSchema } from "../../../persisted-config.js";
import { resolveOpenAiSpeechConfig } from "./config.js";
describe("resolveOpenAiSpeechConfig", () => {
test("treats empty OPENAI_API_KEY as unset", () => {
const persisted = PersistedConfigSchema.parse({});
const env = {
OPENAI_API_KEY: "",
} as NodeJS.ProcessEnv;
const resolved = resolveOpenAiSpeechConfig({
env,
persisted,
providers: {
dictationStt: { provider: "local", explicit: false },
voiceStt: { provider: "local", explicit: false },
voiceTts: { provider: "local", explicit: false },
},
});
expect(resolved).toBeUndefined();
});
test("uses trimmed OPENAI_API_KEY when configured", () => {
const persisted = PersistedConfigSchema.parse({});
const env = {
OPENAI_API_KEY: " sk-test ",
} as NodeJS.ProcessEnv;
const resolved = resolveOpenAiSpeechConfig({
env,
persisted,
providers: {
dictationStt: { provider: "openai", explicit: true },
voiceStt: { provider: "openai", explicit: true },
voiceTts: { provider: "openai", explicit: true },
},
});
expect(resolved?.apiKey).toBe("sk-test");
expect(resolved?.stt?.apiKey).toBe("sk-test");
expect(resolved?.tts?.apiKey).toBe("sk-test");
});
});

126
packages/server/src/server/speech/providers/openai/config.ts Normal file
View File

@@ -0,0 +1,126 @@
import { z } from "zod";
import type { PersistedConfig } from "../../../persisted-config.js";
import type { RequestedSpeechProviders } from "../../speech-types.js";
import type { STTConfig } from "./stt.js";
import type { TTSConfig } from "./tts.js";
export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
export const DEFAULT_OPENAI_TTS_MODEL = "tts-1";
export type OpenAiSpeechProviderConfig = {
apiKey?: string;
stt?: Partial<STTConfig> & { apiKey?: string };
tts?: Partial<TTSConfig> & { apiKey?: string };
realtimeTranscriptionModel?: string;
};
const OpenAiTtsVoiceSchema = z.enum([
"alloy",
"echo",
"fable",
"onyx",
"nova",
"shimmer",
]);
const OpenAiTtsModelSchema = z.enum(["tts-1", "tts-1-hd"]);
const NumberLikeSchema = z.union([
z.number(),
z.string().trim().min(1),
]);
const OptionalFiniteNumberSchema = NumberLikeSchema
.pipe(z.coerce.number().finite())
.optional();
const OptionalTrimmedStringSchema = z
.string()
.trim()
.optional()
.transform((value) => (value && value.length > 0 ? value : undefined));
const OpenAiSpeechResolutionSchema = z.object({
apiKey: OptionalTrimmedStringSchema,
sttConfidenceThreshold: OptionalFiniteNumberSchema,
sttModel: OptionalTrimmedStringSchema,
ttsVoice: z
.string()
.trim()
.toLowerCase()
.pipe(OpenAiTtsVoiceSchema)
.default("alloy"),
ttsModel: z
.string()
.trim()
.toLowerCase()
.pipe(OpenAiTtsModelSchema)
.default(DEFAULT_OPENAI_TTS_MODEL),
realtimeTranscriptionModel: OptionalTrimmedStringSchema.default(
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL
),
});
export function resolveOpenAiSpeechConfig(params: {
env: NodeJS.ProcessEnv;
persisted: PersistedConfig;
providers: RequestedSpeechProviders;
}): OpenAiSpeechProviderConfig | undefined {
const parsed = OpenAiSpeechResolutionSchema.parse({
apiKey: params.env.OPENAI_API_KEY ?? params.persisted.providers?.openai?.apiKey,
sttConfidenceThreshold:
params.env.STT_CONFIDENCE_THRESHOLD ??
params.persisted.features?.dictation?.stt?.confidenceThreshold,
sttModel:
params.env.STT_MODEL ??
(params.providers.voiceStt.provider === "openai"
? params.persisted.features?.voiceMode?.stt?.model
: undefined) ??
(params.providers.dictationStt.provider === "openai"
? params.persisted.features?.dictation?.stt?.model
: undefined),
ttsVoice:
params.env.TTS_VOICE ??
(params.providers.voiceTts.provider === "openai"
? params.persisted.features?.voiceMode?.tts?.voice
: undefined) ??
"alloy",
ttsModel:
params.env.TTS_MODEL ??
(params.providers.voiceTts.provider === "openai"
? params.persisted.features?.voiceMode?.tts?.model
: undefined) ??
DEFAULT_OPENAI_TTS_MODEL,
realtimeTranscriptionModel:
params.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ??
(params.providers.dictationStt.provider === "openai"
? params.persisted.features?.dictation?.stt?.model
: undefined) ??
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
});
if (!parsed.apiKey) {
return undefined;
}
return {
apiKey: parsed.apiKey,
stt: {
apiKey: parsed.apiKey,
...(parsed.sttConfidenceThreshold !== undefined
? { confidenceThreshold: parsed.sttConfidenceThreshold }
: {}),
...(parsed.sttModel
? { model: parsed.sttModel }
: {}),
},
tts: {
apiKey: parsed.apiKey,
voice: parsed.ttsVoice,
model: parsed.ttsModel,
responseFormat: "pcm",
},
realtimeTranscriptionModel: parsed.realtimeTranscriptionModel,
};
}

171
packages/server/src/server/speech/providers/openai/runtime.ts Normal file
View File

@@ -0,0 +1,171 @@
import type { Logger } from "pino";
import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js";
import type { RequestedSpeechProviders } from "../../speech-types.js";
import {
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
DEFAULT_OPENAI_TTS_MODEL,
type OpenAiSpeechProviderConfig,
} from "./config.js";
import { OpenAIRealtimeTranscriptionSession } from "./realtime-transcription-session.js";
import { OpenAISTT } from "./stt.js";
import { OpenAITTS } from "./tts.js";
type OpenAiCredentialState = {
openaiSttApiKey: string | undefined;
openaiTtsApiKey: string | undefined;
openaiDictationApiKey: string | undefined;
};
export type OpenAiSpeechAvailability = {
stt: boolean;
tts: boolean;
dictationStt: boolean;
};
export type SpeechServices = {
sttService: SpeechToTextProvider | null;
ttsService: TextToSpeechProvider | null;
dictationSttService: SpeechToTextProvider | null;
};
function resolveOpenAiCredentials(
openaiConfig: OpenAiSpeechProviderConfig | undefined
): OpenAiCredentialState {
const openaiApiKey = openaiConfig?.apiKey;
return {
openaiSttApiKey: openaiConfig?.stt?.apiKey ?? openaiApiKey,
openaiTtsApiKey: openaiConfig?.tts?.apiKey ?? openaiApiKey,
openaiDictationApiKey: openaiApiKey,
};
}
export function getOpenAiSpeechAvailability(
openaiConfig: OpenAiSpeechProviderConfig | undefined
): OpenAiSpeechAvailability {
const credentials = resolveOpenAiCredentials(openaiConfig);
return {
stt: Boolean(credentials.openaiSttApiKey),
tts: Boolean(credentials.openaiTtsApiKey),
dictationStt: Boolean(credentials.openaiDictationApiKey),
};
}
export function validateOpenAiCredentialRequirements(params: {
providers: RequestedSpeechProviders;
openaiConfig: OpenAiSpeechProviderConfig | undefined;
logger: Logger;
}): void {
const { providers, logger, openaiConfig } = params;
const openAiCredentials = resolveOpenAiCredentials(openaiConfig);
const missingOpenAiCredentialsFor: string[] = [];
if (providers.voiceStt.provider === "openai" && !openAiCredentials.openaiSttApiKey) {
missingOpenAiCredentialsFor.push("voice.stt");
}
if (providers.voiceTts.provider === "openai" && !openAiCredentials.openaiTtsApiKey) {
missingOpenAiCredentialsFor.push("voice.tts");
}
if (
providers.dictationStt.provider === "openai" &&
!openAiCredentials.openaiDictationApiKey
) {
missingOpenAiCredentialsFor.push("dictation.stt");
}
if (missingOpenAiCredentialsFor.length > 0) {
logger.error(
{
requestedProviders: {
dictationStt: providers.dictationStt.provider,
voiceStt: providers.voiceStt.provider,
voiceTts: providers.voiceTts.provider,
},
missingOpenAiCredentialsFor,
},
"Invalid speech configuration: OpenAI provider selected but credentials are missing"
);
throw new Error(
`Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}`
);
}
}
export function initializeOpenAiSpeechServices(params: {
providers: RequestedSpeechProviders;
openaiConfig: OpenAiSpeechProviderConfig | undefined;
existing: SpeechServices;
logger: Logger;
}): SpeechServices {
const { providers, openaiConfig, existing, logger } = params;
const openAiCredentials = resolveOpenAiCredentials(openaiConfig);
let sttService = existing.sttService;
let ttsService = existing.ttsService;
let dictationSttService = existing.dictationSttService;
const needsOpenAiStt = !sttService && providers.voiceStt.provider === "openai";
const needsOpenAiTts = !ttsService && providers.voiceTts.provider === "openai";
const needsOpenAiDictation =
!dictationSttService && providers.dictationStt.provider === "openai";
if (
(needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) &&
(openAiCredentials.openaiSttApiKey ||
openAiCredentials.openaiTtsApiKey ||
openAiCredentials.openaiDictationApiKey)
) {
logger.info("OpenAI speech provider initialized");
if (needsOpenAiStt && openAiCredentials.openaiSttApiKey) {
const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {};
sttService = new OpenAISTT(
{
apiKey: openAiCredentials.openaiSttApiKey,
...sttConfig,
},
logger
);
}
if (needsOpenAiTts && openAiCredentials.openaiTtsApiKey) {
const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {};
ttsService = new OpenAITTS(
{
apiKey: openAiCredentials.openaiTtsApiKey,
voice: "alloy",
model: DEFAULT_OPENAI_TTS_MODEL,
responseFormat: "pcm",
...ttsConfig,
},
logger
);
}
const dictationApiKey = openAiCredentials.openaiDictationApiKey;
if (needsOpenAiDictation && dictationApiKey) {
dictationSttService = {
id: "openai",
createSession: ({ logger: sessionLogger, language, prompt }) =>
new OpenAIRealtimeTranscriptionSession({
apiKey: dictationApiKey,
logger: sessionLogger,
transcriptionModel:
openaiConfig?.realtimeTranscriptionModel ??
DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
...(language ? { language } : {}),
...(prompt ? { prompt } : {}),
turnDetection: null,
}),
};
}
} else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) {
logger.warn("OpenAI speech providers are configured but credentials are missing");
}
return {
sttService,
ttsService,
dictationSttService,
};
}

127
packages/server/src/server/speech/speech-config-resolver.test.ts Normal file
View File

@@ -0,0 +1,127 @@
import path from "node:path";
import { describe, expect, test } from "vitest";
import { PersistedConfigSchema } from "../persisted-config.js";
import { resolveSpeechConfig } from "./speech-config-resolver.js";
describe("resolveSpeechConfig", () => {
test("resolves local-first defaults without env overrides", () => {
const paseoHome = "/tmp/paseo-home";
const persisted = PersistedConfigSchema.parse({});
const env = {} as NodeJS.ProcessEnv;
const result = resolveSpeechConfig({
paseoHome,
env,
persisted,
});
expect(result.openai).toBeUndefined();
expect(result.speech.providers.dictationStt).toEqual({
provider: "local",
explicit: false,
});
expect(result.speech.providers.voiceStt).toEqual({
provider: "local",
explicit: false,
});
expect(result.speech.providers.voiceTts).toEqual({
provider: "local",
explicit: false,
});
expect(result.speech.local).toEqual({
modelsDir: path.join(paseoHome, "models", "local-speech"),
autoDownload: true,
models: {
dictationStt: "parakeet-tdt-0.6b-v3-int8",
voiceStt: "parakeet-tdt-0.6b-v3-int8",
voiceTts: "pocket-tts-onnx-int8",
},
});
expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8");
});
test("resolves feature-scoped local model env vars", () => {
const persisted = PersistedConfigSchema.parse({
features: {
voiceMode: {
stt: { provider: "openai", model: "gpt-4o-transcribe" },
},
},
providers: {
openai: { apiKey: "persisted-key" },
},
});
const env = {
PASEO_DICTATION_LOCAL_STT_MODEL: "zipformer",
PASEO_VOICE_LOCAL_STT_MODEL: "parakeet",
PASEO_VOICE_LOCAL_TTS_MODEL: "kitten",
PASEO_VOICE_LOCAL_TTS_SPEAKER_ID: "5",
PASEO_VOICE_LOCAL_TTS_SPEED: "1.35",
PASEO_LOCAL_MODELS_DIR: "/tmp/models",
PASEO_LOCAL_AUTO_DOWNLOAD: "0",
OPENAI_API_KEY: "env-key",
PASEO_VOICE_STT_PROVIDER: "openai",
PASEO_DICTATION_STT_PROVIDER: "local",
PASEO_VOICE_TTS_PROVIDER: "local",
} as NodeJS.ProcessEnv;
const result = resolveSpeechConfig({
paseoHome: "/tmp/paseo-home",
env,
persisted,
});
expect(result.speech.local).toEqual({
modelsDir: "/tmp/models",
autoDownload: false,
models: {
dictationStt: "zipformer-bilingual-zh-en-2023-02-20",
voiceStt: "parakeet-tdt-0.6b-v3-int8",
voiceTts: "kitten-nano-en-v0_1-fp16",
voiceTtsSpeakerId: 5,
voiceTtsSpeed: 1.35,
},
});
expect(result.speech.providers.dictationStt).toEqual({
provider: "local",
explicit: true,
});
expect(result.speech.providers.voiceStt).toEqual({
provider: "openai",
explicit: true,
});
expect(result.speech.providers.voiceTts).toEqual({
provider: "local",
explicit: true,
});
expect(result.speech.local?.models.dictationStt).toBe("zipformer-bilingual-zh-en-2023-02-20");
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.local?.models.voiceTts).toBe("kitten-nano-en-v0_1-fp16");
expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(5);
expect(result.speech.local?.models.voiceTtsSpeed).toBe(1.35);
expect(result.openai?.apiKey).toBe("env-key");
expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe");
});
test("ignores deprecated shared local model env vars", () => {
const persisted = PersistedConfigSchema.parse({});
const env = {
PASEO_LOCAL_STT_MODEL: "zipformer-bilingual-zh-en-2023-02-20",
PASEO_LOCAL_TTS_MODEL: "kitten-nano-en-v0_1-fp16",
} as NodeJS.ProcessEnv;
const result = resolveSpeechConfig({
paseoHome: "/tmp/paseo-home",
env,
persisted,
});
expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v3-int8");
expect(result.speech.local?.models.voiceTts).toBe("pocket-tts-onnx-int8");
});
});

105
packages/server/src/server/speech/speech-config-resolver.ts Normal file
View File

@@ -0,0 +1,105 @@
import { z } from "zod";
import type { PersistedConfig } from "../persisted-config.js";
import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
import { resolveLocalSpeechConfig } from "./providers/local/config.js";
import { resolveOpenAiSpeechConfig } from "./providers/openai/config.js";
import {
SpeechProviderIdSchema,
type RequestedSpeechProvider,
type RequestedSpeechProviders,
} from "./speech-types.js";
const OptionalSpeechProviderSchema = z
.string()
.trim()
.toLowerCase()
.pipe(SpeechProviderIdSchema)
.optional();
const RequestedSpeechProvidersSchema = z.object({
dictationStt: OptionalSpeechProviderSchema.default("local"),
voiceStt: OptionalSpeechProviderSchema.default("local"),
voiceTts: OptionalSpeechProviderSchema.default("local"),
});
function resolveRequestedSpeechProviders(params: {
env: NodeJS.ProcessEnv;
persisted: PersistedConfig;
}): RequestedSpeechProviders {
const resolveFeatureProvider = (
configuredValue: string | undefined,
parsedValue: z.infer<typeof SpeechProviderIdSchema>
): RequestedSpeechProvider => ({
provider: parsedValue,
explicit: configuredValue !== undefined,
});
const dictationSttProviderFromConfig =
params.env.PASEO_DICTATION_STT_PROVIDER ??
params.persisted.features?.dictation?.stt?.provider;
const voiceSttProviderFromConfig =
params.env.PASEO_VOICE_STT_PROVIDER ??
params.persisted.features?.voiceMode?.stt?.provider;
const voiceTtsProviderFromConfig =
params.env.PASEO_VOICE_TTS_PROVIDER ??
params.persisted.features?.voiceMode?.tts?.provider;
const parsed = RequestedSpeechProvidersSchema.parse({
dictationStt: dictationSttProviderFromConfig ?? "local",
voiceStt: voiceSttProviderFromConfig ?? "local",
voiceTts: voiceTtsProviderFromConfig ?? "local",
});
return {
dictationStt: resolveFeatureProvider(
dictationSttProviderFromConfig,
parsed.dictationStt
),
voiceStt: resolveFeatureProvider(
voiceSttProviderFromConfig,
parsed.voiceStt
),
voiceTts: resolveFeatureProvider(
voiceTtsProviderFromConfig,
parsed.voiceTts
),
};
}
export function resolveSpeechConfig(params: {
paseoHome: string;
env: NodeJS.ProcessEnv;
persisted: PersistedConfig;
}): {
openai: PaseoOpenAIConfig | undefined;
speech: PaseoSpeechConfig;
} {
const providers = resolveRequestedSpeechProviders({
env: params.env,
persisted: params.persisted,
});
const local = resolveLocalSpeechConfig({
paseoHome: params.paseoHome,
env: params.env,
persisted: params.persisted,
providers,
});
const openai = resolveOpenAiSpeechConfig({
env: params.env,
persisted: params.persisted,
providers,
});
return {
openai,
speech: {
providers,
...(local.local
? { local: local.local }
: {}),
},
};
}

173
packages/server/src/server/speech/speech-runtime.ts Normal file
View File

@@ -0,0 +1,173 @@
import type { Logger } from "pino";
import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
import {
getLocalSpeechAvailability,
initializeLocalSpeechServices,
} from "./providers/local/runtime.js";
import type { LocalSpeechModelId } from "./providers/local/config.js";
import {
getOpenAiSpeechAvailability,
initializeOpenAiSpeechServices,
validateOpenAiCredentialRequirements,
} from "./providers/openai/runtime.js";
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js";
import type { RequestedSpeechProviders } from "./speech-types.js";
function resolveRequestedSpeechProviders(
speechConfig: PaseoSpeechConfig | null
): RequestedSpeechProviders {
const fromConfig = speechConfig?.providers;
if (fromConfig) {
return fromConfig;
}
return {
dictationStt: { provider: "local", explicit: false },
voiceStt: { provider: "local", explicit: false },
voiceTts: { provider: "local", explicit: false },
};
}
export type InitializedSpeechRuntime = {
sttService: SpeechToTextProvider | null;
ttsService: TextToSpeechProvider | null;
dictationSttService: SpeechToTextProvider | null;
cleanup: () => void;
localModelConfig: {
modelsDir: string;
defaultModelIds: LocalSpeechModelId[];
} | null;
};
export async function initializeSpeechRuntime(params: {
logger: Logger;
openaiConfig?: PaseoOpenAIConfig;
speechConfig?: PaseoSpeechConfig;
}): Promise<InitializedSpeechRuntime> {
const logger = params.logger;
const speechConfig = params.speechConfig ?? null;
const openaiConfig = params.openaiConfig;
const providers = resolveRequestedSpeechProviders(speechConfig);
validateOpenAiCredentialRequirements({
providers,
openaiConfig,
logger,
});
logger.info(
{
requestedProviders: {
dictationStt: providers.dictationStt.provider,
voiceStt: providers.voiceStt.provider,
voiceTts: providers.voiceTts.provider,
},
availability: {
openai: getOpenAiSpeechAvailability(openaiConfig),
local: getLocalSpeechAvailability(speechConfig),
},
},
"Speech provider reconciliation started"
);
const localSpeech = await initializeLocalSpeechServices({
providers,
speechConfig,
logger,
});
const openAiSpeech = initializeOpenAiSpeechServices({
providers,
openaiConfig,
existing: {
sttService: localSpeech.sttService,
ttsService: localSpeech.ttsService,
dictationSttService: localSpeech.dictationSttService,
},
logger,
});
const effectiveProviders = {
dictationStt: openAiSpeech.dictationSttService?.id ?? "unavailable",
voiceStt: openAiSpeech.sttService?.id ?? "unavailable",
voiceTts:
!openAiSpeech.ttsService
? "unavailable"
: openAiSpeech.ttsService === localSpeech.localVoiceTtsProvider
? "local"
: "openai",
};
const unavailableFeatures = [
!openAiSpeech.dictationSttService ? "dictation.stt" : null,
!openAiSpeech.sttService ? "voice.stt" : null,
!openAiSpeech.ttsService ? "voice.tts" : null,
].filter((feature): feature is string => feature !== null);
const explicitlyConfiguredUnavailableFeatures = unavailableFeatures.filter((feature) => {
if (feature === "dictation.stt") {
return providers.dictationStt.explicit;
}
if (feature === "voice.stt") {
return providers.voiceStt.explicit;
}
return providers.voiceTts.explicit;
});
if (explicitlyConfiguredUnavailableFeatures.length > 0) {
logger.error(
{
requestedProviders: {
dictationStt: providers.dictationStt.provider,
voiceStt: providers.voiceStt.provider,
voiceTts: providers.voiceTts.provider,
},
explicitProviders: {
dictationStt: providers.dictationStt.explicit,
voiceStt: providers.voiceStt.explicit,
voiceTts: providers.voiceTts.explicit,
},
effectiveProviders,
unavailableFeatures: explicitlyConfiguredUnavailableFeatures,
},
"Speech provider reconciliation failed: configured features are unavailable"
);
throw new Error(
`Configured speech features unavailable: ${explicitlyConfiguredUnavailableFeatures.join(", ")}`
);
}
if (unavailableFeatures.length > 0) {
logger.warn(
{
requestedProviders: {
dictationStt: providers.dictationStt.provider,
voiceStt: providers.voiceStt.provider,
voiceTts: providers.voiceTts.provider,
},
explicitProviders: {
dictationStt: providers.dictationStt.explicit,
voiceStt: providers.voiceStt.explicit,
voiceTts: providers.voiceTts.explicit,
},
effectiveProviders,
unavailableFeatures,
},
"Speech provider reconciliation completed with unavailable default features"
);
} else {
logger.info(
{
effectiveProviders,
},
"Speech provider reconciliation completed"
);
}
return {
sttService: openAiSpeech.sttService,
ttsService: openAiSpeech.ttsService,
dictationSttService: openAiSpeech.dictationSttService,
cleanup: localSpeech.cleanup,
localModelConfig: localSpeech.localModelConfig,
};
}

16
packages/server/src/server/speech/speech-types.ts Normal file
View File

@@ -0,0 +1,16 @@
import { z } from "zod";
export const SpeechProviderIdSchema = z.enum(["openai", "local"]);
export type SpeechProviderId = z.infer<typeof SpeechProviderIdSchema>;
export const RequestedSpeechProviderSchema = z.object({
provider: SpeechProviderIdSchema,
explicit: z.boolean(),
});
export type RequestedSpeechProvider = z.infer<typeof RequestedSpeechProviderSchema>;
export type RequestedSpeechProviders = {
dictationStt: RequestedSpeechProvider;
voiceStt: RequestedSpeechProvider;
voiceTts: RequestedSpeechProvider;
};

7
packages/server/src/server/test-utils/paseo-daemon.ts
View File

@@ -21,6 +21,9 @@ type TestPaseoDaemonOptions = {
cleanup?: boolean;
openai?: PaseoOpenAIConfig;
speech?: PaseoSpeechConfig;
voiceLlmProvider?: PaseoDaemonConfig["voiceLlmProvider"];
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
dictationFinalTimeoutMs?: number;
};
@@ -78,7 +81,9 @@ export async function createTestPaseoDaemon(
appBaseUrl: "https://app.paseo.sh",
openai: options.openai,
speech: options.speech,
openrouterApiKey: null,
voiceLlmProvider: options.voiceLlmProvider ?? null,
voiceLlmProviderExplicit: options.voiceLlmProviderExplicit ?? false,
voiceLlmModel: options.voiceLlmModel ?? null,
dictationFinalTimeoutMs: options.dictationFinalTimeoutMs,
downloadTokenTtlMs: options.downloadTokenTtlMs,
};

140
packages/server/src/server/voice-conversation-store.ts
View File

@@ -1,140 +0,0 @@
import { readFile, writeFile, readdir, unlink, mkdir, stat } from "fs/promises";
import { join } from "path";
import type { ModelMessage } from "@ai-sdk/provider-utils";
import { standardizePrompt } from "ai/internal";
type LoggerLike = {
child(bindings: Record<string, unknown>): LoggerLike;
info(...args: any[]): void;
debug(...args: any[]): void;
warn(...args: any[]): void;
error(...args: any[]): void;
};
function getLogger(logger: LoggerLike): LoggerLike {
return logger.child({ module: "voice-conversation-store" });
}
export interface VoiceConversationMetadata {
id: string;
lastUpdated: Date;
messageCount: number;
}
interface VoiceConversationData {
voiceConversationId: string;
lastUpdated: string;
messageCount: number;
messages: ModelMessage[];
}
export class VoiceConversationStore {
private readonly baseDir: string;
constructor(baseDir: string) {
this.baseDir = baseDir;
}
private async ensureBaseDir(): Promise<void> {
await mkdir(this.baseDir, { recursive: true });
}
public async save(
logger: LoggerLike,
voiceConversationId: string,
messages: ModelMessage[]
): Promise<void> {
const log = getLogger(logger);
await this.ensureBaseDir();
const filepath = join(this.baseDir, `${voiceConversationId}.json`);
const data: VoiceConversationData = {
voiceConversationId,
lastUpdated: new Date().toISOString(),
messageCount: messages.length,
messages,
};
await writeFile(filepath, JSON.stringify(data, null, 2), "utf-8");
log.debug({ voiceConversationId, messageCount: messages.length }, "Saved voice conversation");
}
/**
* Load voice conversation from disk.
* Returns null when missing or invalid (best-effort).
*/
public async load(
logger: LoggerLike,
voiceConversationId: string
): Promise<ModelMessage[] | null> {
const log = getLogger(logger);
const filepath = join(this.baseDir, `${voiceConversationId}.json`);
try {
await stat(filepath);
} catch {
log.debug({ voiceConversationId }, "Voice conversation not found");
return null;
}
try {
const fileContent = await readFile(filepath, "utf-8");
const data: VoiceConversationData = JSON.parse(fileContent);
const result = await standardizePrompt({ prompt: data.messages });
log.debug(
{ voiceConversationId, messageCount: data.messageCount },
"Loaded voice conversation"
);
return result.messages as ModelMessage[];
} catch (error) {
log.warn({ err: error, voiceConversationId }, "Failed to load voice conversation");
return null;
}
}
public async list(logger: LoggerLike): Promise<VoiceConversationMetadata[]> {
const log = getLogger(logger);
try {
await this.ensureBaseDir();
const files = await readdir(this.baseDir);
const jsonFiles = files.filter((f) => f.endsWith(".json"));
const conversations: VoiceConversationMetadata[] = [];
for (const file of jsonFiles) {
try {
const filepath = join(this.baseDir, file);
const fileContent = await readFile(filepath, "utf-8");
const data: VoiceConversationData = JSON.parse(fileContent);
conversations.push({
id: data.voiceConversationId,
lastUpdated: new Date(data.lastUpdated),
messageCount: data.messageCount,
});
} catch (error) {
log.warn({ err: error, file }, "Failed to read voice conversation file");
}
}
conversations.sort(
(a, b) => b.lastUpdated.getTime() - a.lastUpdated.getTime()
);
return conversations;
} catch (error) {
log.warn({ err: error }, "Failed to list voice conversations");
return [];
}
}
public async delete(logger: LoggerLike, voiceConversationId: string): Promise<void> {
const log = getLogger(logger);
const filepath = join(this.baseDir, `${voiceConversationId}.json`);
await unlink(filepath);
log.debug({ voiceConversationId }, "Deleted voice conversation");
}
}

147
packages/server/src/server/voice-conversations.e2e.test.ts
View File

@@ -1,147 +0,0 @@
import { describe, test, expect } from "vitest";
import { existsSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { v4 as uuidv4 } from "uuid";
import { createTestPaseoDaemon } from "./test-utils/paseo-daemon.js";
import { DaemonClient } from "./test-utils/daemon-client.js";
async function waitForFile(filepath: string, timeoutMs = 5000): Promise<void> {
const start = Date.now();
// eslint-disable-next-line no-constant-condition
while (true) {
if (existsSync(filepath)) {
return;
}
if (Date.now() - start > timeoutMs) {
throw new Error(`Timed out waiting for file: ${filepath}`);
}
await new Promise((r) => setTimeout(r, 50));
}
}
async function waitForJsonFile<T>(
filepath: string,
timeoutMs = 5000
): Promise<T> {
const start = Date.now();
// eslint-disable-next-line no-constant-condition
while (true) {
if (existsSync(filepath)) {
try {
const raw = readFileSync(filepath, "utf8");
if (raw.trim().length > 0) {
return JSON.parse(raw) as T;
}
} catch {
// File may exist but still be mid-write; retry.
}
}
if (Date.now() - start > timeoutMs) {
throw new Error(`Timed out waiting for valid JSON: ${filepath}`);
}
await new Promise((r) => setTimeout(r, 50));
}
}
describe("voice conversations - daemon E2E", () => {
test(
"two concurrent clients persist independently under paseoHome/voice-conversations",
async () => {
const daemon = await createTestPaseoDaemon();
const url = `ws://127.0.0.1:${daemon.port}/ws`;
const clientA = new DaemonClient({ url });
const clientB = new DaemonClient({ url });
await clientA.connect();
await clientB.connect();
try {
const voiceConversationIdA = uuidv4();
const voiceConversationIdB = uuidv4();
await clientA.setVoiceConversation(true, voiceConversationIdA);
await clientB.setVoiceConversation(true, voiceConversationIdB);
// Minimal traffic to cause a persist without requiring external APIs.
await clientA.setVoiceConversation(false);
await clientB.setVoiceConversation(false);
const fileA = join(
daemon.paseoHome,
"voice-conversations",
`${voiceConversationIdA}.json`
);
const fileB = join(
daemon.paseoHome,
"voice-conversations",
`${voiceConversationIdB}.json`
);
await waitForFile(fileA);
await waitForFile(fileB);
const dataA = await waitForJsonFile<{
voiceConversationId: string;
messageCount: number;
messages: unknown[];
}>(fileA);
const dataB = await waitForJsonFile<{
voiceConversationId: string;
messageCount: number;
messages: unknown[];
}>(fileB);
expect(dataA.voiceConversationId).toBe(voiceConversationIdA);
expect(dataB.voiceConversationId).toBe(voiceConversationIdB);
expect(dataA.messageCount).toBe(0);
expect(dataB.messageCount).toBe(0);
expect(Array.isArray(dataA.messages)).toBe(true);
expect(Array.isArray(dataB.messages)).toBe(true);
} finally {
await clientA.close().catch(() => undefined);
await clientB.close().catch(() => undefined);
await daemon.close();
}
},
30000
);
test(
"WS attach ignores URL conversationId param for voice conversation state",
async () => {
const daemon = await createTestPaseoDaemon();
const urlConversationId = `url-${uuidv4()}`;
const url = `ws://127.0.0.1:${daemon.port}/ws?conversationId=${encodeURIComponent(
urlConversationId
)}`;
const client = new DaemonClient({ url });
await client.connect();
try {
const voiceConversationId = `client-${uuidv4()}`;
await client.setVoiceConversation(true, voiceConversationId);
await client.setVoiceConversation(false);
const file = join(
daemon.paseoHome,
"voice-conversations",
`${voiceConversationId}.json`
);
const urlFile = join(
daemon.paseoHome,
"voice-conversations",
`${urlConversationId}.json`
);
await waitForFile(file);
expect(existsSync(urlFile)).toBe(false);
} finally {
await client.close().catch(() => undefined);
await daemon.close();
}
},
30000
);
});

136
packages/server/src/server/voice-local-agent.e2e.test.ts Normal file
View File

@@ -0,0 +1,136 @@
import { afterAll, beforeAll, describe, expect, test } from "vitest";
import { randomUUID } from "node:crypto";
import path from "node:path";
import { readFile } from "node:fs/promises";
import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js";
const openaiApiKey = process.env.OPENAI_API_KEY ?? null;
const shouldRun =
process.env.PASEO_VOICE_LOCAL_AGENT_E2E === "1" &&
Boolean(openaiApiKey) &&
!process.env.CI;
function waitForSignal<T>(
timeoutMs: number,
setup: (
resolve: (value: T) => void,
reject: (error: Error) => void
) => () => void
): Promise<T> {
return new Promise((resolve, reject) => {
let cleanup: (() => void) | null = null;
const timeout = setTimeout(() => {
cleanup?.();
reject(new Error(`Timeout waiting for event after ${timeoutMs}ms`));
}, timeoutMs);
cleanup = setup(
(value) => {
clearTimeout(timeout);
cleanup?.();
resolve(value);
},
(error) => {
clearTimeout(timeout);
cleanup?.();
reject(error);
}
);
});
}
(shouldRun ? describe : describe.skip)(
"voice local-agent e2e",
() => {
let ctx: DaemonTestContext;
beforeAll(async () => {
ctx = await createDaemonTestContext({
agentClients: {},
openai: { apiKey: openaiApiKey! },
speech: {
providers: {
dictationStt: { provider: "openai", explicit: true },
voiceStt: { provider: "openai", explicit: true },
voiceTts: { provider: "openai", explicit: true },
},
},
voiceLlmProvider: "codex",
voiceLlmProviderExplicit: true,
voiceLlmModel: "gpt-5.2-mini",
});
}, 120000);
afterAll(async () => {
await ctx.cleanup();
}, 60000);
test(
"routes voice turns through local agent speak tool",
async () => {
await ctx.client.setVoiceMode(true, randomUUID());
const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => {
const offAudio = ctx.client.on("audio_output", (msg) => {
if (msg.type !== "audio_output") return;
resolve({ chunkId: msg.payload.id });
});
const offError = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "error") return;
reject(new Error(String(msg.payload.content)));
});
return () => {
offAudio();
offError();
};
});
const assistantLogPromise = waitForSignal<string>(120000, (resolve, reject) => {
const offLog = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "assistant") return;
const content = String(msg.payload.content ?? "");
if (!content.trim()) return;
resolve(content);
});
const offError = ctx.client.on("activity_log", (msg) => {
if (msg.type !== "activity_log") return;
if (msg.payload.type !== "error") return;
reject(new Error(String(msg.payload.content)));
});
return () => {
offLog();
offError();
};
});
const fixturePath = path.resolve(
process.cwd(),
"..",
"app",
"e2e",
"fixtures",
"recording.wav"
);
const wav = await readFile(fixturePath);
await ctx.client.sendVoiceAudioChunk(wav.toString("base64"), "audio/wav", true);
const [{ chunkId }, assistantText] = await Promise.all([
audioPromise,
assistantLogPromise,
]);
expect(chunkId.length).toBeGreaterThan(0);
expect(assistantText.trim().length).toBeGreaterThan(0);
const agents = await ctx.client.fetchAgents();
expect(
agents.some((agent) => String(agent.labels?.surface ?? "") === "voice")
).toBe(false);
},
180000
);
}
);

93
packages/server/src/server/voice-mcp-bridge.test.ts Normal file
View File

@@ -0,0 +1,93 @@
import os from "node:os";
import path from "node:path";
import { mkdtemp, rm } from "node:fs/promises";
import { describe, expect, test } from "vitest";
import { experimental_createMCPClient } from "ai";
import { z } from "zod";
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
import pino from "pino";
import { createVoiceMcpBridgeSocketServer } from "./voice-mcp-bridge.js";
describe("voice MCP bridge", () => {
test("proxies stdio MCP messages through unix socket bridge", async () => {
const tmpRoot = await mkdtemp(path.join(os.tmpdir(), "paseo-voice-mcp-bridge-"));
const socketPath = path.join(tmpRoot, "voice-mcp.sock");
const callerAgentId = "voice-agent-bridge-test";
const bridge = createVoiceMcpBridgeSocketServer({
socketPath,
logger: pino({ level: "silent" }),
createAgentMcpServerForCaller: async (callerId) => {
const server = new McpServer({
name: "bridge-test-server",
version: "1.0.0",
});
server.registerTool(
"echo_caller",
{
value: z.string().optional(),
},
async (args) => {
return {
content: [
{
type: "text",
text: JSON.stringify({
callerAgentId: callerId,
value: args.value ?? null,
}),
},
],
structuredContent: {
callerAgentId: callerId,
value: args.value ?? null,
},
};
}
);
return server;
},
});
await bridge.start();
const tsxBin = path.resolve(process.cwd(), "../../node_modules/.bin/tsx");
const serverIndex = path.resolve(process.cwd(), "src/server/index.ts");
const transport = new StdioClientTransport({
command: process.execPath,
args: [
tsxBin,
serverIndex,
"__paseo_voice_mcp_bridge",
"--socket",
socketPath,
"--caller-agent-id",
callerAgentId,
],
});
const client = await experimental_createMCPClient({ transport });
try {
const result = await client.callTool({
name: "echo_caller",
args: { value: "ok" },
});
const payload =
((result as { structuredContent?: { callerAgentId?: string; value?: string | null } })
.structuredContent) ?? null;
expect(payload?.callerAgentId).toBe(callerAgentId);
} finally {
await client.close();
await bridge.stop();
await rm(tmpRoot, { recursive: true, force: true });
}
}, 30_000);
});

337
packages/server/src/server/voice-mcp-bridge.ts Normal file
View File

@@ -0,0 +1,337 @@
import net from "node:net";
import path from "node:path";
import { mkdir, rm } from "node:fs/promises";
import type { Logger } from "pino";
import pino from "pino";
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
import type { JSONRPCMessage } from "@modelcontextprotocol/sdk/types.js";
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
type BridgeEnvelope =
| { type: "init"; callerAgentId: string }
| { type: "mcp"; message: JSONRPCMessage }
| { type: "ready" }
| { type: "error"; message: string };
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}
function parseEnvelope(raw: string): BridgeEnvelope {
const parsed = JSON.parse(raw);
if (!isRecord(parsed) || typeof parsed.type !== "string") {
throw new Error("Invalid bridge envelope");
}
if (parsed.type === "init") {
const callerAgentId = typeof parsed.callerAgentId === "string" ? parsed.callerAgentId.trim() : "";
if (!callerAgentId) {
throw new Error("Invalid init payload: callerAgentId is required");
}
return { type: "init", callerAgentId };
}
if (parsed.type === "mcp") {
if (!("message" in parsed)) {
throw new Error("Invalid mcp payload: message is required");
}
return { type: "mcp", message: parsed.message as JSONRPCMessage };
}
if (parsed.type === "ready") return { type: "ready" };
if (parsed.type === "error") {
return { type: "error", message: typeof parsed.message === "string" ? parsed.message : "Unknown error" };
}
throw new Error(`Unknown envelope type: ${parsed.type}`);
}
function encodeEnvelope(envelope: BridgeEnvelope): string {
return `${JSON.stringify(envelope)}\n`;
}
export type VoiceMcpBridgeSocketServer = {
socketPath: string;
start: () => Promise<void>;
stop: () => Promise<void>;
};
export function createVoiceMcpBridgeSocketServer(params: {
socketPath: string;
logger: Logger;
createAgentMcpServerForCaller: (callerAgentId: string) => Promise<{ connect: (transport: InMemoryTransport) => Promise<void>; close?: () => Promise<void> }>;
}): VoiceMcpBridgeSocketServer {
const logger = params.logger.child({ module: "voice-mcp-bridge" });
const sockets = new Set<net.Socket>();
const server = net.createServer((socket) => {
sockets.add(socket);
const connectionLogger = logger.child({ component: "connection" });
let readBuffer = "";
let initialized = false;
let clientTransport: InMemoryTransport | null = null;
let mcpServer: { close?: () => Promise<void> } | null = null;
const send = (payload: BridgeEnvelope) => {
socket.write(encodeEnvelope(payload));
};
const fail = (message: string) => {
send({ type: "error", message });
socket.end();
};
const cleanup = async () => {
sockets.delete(socket);
const closeTasks: Promise<unknown>[] = [];
if (clientTransport) {
closeTasks.push(clientTransport.close().catch(() => undefined));
}
if (mcpServer?.close) {
closeTasks.push(mcpServer.close().catch(() => undefined));
}
await Promise.all(closeTasks);
};
socket.on("data", (chunk) => {
readBuffer += chunk.toString("utf8");
while (true) {
const newlineIndex = readBuffer.indexOf("\n");
if (newlineIndex < 0) break;
const line = readBuffer.slice(0, newlineIndex).trim();
readBuffer = readBuffer.slice(newlineIndex + 1);
if (!line) continue;
let message: BridgeEnvelope;
try {
message = parseEnvelope(line);
} catch (error) {
fail(error instanceof Error ? error.message : String(error));
return;
}
if (message.type === "init") {
if (initialized) {
fail("Bridge already initialized");
return;
}
initialized = true;
void (async () => {
try {
const [proxyClient, proxyServer] = InMemoryTransport.createLinkedPair();
const serverInstance = await params.createAgentMcpServerForCaller(message.callerAgentId);
await serverInstance.connect(proxyServer);
await proxyClient.start();
proxyClient.onmessage = (jsonrpcMessage) => {
send({ type: "mcp", message: jsonrpcMessage });
};
clientTransport = proxyClient;
mcpServer = serverInstance;
send({ type: "ready" });
} catch (error) {
connectionLogger.error({ err: error }, "Failed to initialize voice MCP bridge connection");
fail(error instanceof Error ? error.message : String(error));
}
})();
continue;
}
if (message.type === "mcp") {
if (!clientTransport) {
fail("Bridge is not initialized");
return;
}
void clientTransport.send(message.message).catch((error) => {
connectionLogger.error({ err: error }, "Failed to forward MCP message");
fail(error instanceof Error ? error.message : String(error));
});
continue;
}
}
});
socket.on("error", (error) => {
connectionLogger.error({ err: error }, "Voice MCP bridge socket error");
});
socket.on("close", () => {
void cleanup();
});
});
return {
socketPath: params.socketPath,
async start() {
await mkdir(path.dirname(params.socketPath), { recursive: true });
await rm(params.socketPath, { force: true }).catch(() => undefined);
await new Promise<void>((resolve, reject) => {
server.once("error", reject);
server.listen(params.socketPath, () => {
server.off("error", reject);
resolve();
});
});
logger.info({ socketPath: params.socketPath }, "Voice MCP bridge socket server listening");
},
async stop() {
for (const socket of sockets) {
socket.destroy();
}
await new Promise<void>((resolve, reject) => {
server.close((error) => {
if (error) reject(error);
else resolve();
});
});
await rm(params.socketPath, { force: true }).catch(() => undefined);
},
};
}
function parseBridgeCliArgs(argv: string[]): { socketPath: string; callerAgentId: string } {
let socketPath: string | null = null;
let callerAgentId: string | null = null;
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--socket") {
socketPath = argv[index + 1] ?? null;
index += 1;
continue;
}
if (arg === "--caller-agent-id") {
callerAgentId = argv[index + 1] ?? null;
index += 1;
continue;
}
}
if (!socketPath?.trim()) {
throw new Error("Missing required --socket <path>");
}
if (!callerAgentId?.trim()) {
throw new Error("Missing required --caller-agent-id <id>");
}
return {
socketPath: socketPath.trim(),
callerAgentId: callerAgentId.trim(),
};
}
export async function runVoiceMcpBridgeCli(argv: string[], logger?: Logger): Promise<void> {
const bridgeLogger = logger ?? pino({ level: "error" });
const parsed = parseBridgeCliArgs(argv);
const socket = net.createConnection(parsed.socketPath);
const stdioTransport = new StdioServerTransport(process.stdin, process.stdout);
let socketBuffer = "";
let stdioStarted = false;
const pendingMcpMessages: JSONRPCMessage[] = [];
let resolveReady: (() => void) | null = null;
let rejectReady: ((error: Error) => void) | null = null;
const readyPromise = new Promise<void>((resolve, reject) => {
resolveReady = () => {
resolveReady = null;
rejectReady = null;
resolve();
};
rejectReady = (error: Error) => {
resolveReady = null;
rejectReady = null;
reject(error);
};
});
const failReady = (message: string) => {
if (rejectReady) {
rejectReady(new Error(message));
}
};
const sendEnvelope = (payload: BridgeEnvelope) => {
socket.write(encodeEnvelope(payload));
};
socket.on("data", (chunk) => {
socketBuffer += chunk.toString("utf8");
while (true) {
const newlineIndex = socketBuffer.indexOf("\n");
if (newlineIndex < 0) break;
const line = socketBuffer.slice(0, newlineIndex).trim();
socketBuffer = socketBuffer.slice(newlineIndex + 1);
if (!line) continue;
let envelope: BridgeEnvelope;
try {
envelope = parseEnvelope(line);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
bridgeLogger.error({ err: error }, "Failed to parse voice MCP bridge envelope");
failReady(`Failed to parse voice MCP bridge envelope: ${message}`);
socket.destroy();
return;
}
if (envelope.type === "ready") {
resolveReady?.();
continue;
}
if (envelope.type === "error") {
failReady(`Voice MCP bridge error: ${envelope.message}`);
socket.destroy();
return;
}
if (envelope.type === "mcp") {
if (!stdioStarted) {
pendingMcpMessages.push(envelope.message);
continue;
}
void stdioTransport.send(envelope.message).catch((error) => {
bridgeLogger.error({ err: error }, "Failed to forward MCP message to stdio transport");
socket.destroy();
});
}
}
});
socket.on("error", (error) => {
bridgeLogger.error({ err: error }, "Voice MCP bridge socket client error");
failReady(`Voice MCP bridge socket client error: ${error.message}`);
});
await new Promise<void>((resolve, reject) => {
socket.once("connect", () => resolve());
socket.once("error", reject);
});
sendEnvelope({ type: "init", callerAgentId: parsed.callerAgentId });
const readyTimeoutPromise = new Promise<void>((_, reject) => {
const timeout = setTimeout(() => {
reject(new Error("Timed out waiting for voice MCP bridge initialization"));
}, 10000);
readyPromise.finally(() => clearTimeout(timeout)).catch(() => undefined);
});
await Promise.race([readyPromise, readyTimeoutPromise]);
stdioTransport.onmessage = (message) => {
sendEnvelope({ type: "mcp", message });
};
stdioTransport.onerror = (error) => {
bridgeLogger.error({ err: error }, "Voice MCP stdio transport error");
socket.destroy();
};
stdioTransport.onclose = () => {
socket.end();
};
await stdioTransport.start();
stdioStarted = true;
for (const message of pendingMcpMessages) {
await stdioTransport.send(message);
}
await new Promise<void>((resolve) => {
socket.once("close", () => resolve());
process.stdin.once("end", () => resolve());
});
await stdioTransport.close().catch(() => undefined);
}

63
packages/server/src/server/voice-permission-policy.test.ts Normal file
View File

@@ -0,0 +1,63 @@
import { describe, expect, test } from "vitest";
import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js";
import { isVoicePermissionAllowed } from "./voice-permission-policy.js";
function buildRequest(partial: Partial<AgentPermissionRequest>): AgentPermissionRequest {
return {
id: "req-1",
provider: "codex",
name: "unknown",
kind: "tool",
...partial,
};
}
describe("isVoicePermissionAllowed", () => {
test("allows speak tool", () => {
const result = isVoicePermissionAllowed(
buildRequest({ name: "speak" })
);
expect(result).toBe(true);
});
test("allows explicit MCP/paseo tool names", () => {
expect(
isVoicePermissionAllowed(buildRequest({ name: "mcp__paseo__create_agent" }))
).toBe(true);
expect(
isVoicePermissionAllowed(buildRequest({ name: "paseo_create_agent" }))
).toBe(true);
});
test("denies non-tool permission kinds", () => {
const result = isVoicePermissionAllowed(
buildRequest({ kind: "mode", name: "mcp__paseo__create_agent" })
);
expect(result).toBe(false);
});
test("allows codextool only when metadata references MCP/paseo", () => {
const allowed = isVoicePermissionAllowed(
buildRequest({
name: "codextool",
metadata: {
questions: [{ question: "Allow codextool to call mcp paseo create_agent?" }],
},
})
);
expect(allowed).toBe(true);
});
test("denies codextool when metadata includes shell-like operations", () => {
const denied = isVoicePermissionAllowed(
buildRequest({
name: "codextool",
metadata: {
questions: [{ question: "Allow codextool to execute shell command?" }],
},
})
);
expect(denied).toBe(false);
});
});

66
packages/server/src/server/voice-permission-policy.ts Normal file
View File

@@ -0,0 +1,66 @@
import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js";
const ALLOWED_TOKEN_SET = ["mcp", "paseo", "speak"];
const DENIED_TOKEN_SET = [
"bash",
"shell",
"terminal",
"command",
"execute",
"edit",
"write",
"read",
"fetch",
"http",
"web",
];
function containsAny(text: string, tokens: readonly string[]): boolean {
return tokens.some((token) => text.includes(token));
}
function stringifyMetadata(metadata: unknown): string {
try {
return JSON.stringify(metadata ?? {})?.toLowerCase() ?? "";
} catch {
return "";
}
}
/**
* Voice assistant policy: allow only MCP/paseo/speak tool requests.
* All non-tool permission requests are denied.
*/
export function isVoicePermissionAllowed(request: AgentPermissionRequest): boolean {
if (request.kind !== "tool") {
return false;
}
const normalizedName = request.name.trim().toLowerCase();
if (!normalizedName) {
return false;
}
if (normalizedName === "speak") {
return true;
}
if (normalizedName.includes("mcp") || normalizedName.includes("paseo")) {
return true;
}
if (normalizedName !== "codextool") {
return false;
}
const metadataText = stringifyMetadata({
metadata: request.metadata ?? null,
input: request.input ?? null,
});
if (!metadataText) {
return false;
}
const mentionsAllowedTooling = containsAny(metadataText, ALLOWED_TOKEN_SET);
const mentionsDeniedTooling = containsAny(metadataText, DENIED_TOKEN_SET);
return mentionsAllowedTooling && !mentionsDeniedTooling;
}

18
packages/server/src/server/voice-types.ts Normal file
View File

@@ -0,0 +1,18 @@
export type VoiceSpeakHandler = (params: {
text: string;
callerAgentId: string;
signal?: AbortSignal;
}) => Promise<void>;
export type VoiceCallerContext = {
childAgentDefaultLabels?: Record<string, string>;
lockedCwd?: string;
allowCustomCwd?: boolean;
enableVoiceTools?: boolean;
};
export type VoiceMcpStdioConfig = {
command: string;
baseArgs: string[];
env?: Record<string, string>;
};

91
packages/server/src/server/websocket-server.ts
View File

@@ -19,8 +19,13 @@ import { Session } from "./session.js";
import type { AgentProvider } from "./agent/agent-sdk-types.js";
import { PushTokenStore } from "./push/token-store.js";
import { PushService } from "./push/push-service.js";
import { VoiceConversationStore } from "./voice-conversation-store.js";
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
import type { LocalSpeechModelId } from "./speech/providers/local/models.js";
import type {
VoiceCallerContext,
VoiceMcpStdioConfig,
VoiceSpeakHandler,
} from "./voice-types.js";
export type AgentMcpTransportFactory = () => Promise<Transport>;
@@ -69,15 +74,26 @@ export class VoiceAssistantWebSocketServer {
private readonly stt: SpeechToTextProvider | null;
private readonly tts: TextToSpeechProvider | null;
private readonly terminalManager: TerminalManager | null;
private readonly voiceConversationStore: VoiceConversationStore;
private readonly dictation: {
finalTimeoutMs?: number;
stt?: SpeechToTextProvider | null;
localModels?: {
modelsDir: string;
defaultModelIds: LocalSpeechModelId[];
};
} | null;
private readonly voice: {
openrouterApiKey?: string | null;
voiceLlmProvider?: AgentProvider | null;
voiceLlmModeId?: string | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
} | null;
private readonly voiceSpeakHandlers = new Map<
string,
VoiceSpeakHandler
>();
private readonly voiceCallerContexts = new Map<string, VoiceCallerContext>();
constructor(
server: HTTPServer,
@@ -92,12 +108,19 @@ export class VoiceAssistantWebSocketServer {
speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null },
terminalManager?: TerminalManager | null,
voice?: {
openrouterApiKey?: string | null;
voiceLlmProvider?: AgentProvider | null;
voiceLlmModeId?: string | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
},
dictation?: {
finalTimeoutMs?: number;
stt?: SpeechToTextProvider | null;
localModels?: {
modelsDir: string;
defaultModelIds: LocalSpeechModelId[];
};
}
) {
this.logger = logger.child({ module: "websocket-server" });
@@ -110,9 +133,6 @@ export class VoiceAssistantWebSocketServer {
this.stt = speech?.stt ?? null;
this.tts = speech?.tts ?? null;
this.terminalManager = terminalManager ?? null;
this.voiceConversationStore = new VoiceConversationStore(
join(paseoHome, "voice-conversations")
);
this.voice = voice ?? null;
this.dictation = dictation ?? null;
@@ -208,25 +228,38 @@ export class VoiceAssistantWebSocketServer {
const clientId = `client-${++this.clientIdCounter}`;
const connectionLogger = this.logger.child({ clientId });
const session = new Session(
const session = new Session({
clientId,
(msg) => {
onMessage: (msg) => {
this.sendToClient(ws, wrapSessionMessage(msg));
},
connectionLogger.child({ module: "session" }),
this.downloadTokenStore,
this.pushTokenStore,
this.paseoHome,
this.agentManager,
this.agentStorage,
this.createAgentMcpTransport,
this.stt,
this.tts,
this.terminalManager,
this.voiceConversationStore,
this.voice ?? undefined,
this.dictation ?? undefined
);
logger: connectionLogger.child({ module: "session" }),
downloadTokenStore: this.downloadTokenStore,
pushTokenStore: this.pushTokenStore,
paseoHome: this.paseoHome,
agentManager: this.agentManager,
agentStorage: this.agentStorage,
createAgentMcpTransport: this.createAgentMcpTransport,
stt: this.stt,
tts: this.tts,
terminalManager: this.terminalManager,
voice: this.voice ?? undefined,
voiceBridge: {
registerVoiceSpeakHandler: (agentId, handler) => {
this.voiceSpeakHandlers.set(agentId, handler);
},
unregisterVoiceSpeakHandler: (agentId) => {
this.voiceSpeakHandlers.delete(agentId);
},
registerVoiceCallerContext: (agentId, context) => {
this.voiceCallerContexts.set(agentId, context);
},
unregisterVoiceCallerContext: (agentId) => {
this.voiceCallerContexts.delete(agentId);
},
},
dictation: this.dictation ?? undefined,
});
this.sessions.set(ws, session);
@@ -263,6 +296,18 @@ export class VoiceAssistantWebSocketServer {
});
}
public resolveVoiceSpeakHandler(
callerAgentId: string
): VoiceSpeakHandler | null {
return this.voiceSpeakHandlers.get(callerAgentId) ?? null;
}
public resolveVoiceCallerContext(
callerAgentId: string
): VoiceCallerContext | null {
return this.voiceCallerContexts.get(callerAgentId) ?? null;
}
private async detachSocket(
ws: WebSocketLike,
connectionLogger: pino.Logger,

80
packages/server/src/shared/messages.ts
View File

@@ -294,11 +294,6 @@ export type AgentStreamEventPayload = z.infer<
// Session Inbound Messages (Session receives these)
// ============================================================================
export const UserTextMessageSchema = z.object({
type: z.literal("user_text"),
text: z.string(),
});
export const VoiceAudioChunkMessageSchema = z.object({
type: z.literal("voice_audio_chunk"),
audio: z.string(), // base64 encoded
@@ -337,23 +332,6 @@ export const UnsubscribeAgentUpdatesMessageSchema = z.object({
subscriptionId: z.string(),
});
export const LoadVoiceConversationRequestMessageSchema = z.object({
type: z.literal("load_voice_conversation_request"),
voiceConversationId: z.string(),
requestId: z.string(),
});
export const ListVoiceConversationsRequestMessageSchema = z.object({
type: z.literal("list_voice_conversations_request"),
requestId: z.string(),
});
export const DeleteVoiceConversationRequestMessageSchema = z.object({
type: z.literal("delete_voice_conversation_request"),
voiceConversationId: z.string(),
requestId: z.string(),
});
export const DeleteAgentRequestMessageSchema = z.object({
type: z.literal("delete_agent_request"),
agentId: z.string(),
@@ -366,10 +344,10 @@ export const ArchiveAgentRequestMessageSchema = z.object({
requestId: z.string(),
});
export const SetVoiceConversationMessageSchema = z.object({
type: z.literal("set_voice_conversation"),
export const SetVoiceModeMessageSchema = z.object({
type: z.literal("set_voice_mode"),
enabled: z.boolean(),
voiceConversationId: z.string().optional(),
voiceAgentId: z.string().optional(),
});
export const SendAgentMessageSchema = z.object({
@@ -876,7 +854,6 @@ export const KillTerminalRequestSchema = z.object({
});
export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
UserTextMessageSchema,
VoiceAudioChunkMessageSchema,
AbortRequestMessageSchema,
AudioPlayedMessageSchema,
@@ -884,12 +861,9 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
FetchAgentRequestMessageSchema,
SubscribeAgentUpdatesMessageSchema,
UnsubscribeAgentUpdatesMessageSchema,
LoadVoiceConversationRequestMessageSchema,
ListVoiceConversationsRequestMessageSchema,
DeleteVoiceConversationRequestMessageSchema,
DeleteAgentRequestMessageSchema,
ArchiveAgentRequestMessageSchema,
SetVoiceConversationMessageSchema,
SetVoiceModeMessageSchema,
SendAgentMessageRequestSchema,
WaitForFinishRequestSchema,
DictationStreamStartMessageSchema,
@@ -1127,15 +1101,6 @@ export const ArtifactMessageSchema = z.object({
}),
});
export const VoiceConversationLoadedMessageSchema = z.object({
type: z.literal("voice_conversation_loaded"),
payload: z.object({
voiceConversationId: z.string(),
messageCount: z.number(),
requestId: z.string(),
}),
});
export const AgentUpdateMessageSchema = z.object({
type: z.literal("agent_update"),
payload: z.discriminatedUnion("kind", [
@@ -1225,30 +1190,6 @@ export const WaitForFinishResponseMessageSchema = z.object({
}),
});
export const ListVoiceConversationsResponseMessageSchema = z.object({
type: z.literal("list_voice_conversations_response"),
payload: z.object({
conversations: z.array(
z.object({
id: z.string(),
lastUpdated: z.string(),
messageCount: z.number(),
})
),
requestId: z.string(),
}),
});
export const DeleteVoiceConversationResponseMessageSchema = z.object({
type: z.literal("delete_voice_conversation_response"),
payload: z.object({
voiceConversationId: z.string(),
success: z.boolean(),
error: z.string().optional(),
requestId: z.string(),
}),
});
export const AgentPermissionRequestMessageSchema = z.object({
type: z.literal("agent_permission_request"),
payload: z.object({
@@ -1679,7 +1620,6 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
RpcErrorMessageSchema,
InitializeAgentResponseMessageSchema,
ArtifactMessageSchema,
VoiceConversationLoadedMessageSchema,
AgentUpdateMessageSchema,
AgentStreamMessageSchema,
AgentStreamSnapshotMessageSchema,
@@ -1691,8 +1631,6 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
SetAgentModelResponseMessageSchema,
SetAgentThinkingResponseMessageSchema,
WaitForFinishResponseMessageSchema,
ListVoiceConversationsResponseMessageSchema,
DeleteVoiceConversationResponseMessageSchema,
AgentPermissionRequestMessageSchema,
AgentPermissionResolvedMessageSchema,
AgentDeletedMessageSchema,
@@ -1737,9 +1675,6 @@ export type TranscriptionResultMessage = z.infer<typeof TranscriptionResultMessa
export type StatusMessage = z.infer<typeof StatusMessageSchema>;
export type RpcErrorMessage = z.infer<typeof RpcErrorMessageSchema>;
export type ArtifactMessage = z.infer<typeof ArtifactMessageSchema>;
export type VoiceConversationLoadedMessage = z.infer<
typeof VoiceConversationLoadedMessageSchema
>;
export type AgentUpdateMessage = z.infer<typeof AgentUpdateMessageSchema>;
export type AgentStreamMessage = z.infer<typeof AgentStreamMessageSchema>;
export type AgentStreamSnapshotMessage = z.infer<
@@ -1758,12 +1693,6 @@ export type SendAgentMessageResponseMessage = z.infer<
export type WaitForFinishResponseMessage = z.infer<
typeof WaitForFinishResponseMessageSchema
>;
export type ListVoiceConversationsResponseMessage = z.infer<
typeof ListVoiceConversationsResponseMessageSchema
>;
export type DeleteVoiceConversationResponseMessage = z.infer<
typeof DeleteVoiceConversationResponseMessageSchema
>;
export type AgentPermissionRequestMessage = z.infer<typeof AgentPermissionRequestMessageSchema>;
export type AgentPermissionResolvedMessage = z.infer<typeof AgentPermissionResolvedMessageSchema>;
export type AgentDeletedMessage = z.infer<typeof AgentDeletedMessageSchema>;
@@ -1778,7 +1707,6 @@ export type InitializeAgentResponseMessage = z.infer<typeof InitializeAgentRespo
export type ActivityLogPayload = z.infer<typeof ActivityLogPayloadSchema>;
// Type exports for inbound message types
export type UserTextMessage = z.infer<typeof UserTextMessageSchema>;
export type VoiceAudioChunkMessage = z.infer<typeof VoiceAudioChunkMessageSchema>;
export type FetchAgentsRequestMessage = z.infer<typeof FetchAgentsRequestMessageSchema>;
export type FetchAgentRequestMessage = z.infer<typeof FetchAgentRequestMessageSchema>;

3
packages/server/src/test-utils/vitest-setup.ts
View File

@@ -1,7 +1,8 @@
import path from "node:path";
import dotenv from "dotenv";
// Load repo-root .env for integration/E2E tests (OpenAI, etc.)
// Load package-local .env.test first for integration/E2E credentials, then repo-root .env fallback.
dotenv.config({ path: path.resolve(process.cwd(), ".env.test"), override: true });
dotenv.config({ path: path.resolve(process.cwd(), "../.env") });
process.env.GIT_TERMINAL_PROMPT = "0";

1
packages/server/src/utils/tool-call-parsers.ts
View File

@@ -28,6 +28,7 @@ const PrincipalParamSchema = z.union([
z.object({ pattern: z.string() }).transform((d) => ({ type: "text" as const, value: d.pattern })),
z.object({ query: z.string() }).transform((d) => ({ type: "text" as const, value: d.query })),
z.object({ url: z.string() }).transform((d) => ({ type: "text" as const, value: d.url })),
z.object({ text: z.string() }).transform((d) => ({ type: "text" as const, value: d.text })),
// Files array (Codex apply_patch)
z.object({ files: z.array(FileEntrySchema).nonempty() }).transform((d) => ({ type: "path" as const, value: d.files[0].path })),
// TodoWrite - show in_progress item or count

32
packages/website/public/schemas/paseo.config.v1.json
View File

@@ -86,8 +86,18 @@
},
"additionalProperties": false
},
"openrouter": {
"$ref": "#/definitions/PaseoConfigV1/properties/providers/properties/openai"
"local": {
"type": "object",
"properties": {
"modelsDir": {
"type": "string",
"minLength": 1
},
"autoDownload": {
"type": "boolean"
}
},
"additionalProperties": false
}
},
"additionalProperties": false
@@ -104,7 +114,8 @@
"provider": {
"type": "string",
"enum": [
"openai"
"openai",
"local"
]
},
"model": {
@@ -129,7 +140,9 @@
"provider": {
"type": "string",
"enum": [
"openrouter"
"claude",
"codex",
"opencode"
]
},
"model": {
@@ -145,7 +158,8 @@
"provider": {
"type": "string",
"enum": [
"openai"
"openai",
"local"
]
},
"model": {
@@ -161,15 +175,13 @@
"provider": {
"type": "string",
"enum": [
"openai"
"openai",
"local"
]
},
"model": {
"type": "string",
"enum": [
"tts-1",
"tts-1-hd"
]
"minLength": 1
},
"voice": {
"type": "string",

21
packages/website/src/routeTree.gen.ts
View File

@@ -13,6 +13,7 @@ import { Route as DocsRouteImport } from './routes/docs'
import { Route as IndexRouteImport } from './routes/index'
import { Route as DocsIndexRouteImport } from './routes/docs/index'
import { Route as DocsWorktreesRouteImport } from './routes/docs/worktrees'
import { Route as DocsVoiceRouteImport } from './routes/docs/voice'
import { Route as DocsSecurityRouteImport } from './routes/docs/security'
import { Route as DocsConfigurationRouteImport } from './routes/docs/configuration'
import { Route as DocsCliRouteImport } from './routes/docs/cli'
@@ -38,6 +39,11 @@ const DocsWorktreesRoute = DocsWorktreesRouteImport.update({
path: '/worktrees',
getParentRoute: () => DocsRoute,
} as any)
const DocsVoiceRoute = DocsVoiceRouteImport.update({
id: '/voice',
path: '/voice',
getParentRoute: () => DocsRoute,
} as any)
const DocsSecurityRoute = DocsSecurityRouteImport.update({
id: '/security',
path: '/security',
@@ -66,6 +72,7 @@ export interface FileRoutesByFullPath {
'/docs/cli': typeof DocsCliRoute
'/docs/configuration': typeof DocsConfigurationRoute
'/docs/security': typeof DocsSecurityRoute
'/docs/voice': typeof DocsVoiceRoute
'/docs/worktrees': typeof DocsWorktreesRoute
'/docs/': typeof DocsIndexRoute
}
@@ -75,6 +82,7 @@ export interface FileRoutesByTo {
'/docs/cli': typeof DocsCliRoute
'/docs/configuration': typeof DocsConfigurationRoute
'/docs/security': typeof DocsSecurityRoute
'/docs/voice': typeof DocsVoiceRoute
'/docs/worktrees': typeof DocsWorktreesRoute
'/docs': typeof DocsIndexRoute
}
@@ -86,6 +94,7 @@ export interface FileRoutesById {
'/docs/cli': typeof DocsCliRoute
'/docs/configuration': typeof DocsConfigurationRoute
'/docs/security': typeof DocsSecurityRoute
'/docs/voice': typeof DocsVoiceRoute
'/docs/worktrees': typeof DocsWorktreesRoute
'/docs/': typeof DocsIndexRoute
}
@@ -98,6 +107,7 @@ export interface FileRouteTypes {
| '/docs/cli'
| '/docs/configuration'
| '/docs/security'
| '/docs/voice'
| '/docs/worktrees'
| '/docs/'
fileRoutesByTo: FileRoutesByTo
@@ -107,6 +117,7 @@ export interface FileRouteTypes {
| '/docs/cli'
| '/docs/configuration'
| '/docs/security'
| '/docs/voice'
| '/docs/worktrees'
| '/docs'
id:
@@ -117,6 +128,7 @@ export interface FileRouteTypes {
| '/docs/cli'
| '/docs/configuration'
| '/docs/security'
| '/docs/voice'
| '/docs/worktrees'
| '/docs/'
fileRoutesById: FileRoutesById
@@ -156,6 +168,13 @@ declare module '@tanstack/react-router' {
preLoaderRoute: typeof DocsWorktreesRouteImport
parentRoute: typeof DocsRoute
}
'/docs/voice': {
id: '/docs/voice'
path: '/voice'
fullPath: '/docs/voice'
preLoaderRoute: typeof DocsVoiceRouteImport
parentRoute: typeof DocsRoute
}
'/docs/security': {
id: '/docs/security'
path: '/security'
@@ -192,6 +211,7 @@ interface DocsRouteChildren {
DocsCliRoute: typeof DocsCliRoute
DocsConfigurationRoute: typeof DocsConfigurationRoute
DocsSecurityRoute: typeof DocsSecurityRoute
DocsVoiceRoute: typeof DocsVoiceRoute
DocsWorktreesRoute: typeof DocsWorktreesRoute
DocsIndexRoute: typeof DocsIndexRoute
}
@@ -201,6 +221,7 @@ const DocsRouteChildren: DocsRouteChildren = {
DocsCliRoute: DocsCliRoute,
DocsConfigurationRoute: DocsConfigurationRoute,
DocsSecurityRoute: DocsSecurityRoute,
DocsVoiceRoute: DocsVoiceRoute,
DocsWorktreesRoute: DocsWorktreesRoute,
DocsIndexRoute: DocsIndexRoute,
}

1
packages/website/src/routes/docs.tsx
View File

@@ -7,6 +7,7 @@ export const Route = createFileRoute('/docs')({
const navigation = [
{ name: 'Getting started', href: '/docs' },
{ name: 'Voice', href: '/docs/voice' },
{ name: 'Git worktrees', href: '/docs/worktrees' },
{ name: 'CLI', href: '/docs/cli' },
{ name: 'Configuration', href: '/docs/configuration' },

25
packages/website/src/routes/docs/configuration.tsx
View File

@@ -67,8 +67,7 @@ function Configuration() {
"$schema": "https://paseo.sh/schemas/paseo.config.v1.json",
"version": 1,
"providers": {
"openai": { "apiKey": "..." },
"openrouter": { "apiKey": "..." }
"openai": { "apiKey": "..." }
},
"daemon": {
"listen": "127.0.0.1:6767",
@@ -79,13 +78,32 @@ function Configuration() {
</pre>
</section>
<section className="space-y-4">
<h2 className="text-xl font-medium">Voice</h2>
<p className="text-white/60 leading-relaxed">
Voice is configured through <code className="font-mono">features.dictation</code> and{' '}
<code className="font-mono">features.voiceMode</code>, with provider credentials under{' '}
<code className="font-mono">providers</code>.
</p>
<p className="text-white/60 leading-relaxed">
For voice philosophy, architecture, and complete local/OpenAI setup examples, see{' '}
<a href="/docs/voice" className="underline hover:text-white/80">Voice docs</a>.
</p>
</section>
<section className="space-y-4">
<h2 className="text-xl font-medium">Common env vars</h2>
<ul className="text-white/60 space-y-2 list-disc list-inside">
<li><code className="font-mono">PASEO_HOME</code> — set Paseo home directory</li>
<li><code className="font-mono">PASEO_LISTEN</code> — override <code className="font-mono">daemon.listen</code></li>
<li><code className="font-mono">PASEO_ALLOWED_HOSTS</code> — override/extend <code className="font-mono">daemon.allowedHosts</code></li>
<li><code className="font-mono">OPENAI_API_KEY</code> and <code className="font-mono">OPENROUTER_API_KEY</code> — override provider keys</li>
<li><code className="font-mono">OPENAI_API_KEY</code> — override OpenAI provider key</li>
<li><code className="font-mono">PASEO_VOICE_LLM_PROVIDER</code> — override voice LLM provider (<code className="font-mono">claude</code>, <code className="font-mono">codex</code>, <code className="font-mono">opencode</code>)</li>
<li><code className="font-mono">PASEO_DICTATION_STT_PROVIDER</code>, <code className="font-mono">PASEO_VOICE_STT_PROVIDER</code>, <code className="font-mono">PASEO_VOICE_TTS_PROVIDER</code> — override voice provider selection (<code className="font-mono">local</code> or <code className="font-mono">openai</code>)</li>
<li><code className="font-mono">PASEO_LOCAL_MODELS_DIR</code> and <code className="font-mono">PASEO_LOCAL_AUTO_DOWNLOAD</code> — control local model directory and download behavior</li>
<li><code className="font-mono">PASEO_DICTATION_LOCAL_STT_MODEL</code> — override local dictation STT model</li>
<li><code className="font-mono">PASEO_VOICE_LOCAL_STT_MODEL</code>, <code className="font-mono">PASEO_VOICE_LOCAL_TTS_MODEL</code> — override local voice STT/TTS models</li>
<li><code className="font-mono">PASEO_VOICE_LOCAL_TTS_SPEAKER_ID</code>, <code className="font-mono">PASEO_VOICE_LOCAL_TTS_SPEED</code> — optional local voice TTS tuning</li>
</ul>
</section>
@@ -101,4 +119,3 @@ function Configuration() {
</div>
)
}

17
packages/website/src/routes/docs/index.tsx
View File

@@ -87,20 +87,25 @@ function GettingStarted() {
<section className="space-y-4">
<h2 className="text-xl font-medium">Voice Setup</h2>
<p className="text-white/60">
Voice features currently require an OpenAI API key. Set it as an environment variable before running the server:
Paseo includes first-class voice support with a local-first architecture and configurable speech
providers.
</p>
<div className="bg-card border border-border rounded-lg p-4 font-mono text-sm">
<span className="text-muted-foreground select-none">$ </span>
<span>export OPENAI_API_KEY=your-key-here</span>
</div>
<p className="text-white/60">
Local voice support is coming soon.
For architecture, local model behavior, and provider configuration, see the Voice docs page.
</p>
<a href="/docs/voice" className="underline hover:text-white/80">
Voice docs
</a>
</section>
<section className="space-y-4">
<h2 className="text-xl font-medium">Next</h2>
<ul className="text-white/60 space-y-2 list-disc list-inside">
<li>
<a href="/docs/voice" className="underline hover:text-white/80">
Voice
</a>
</li>
<li>
<a href="/docs/configuration" className="underline hover:text-white/80">
Configuration

126
packages/website/src/routes/docs/voice.tsx Normal file
View File

@@ -0,0 +1,126 @@
import { createFileRoute } from '@tanstack/react-router'
export const Route = createFileRoute('/docs/voice')({
head: () => ({
meta: [
{ title: 'Voice - Paseo Docs' },
{
name: 'description',
content: 'Paseo voice architecture, local-first model execution, and provider configuration.',
},
],
}),
component: VoiceDocs,
})
function VoiceDocs() {
return (
<div className="space-y-8">
<div>
<h1 className="text-3xl font-medium font-title mb-4">Voice</h1>
<p className="text-white/60 leading-relaxed">
Paseo has first-class voice support for dictation and realtime conversations with your coding
environment.
</p>
</div>
<section className="space-y-4">
<h2 className="text-xl font-medium">Philosophy</h2>
<p className="text-white/60 leading-relaxed">
Voice is local-first. You can run speech fully on-device, or choose OpenAI for speech features.
For voice reasoning/orchestration, Paseo reuses agent providers already installed and authenticated
on your machine.
</p>
<p className="text-white/60 leading-relaxed">
This keeps credentials and execution in your environment and avoids introducing a separate
cloud-only voice stack.
</p>
</section>
<section className="space-y-4">
<h2 className="text-xl font-medium">Architecture</h2>
<ul className="text-white/60 space-y-2 list-disc list-inside">
<li>Speech I/O: STT and TTS providers per feature (<code className="font-mono">local</code> or <code className="font-mono">openai</code>)</li>
<li>Local speech runtime: ONNX models executed on CPU by default</li>
<li>Voice LLM orchestration: hidden agent session using your configured provider (<code className="font-mono">claude</code>, <code className="font-mono">codex</code>, or <code className="font-mono">opencode</code>)</li>
<li>Tooling path: MCP stdio bridge for voice tools and agent control</li>
</ul>
</section>
<section className="space-y-4">
<h2 className="text-xl font-medium">Local Speech</h2>
<p className="text-white/60 leading-relaxed">
Local speech defaults to model IDs <code className="font-mono">parakeet-tdt-0.6b-v3-int8</code>{' '}
(STT) and <code className="font-mono">pocket-tts-onnx-int8</code> (TTS).
</p>
<p className="text-white/60 leading-relaxed">
Missing models are downloaded at daemon startup into{' '}
<code className="font-mono">$PASEO_HOME/models/local-speech</code> when auto-download is enabled.
Downloads happen only for missing files.
</p>
<pre className="bg-card border border-border rounded-lg p-4 font-mono text-sm overflow-x-auto text-white/80">
{`{
"version": 1,
"features": {
"dictation": { "stt": { "provider": "local", "model": "parakeet-tdt-0.6b-v3-int8" } },
"voiceMode": {
"llm": { "provider": "claude", "model": "haiku" },
"stt": { "provider": "local", "model": "parakeet-tdt-0.6b-v3-int8" },
"tts": { "provider": "local", "model": "pocket-tts-onnx-int8" }
}
},
"providers": {
"local": {
"modelsDir": "~/.paseo/models/local-speech",
"autoDownload": true
}
}
}`}
</pre>
</section>
<section className="space-y-4">
<h2 className="text-xl font-medium">OpenAI Speech Option</h2>
<p className="text-white/60 leading-relaxed">
You can switch dictation, voice STT, and voice TTS to OpenAI by setting provider fields to{' '}
<code className="font-mono">openai</code> and providing <code className="font-mono">OPENAI_API_KEY</code>.
</p>
<pre className="bg-card border border-border rounded-lg p-4 font-mono text-sm overflow-x-auto text-white/80">
{`{
"version": 1,
"features": {
"dictation": { "stt": { "provider": "openai" } },
"voiceMode": {
"stt": { "provider": "openai" },
"tts": { "provider": "openai" }
}
},
"providers": {
"openai": { "apiKey": "..." }
}
}`}
</pre>
</section>
<section className="space-y-4">
<h2 className="text-xl font-medium">Environment Variables</h2>
<ul className="text-white/60 space-y-2 list-disc list-inside">
<li><code className="font-mono">OPENAI_API_KEY</code> — OpenAI speech credentials</li>
<li><code className="font-mono">PASEO_VOICE_LLM_PROVIDER</code> — voice agent provider override</li>
<li><code className="font-mono">PASEO_LOCAL_MODELS_DIR</code>, <code className="font-mono">PASEO_LOCAL_AUTO_DOWNLOAD</code> — local model storage and download policy</li>
<li><code className="font-mono">PASEO_DICTATION_LOCAL_STT_MODEL</code> — local dictation STT model ID</li>
<li><code className="font-mono">PASEO_VOICE_LOCAL_STT_MODEL</code>, <code className="font-mono">PASEO_VOICE_LOCAL_TTS_MODEL</code> — local voice STT/TTS model IDs</li>
<li><code className="font-mono">PASEO_VOICE_LOCAL_TTS_SPEAKER_ID</code>, <code className="font-mono">PASEO_VOICE_LOCAL_TTS_SPEED</code> — optional local voice TTS tuning</li>
</ul>
</section>
<section className="space-y-4">
<h2 className="text-xl font-medium">Operational Notes</h2>
<p className="text-white/60 leading-relaxed">
Realtime voice can launch and control agents. Treat voice prompts with the same care as direct
agent instructions, especially when specifying working directories or destructive operations.
</p>
</section>
</div>
)
}

4
packages/website/src/routes/index.tsx
View File

@@ -286,8 +286,8 @@ function FAQ() {
<FAQItem question="Is this free?">
Paseo is free and open source. It wraps CLI tools like Claude Code and
Codex, which you'll need to have installed and configured with your
own credentials. Voice features currently require an OpenAI API key,
but local voice is coming soon.
own credentials. Voice is local-first by default and can optionally use
OpenAI speech providers if you configure them.
</FAQItem>
<FAQItem question="Does my code leave my machine?">
Paseo itself doesn't send your code anywhere. Agents run locally and