puter/paseo
1
0
Fork 0
mirror of https://github.com/getpaseo/paseo.git synced 2026-07-29 12:01:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update files

Browse Source
This commit is contained in:
Mohamed Boudra
2026-02-06 20:12:55 +07:00
parent 0726f4144f
commit a44038a048
12 changed files with 175 additions and 641 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/server/src/server/agent/agent-mcp.e2e.test.ts
View File

@@ -90,7 +90,6 @@ describe("agent MCP end-to-end (offline)", () => {
mcpDebug: false,
agentClients: createTestAgentClients(),
agentStoragePath: path.join(paseoHome, "agents"),
openrouterApiKey: null,
};
const daemon = await createPaseoDaemon(daemonConfig, pino({ level: "silent" }));

19
packages/server/src/server/agent/provider-manifest.ts
View File

@@ -7,6 +7,11 @@ export interface AgentProviderDefinition {
description: string;
defaultModeId: string | null;
modes: AgentMode[];
voice?: {
enabled: boolean;
defaultModeId: string;
defaultModel?: string;
};
}
const CLAUDE_MODES: AgentMode[] = [
@@ -68,6 +73,11 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [
"Anthropic's multi-tool assistant with MCP support, streaming, and deep reasoning",
defaultModeId: "default",
modes: CLAUDE_MODES,
voice: {
enabled: true,
defaultModeId: "default",
defaultModel: "haiku",
},
},
{
id: "codex",
@@ -76,6 +86,11 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [
"OpenAI's Codex workspace agent with sandbox controls and optional network access",
defaultModeId: "auto",
modes: CODEX_MODES,
voice: {
enabled: true,
defaultModeId: "read-only",
defaultModel: "gpt-5.2-mini",
},
},
{
id: "opencode",
@@ -84,6 +99,10 @@ export const AGENT_PROVIDER_DEFINITIONS: AgentProviderDefinition[] = [
"Open-source coding assistant with multi-provider model support",
defaultModeId: "default",
modes: OPENCODE_MODES,
voice: {
enabled: true,
defaultModeId: "default",
},
},
];

1
packages/server/src/server/bootstrap.smoke.test.ts
View File

@@ -60,7 +60,6 @@ describe("paseo daemon bootstrap", () => {
voiceSttProvider: "openai",
voiceTtsProvider: "openai",
},
openrouterApiKey: null,
};
try {

81
packages/server/src/server/bootstrap.ts
View File

@@ -74,13 +74,12 @@ import type {
AgentClient,
AgentProvider,
} from "./agent/agent-sdk-types.js";
import { AGENT_PROVIDER_DEFINITIONS } from "./agent/provider-manifest.js";
import { acquirePidLock, releasePidLock } from "./pid-lock.js";
import { isHostAllowed, type AllowedHostsConfig } from "./allowed-hosts.js";
import { createVoiceMcpBridgeSocketServer, type VoiceMcpBridgeSocketServer } from "./voice-mcp-bridge.js";
type AgentMcpTransportMap = Map<string, StreamableHTTPServerTransport>;
type VoiceAgentProvider = "claude" | "codex" | "opencode";
const VOICE_AGENT_FALLBACK_ORDER: VoiceAgentProvider[] = ["claude", "codex", "opencode"];
function resolveVoiceMcpBridgeCommand(logger: Logger): { command: string; baseArgs: string[] } {
const explicit = process.env.PASEO_BIN_PATH?.trim();
@@ -147,8 +146,7 @@ export type PaseoDaemonConfig = {
appBaseUrl?: string;
openai?: PaseoOpenAIConfig;
speech?: PaseoSpeechConfig;
openrouterApiKey?: string | null;
voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null;
voiceLlmProvider?: AgentProvider | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
dictationFinalTimeoutMs?: number;
@@ -313,21 +311,24 @@ export async function createPaseoDaemon(
const requestedVoiceLlmProvider = config.voiceLlmProvider ?? null;
const voiceLlmProviderExplicit = config.voiceLlmProviderExplicit ?? false;
const voiceEnabledProviders = AGENT_PROVIDER_DEFINITIONS
.filter((definition) => definition.voice?.enabled)
.map((definition) => definition.id as AgentProvider);
logger.info(
{
requestedVoiceLlmProvider,
voiceLlmProviderExplicit,
voiceEnabledProviders,
},
"Voice LLM provider reconciliation started"
);
const providerClients = createAllClients(logger);
const voiceLlmAvailability: Record<VoiceAgentProvider, boolean> = {
claude: false,
codex: false,
opencode: false,
};
for (const provider of VOICE_AGENT_FALLBACK_ORDER) {
Object.assign(providerClients, config.agentClients);
const voiceLlmAvailability = Object.fromEntries(
voiceEnabledProviders.map((provider) => [provider, false])
) as Record<AgentProvider, boolean>;
for (const provider of voiceEnabledProviders) {
try {
voiceLlmAvailability[provider] = await providerClients[provider].isAvailable();
} catch (error) {
@@ -336,21 +337,17 @@ export async function createPaseoDaemon(
}
}
const voiceLlmDefaultProvider =
VOICE_AGENT_FALLBACK_ORDER.find((provider) => voiceLlmAvailability[provider]) ?? null;
if (requestedVoiceLlmProvider === "openrouter") {
const openrouterApiKey =
config.openrouterApiKey ?? process.env.OPENROUTER_API_KEY ?? null;
if (!openrouterApiKey) {
logger.error("voiceMode.llm.provider is openrouter but no OpenRouter API key is configured");
throw new Error("Missing OpenRouter API key for voiceMode.llm.provider=openrouter");
let resolvedVoiceLlmProvider: AgentProvider | null = null;
if (requestedVoiceLlmProvider) {
if (!voiceEnabledProviders.includes(requestedVoiceLlmProvider)) {
logger.error(
{ provider: requestedVoiceLlmProvider, voiceEnabledProviders },
"Configured voice LLM provider does not support voice mode"
);
throw new Error(
`Configured voice LLM provider '${requestedVoiceLlmProvider}' does not support voice mode`
);
}
} else if (
requestedVoiceLlmProvider === "claude" ||
requestedVoiceLlmProvider === "codex" ||
requestedVoiceLlmProvider === "opencode"
) {
if (!voiceLlmAvailability[requestedVoiceLlmProvider]) {
logger.error(
{ provider: requestedVoiceLlmProvider, voiceLlmAvailability },
@@ -358,20 +355,40 @@ export async function createPaseoDaemon(
);
throw new Error(`Configured voice LLM provider '${requestedVoiceLlmProvider}' is unavailable`);
}
} else if (!voiceLlmDefaultProvider) {
resolvedVoiceLlmProvider = requestedVoiceLlmProvider;
} else {
resolvedVoiceLlmProvider =
voiceEnabledProviders.find((provider) => voiceLlmAvailability[provider]) ?? null;
}
if (!resolvedVoiceLlmProvider) {
logger.error(
{ requestedVoiceLlmProvider, voiceLlmAvailability },
"No local voice LLM provider available for fallback"
"No voice LLM provider available"
);
throw new Error("No local voice LLM provider available (claude/codex/opencode)");
throw new Error("No voice LLM provider available");
}
const resolvedVoiceProviderDefinition = AGENT_PROVIDER_DEFINITIONS.find(
(definition) => definition.id === resolvedVoiceLlmProvider
);
if (!resolvedVoiceProviderDefinition?.voice?.enabled) {
throw new Error(
`Provider '${resolvedVoiceLlmProvider}' is missing voice metadata in agent registry`
);
}
const resolvedVoiceLlmModeId = resolvedVoiceProviderDefinition.voice.defaultModeId;
const resolvedVoiceLlmModel =
config.voiceLlmModel ?? resolvedVoiceProviderDefinition.voice.defaultModel ?? null;
logger.info(
{
requestedVoiceLlmProvider,
voiceLlmProviderExplicit,
resolvedVoiceLlmProvider,
resolvedVoiceLlmModeId,
resolvedVoiceLlmModel,
voiceLlmAvailability,
voiceLlmDefaultProvider,
},
"Voice LLM provider reconciliation completed"
);
@@ -926,12 +943,10 @@ export async function createPaseoDaemon(
{ stt: sttService, tts: ttsService },
terminalManager,
{
openrouterApiKey: config.openrouterApiKey ?? null,
voiceLlmProvider: config.voiceLlmProvider ?? null,
voiceLlmProvider: resolvedVoiceLlmProvider,
voiceLlmModeId: resolvedVoiceLlmModeId,
voiceLlmProviderExplicit,
voiceLlmDefaultProvider,
voiceLlmModel: config.voiceLlmModel ?? null,
voiceLlmAvailability,
voiceLlmModel: resolvedVoiceLlmModel,
voiceAgentMcpStdio: {
command: voiceMcpBridgeCommand.command,
baseArgs: [

20
packages/server/src/server/config.ts
View File

@@ -4,6 +4,8 @@ import type { PaseoDaemonConfig } from "./bootstrap.js";
import type { STTConfig } from "./speech/providers/openai/stt.js";
import type { TTSConfig } from "./speech/providers/openai/tts.js";
import { loadPersistedConfig } from "./persisted-config.js";
import type { AgentProvider } from "./agent/agent-sdk-types.js";
import { AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js";
import {
mergeAllowedHosts,
parseAllowedHostsEnv,
@@ -13,15 +15,6 @@ import {
const DEFAULT_PORT = 6767;
const DEFAULT_RELAY_ENDPOINT = "relay.paseo.sh:443";
const DEFAULT_APP_BASE_URL = "https://app.paseo.sh";
const VOICE_LLM_PROVIDER_IDS = [
"openrouter",
"local-agent",
"claude",
"codex",
"opencode",
] as const;
type VoiceLlmProviderId = (typeof VOICE_LLM_PROVIDER_IDS)[number];
function getDefaultListen(): string {
// Main HTTP server defaults to TCP
return `127.0.0.1:${DEFAULT_PORT}`;
@@ -97,7 +90,7 @@ function parseSpeechProviderId(value: unknown): "openai" | "local" | null {
return null;
}
function parseVoiceLlmProviderId(value: unknown): VoiceLlmProviderId | null {
function parseVoiceLlmProviderId(value: unknown): AgentProvider | null {
if (typeof value !== "string") {
return null;
}
@@ -105,8 +98,8 @@ function parseVoiceLlmProviderId(value: unknown): VoiceLlmProviderId | null {
if (!normalized) {
return null;
}
return (VOICE_LLM_PROVIDER_IDS as readonly string[]).includes(normalized)
? (normalized as VoiceLlmProviderId)
return (AGENT_PROVIDER_IDS as readonly string[]).includes(normalized)
? (normalized as AgentProvider)
: null;
}
@@ -265,8 +258,6 @@ export function loadConfig(
}
: undefined;
const openrouterApiKey =
env.OPENROUTER_API_KEY ?? persisted.providers?.openrouter?.apiKey ?? null;
const envVoiceLlmProvider = parseVoiceLlmProviderId(env.PASEO_VOICE_LLM_PROVIDER);
const persistedVoiceLlmProvider = parseVoiceLlmProviderId(
persisted.features?.voiceMode?.llm?.provider
@@ -299,7 +290,6 @@ export function loadConfig(
voiceTtsProvider,
...(sherpaOnnx ? { sherpaOnnx } : {}),
},
openrouterApiKey,
voiceLlmProvider,
voiceLlmProviderExplicit,
voiceLlmModel,

24
packages/server/src/server/daemon-client.e2e.test.ts
View File

@@ -574,16 +574,16 @@ describe("daemon client E2E", () => {
120000
);
test.runIf(Boolean(process.env.OPENROUTER_API_KEY))(
"streams session activity logs and chunks",
test(
"does not process non-voice LLM turns via OpenRouter",
async () => {
await ctx.client.setVoiceConversation(false);
let sawAssistantChunk = false;
let sawTranscriptLog = false;
let sawAssistantChunk = false;
let sawAssistantLog = false;
const completion = waitForSignal(60000, (resolve) => {
const transcriptSeen = waitForSignal(60000, (resolve) => {
const unsubscribeChunk = ctx.client.on("assistant_chunk", (message) => {
if (message.type !== "assistant_chunk") {
return;
@@ -591,9 +591,6 @@ describe("daemon client E2E", () => {
if (message.payload.chunk.length > 0) {
sawAssistantChunk = true;
}
if (sawAssistantChunk && sawTranscriptLog && sawAssistantLog) {
resolve();
}
});
const unsubscribeActivity = ctx.client.on("activity_log", (message) => {
@@ -602,13 +599,11 @@ describe("daemon client E2E", () => {
}
if (message.payload.type === "transcript") {
sawTranscriptLog = true;
resolve();
}
if (message.payload.type === "assistant") {
sawAssistantLog = true;
}
if (sawAssistantChunk && sawTranscriptLog && sawAssistantLog) {
resolve();
}
});
return () => {
@@ -618,9 +613,14 @@ describe("daemon client E2E", () => {
});
await ctx.client.sendUserMessage("Say 'hello' and nothing else");
await completion;
await transcriptSeen;
await new Promise((resolve) => setTimeout(resolve, 1500));
expect(sawTranscriptLog).toBe(true);
expect(sawAssistantChunk).toBe(false);
expect(sawAssistantLog).toBe(false);
},
120000
90000
);
speechTest(

6
packages/server/src/server/persisted-config.ts
View File

@@ -1,6 +1,7 @@
import { existsSync, readFileSync, writeFileSync } from "node:fs";
import path from "node:path";
import { z } from "zod";
import { AGENT_PROVIDER_IDS } from "./agent/provider-manifest.js";
const LogConfigSchema = z
.object({
@@ -41,7 +42,6 @@ const SherpaOnnxProviderSchema = z
const ProvidersSchema = z
.object({
openai: ProviderCredentialsSchema.optional(),
openrouter: ProviderCredentialsSchema.optional(),
sherpaOnnx: SherpaOnnxProviderSchema.optional(),
})
.strict();
@@ -74,9 +74,7 @@ const FeatureVoiceModeSchema = z
.object({
llm: z
.object({
provider: z
.enum(["openrouter", "local-agent", "claude", "codex", "opencode"])
.optional(),
provider: z.enum(AGENT_PROVIDER_IDS as [string, ...string[]]).optional(),
model: z.string().min(1).optional(),
})
.strict()

582
packages/server/src/server/session.ts
View File

@@ -1,17 +1,10 @@
import { v4 as uuidv4 } from "uuid";
import { readFile, mkdir, writeFile, stat } from "fs/promises";
import { mkdir, stat } from "fs/promises";
import { exec } from "child_process";
import { promisify, inspect } from "util";
import { promisify } from "util";
import { join, resolve, sep } from "path";
import invariant from "tiny-invariant";
import { z } from "zod";
import { streamText, stepCountIs } from "ai";
import type { ToolSet } from "ai";
import type { ModelMessage } from "@ai-sdk/provider-utils";
import {
createOpenRouter,
OpenRouterProviderOptions,
} from "@openrouter/ai-sdk-provider";
import {
serializeAgentStreamEvent,
type AgentSnapshotPayload,
@@ -29,8 +22,6 @@ import {
} from "./messages.js";
import type { TerminalManager } from "../terminal/terminal-manager.js";
import { parseAndHighlightDiff, type ParsedDiffFile } from "./utils/diff-highlighter.js";
import { getSystemPrompt } from "./agent/system-prompt.js";
import { getAllTools } from "./agent/llm-openai.js";
import { TTSManager } from "./agent/tts-manager.js";
import { STTManager } from "./agent/stt-manager.js";
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
@@ -48,8 +39,6 @@ import { experimental_createMCPClient } from "ai";
import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
export type AgentMcpTransportFactory = () => Promise<Transport>;
type VoiceLlmProvider = "openrouter" | "local-agent" | "claude" | "codex" | "opencode";
type VoiceAgentProvider = Exclude<VoiceLlmProvider, "openrouter" | "local-agent">;
type VoiceSpeakHandler = (params: { text: string; callerAgentId: string; signal?: AbortSignal }) => Promise<void>;
type VoiceCallerContext = {
childAgentDefaultLabels?: Record<string, string>;
@@ -144,16 +133,6 @@ const RESTART_EXIT_DELAY_MS = 250;
* Uses Claude Haiku for speed and cost efficiency.
*/
const AUTO_GEN_MODEL = "haiku";
const VOICE_AGENT_FALLBACK_ORDER: VoiceAgentProvider[] = ["claude", "codex", "opencode"];
const VOICE_AGENT_DEFAULT_MODE: Record<VoiceAgentProvider, string> = {
claude: "default",
codex: "read-only",
opencode: "default",
};
const VOICE_AGENT_DEFAULT_MODEL: Partial<Record<VoiceAgentProvider, string>> = {
claude: "haiku",
codex: "gpt-5.2-mini",
};
const VOICE_AGENT_SYSTEM_INSTRUCTION = [
"You are the Paseo voice assistant.",
"The user cannot see your chat messages or tool calls.",
@@ -210,18 +189,6 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round(
);
const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
/**
* Type for present_artifact tool arguments
*/
interface PresentArtifactArgs {
type: "markdown" | "diff" | "image" | "code";
source:
| { type: "file"; path: string }
| { type: "command_output"; command: string }
| { type: "text"; text: string };
title: string;
}
interface AudioBufferState {
chunks: Buffer[];
format: string;
@@ -335,7 +302,6 @@ export class Session {
>();
// Conversation history
private messages: ModelMessage[] = [];
private turnIndex = 0;
// Per-session managers
@@ -368,12 +334,10 @@ export class Session {
} | null = null;
private readonly terminalManager: TerminalManager | null;
private terminalSubscriptions: Map<string, () => void> = new Map();
private readonly openrouterApiKey: string | null;
private readonly voiceLlmProvider: VoiceLlmProvider | null;
private readonly voiceLlmProvider: AgentProvider | null;
private readonly voiceLlmModeId: string | null;
private readonly voiceLlmProviderExplicit: boolean;
private readonly voiceLlmDefaultProvider: VoiceAgentProvider | null;
private readonly voiceLlmModel: string | null;
private readonly voiceLlmAvailability: Record<VoiceAgentProvider, boolean> | null;
private readonly voiceAgentMcpStdio: VoiceMcpStdioConfig | null;
private readonly registerVoiceSpeakHandler?: (
agentId: string,
@@ -401,12 +365,10 @@ export class Session {
tts: TextToSpeechProvider | null,
terminalManager: TerminalManager | null,
voice?: {
openrouterApiKey?: string | null;
voiceLlmProvider?: VoiceLlmProvider | null;
voiceLlmProvider?: AgentProvider | null;
voiceLlmModeId?: string | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmDefaultProvider?: VoiceAgentProvider | null;
voiceLlmModel?: string | null;
voiceLlmAvailability?: Record<VoiceAgentProvider, boolean> | null;
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
},
voiceBridge?: {
@@ -430,12 +392,10 @@ export class Session {
this.agentStorage = agentStorage;
this.createAgentMcpTransport = createAgentMcpTransport;
this.terminalManager = terminalManager;
this.openrouterApiKey = voice?.openrouterApiKey ?? null;
this.voiceLlmProvider = voice?.voiceLlmProvider ?? null;
this.voiceLlmModeId = voice?.voiceLlmModeId ?? null;
this.voiceLlmProviderExplicit = voice?.voiceLlmProviderExplicit ?? false;
this.voiceLlmDefaultProvider = voice?.voiceLlmDefaultProvider ?? null;
this.voiceLlmModel = voice?.voiceLlmModel ?? null;
this.voiceLlmAvailability = voice?.voiceLlmAvailability ?? null;
this.voiceAgentMcpStdio = voice?.voiceAgentMcpStdio ?? null;
this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler;
this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler;
@@ -467,28 +427,6 @@ export class Session {
this.sessionLogger.info("Session created");
}
private escapeXmlText(value: string): string {
return value
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;");
}
private escapeXmlAttribute(value: string): string {
return this.escapeXmlText(value)
.replace(/"/g, "&quot;")
.replace(/'/g, "&apos;");
}
private formatVoiceTranscriptionForLLM(text: string): string {
const trimmed = text.trim();
const focusedAgentId = this.clientActivity?.focusedAgentId ?? null;
const focusedAttr = focusedAgentId
? ` focused-agent-id="${this.escapeXmlAttribute(focusedAgentId)}"`
: "";
return `<voice-transcription${focusedAttr}>${this.escapeXmlText(trimmed)}</voice-transcription>`;
}
/**
* Get the client's current activity state
*/
@@ -1195,19 +1133,6 @@ export class Session {
voiceConversationId: string,
requestId: string
): Promise<void> {
if (this.voiceLlmProvider === "openrouter") {
this.voiceAssistantAgentId = null;
this.messages = [];
this.emit({
type: "voice_conversation_loaded",
payload: {
voiceConversationId,
messageCount: 0,
requestId,
},
});
return;
}
this.voiceAssistantAgentId = voiceConversationId;
this.emit({
@@ -1459,36 +1384,19 @@ export class Session {
}
this.isVoiceMode = true;
if (this.voiceLlmProvider !== "openrouter") {
this.voiceAssistantAgentId = voiceConversationId;
this.sessionLogger.info(
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
"Voice conversation enabled (agent-backed)"
);
return;
}
// OpenRouter voice mode is always ephemeral.
this.voiceAssistantAgentId = null;
this.messages = [];
this.voiceAssistantAgentId = voiceConversationId;
this.sessionLogger.info(
{ messageCount: this.messages.length },
"Voice conversation enabled"
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
"Voice conversation enabled (agent-backed)"
);
return;
}
this.isVoiceMode = false;
if (this.voiceLlmProvider !== "openrouter") {
this.sessionLogger.info(
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
"Voice conversation disabled (agent-backed)"
);
return;
}
this.voiceAssistantAgentId = null;
this.sessionLogger.info("Voice conversation disabled");
this.sessionLogger.info(
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
"Voice conversation disabled (agent-backed)"
);
}
/**
@@ -4342,10 +4250,7 @@ export class Session {
},
});
// Add to conversation
this.messages.push({ role: "user", content: text });
// Process through LLM (TTS enabled in voice mode for voice conversations)
// Process through LLM (voice path is agent-backed only)
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode, text);
await this.currentStreamPromise;
}
@@ -4612,14 +4517,6 @@ export class Session {
},
});
// Add to conversation
this.messages.push({
role: "user",
content: this.isVoiceMode
? this.formatVoiceTranscriptionForLLM(result.text)
: result.text,
});
// Set phase to LLM and process (TTS enabled in voice mode for voice conversations)
this.clearSpeechInProgress("transcription complete");
this.setPhase("llm");
@@ -4642,52 +4539,6 @@ export class Session {
}
}
/**
* Resolve the effective voice LLM provider.
* - explicit provider => strict
* - local-agent / unset => fallback order
*/
private resolveVoiceAgentProvider(): VoiceAgentProvider {
const configured = this.voiceLlmProvider;
const availability = this.voiceLlmAvailability ?? {
claude: true,
codex: true,
opencode: true,
};
if (configured === "openrouter") {
throw new Error("voiceLlmProvider=openrouter cannot be used in local-agent flow");
}
if (configured === "claude" || configured === "codex" || configured === "opencode") {
if (!availability[configured]) {
throw new Error(`Configured voice LLM provider '${configured}' is unavailable`);
}
return configured;
}
const fallbackOrder =
this.voiceLlmDefaultProvider && availability[this.voiceLlmDefaultProvider]
? [this.voiceLlmDefaultProvider, ...VOICE_AGENT_FALLBACK_ORDER.filter((id) => id !== this.voiceLlmDefaultProvider)]
: VOICE_AGENT_FALLBACK_ORDER;
for (const provider of fallbackOrder) {
if (availability[provider]) {
return provider;
}
}
throw new Error("No local voice LLM provider is available (claude/codex/opencode)");
}
private getVoiceAgentModel(provider: VoiceAgentProvider): string | undefined {
const configured = this.voiceLlmModel?.trim();
if (configured) {
return configured;
}
return VOICE_AGENT_DEFAULT_MODEL[provider];
}
private async ensureVoiceAssistantAgent(): Promise<string> {
if (this.voiceAssistantAgentId) {
const existing = this.agentManager.getAgent(this.voiceAssistantAgentId);
@@ -4706,7 +4557,10 @@ export class Session {
}
}
const provider = this.resolveVoiceAgentProvider();
const provider = this.voiceLlmProvider;
if (!provider) {
throw new Error("Voice LLM provider is not configured");
}
const voiceAgentId = this.voiceAssistantAgentId ?? uuidv4();
const cwd = join(this.paseoHome, "voice-agent-workspace");
await mkdir(cwd, { recursive: true });
@@ -4716,11 +4570,11 @@ export class Session {
throw new Error("Voice MCP stdio bridge is not configured");
}
const model = this.getVoiceAgentModel(provider);
const model = this.voiceLlmModel?.trim() || undefined;
const config: AgentSessionConfig = {
provider,
cwd,
modeId: VOICE_AGENT_DEFAULT_MODE[provider],
modeId: this.voiceLlmModeId ?? "default",
...(model ? { model } : {}),
mcpServers: {
paseo: buildVoiceAgentMcpServerConfig({
@@ -4872,372 +4726,22 @@ export class Session {
* Process user message through LLM with streaming and tool execution
*/
private async processWithLLM(enableTTS: boolean, latestUserText?: string): Promise<void> {
if (enableTTS && this.voiceLlmProvider !== "openrouter") {
const text =
typeof latestUserText === "string" && latestUserText.trim().length > 0
? latestUserText
: (() => {
const lastUser = [...this.messages]
.reverse()
.find((message) => message.role === "user");
if (!lastUser) {
return "";
}
return typeof lastUser.content === "string"
? lastUser.content
: JSON.stringify(lastUser.content);
})();
const normalized = text.trim();
try {
if (!enableTTS) {
this.sessionLogger.warn("Ignoring non-voice processWithLLM call; voice is agent-only");
return;
}
const normalized = (latestUserText ?? "").trim();
if (!normalized) {
return;
}
await this.processWithVoiceAgent(normalized);
return;
}
let assistantResponse = "";
let pendingTTS: Promise<void> | null = null;
let textBuffer = "";
let sawTextDelta = false;
const flushTextBuffer = () => {
if (textBuffer.length > 0) {
// TTS handling (capture mode at generation time for drift protection)
if (enableTTS && !this.speechInProgress) {
const modeAtGeneration = this.isVoiceMode;
pendingTTS = this.ttsManager.generateAndWaitForPlayback(
textBuffer,
(msg) => this.emit(msg),
this.abortController.signal,
modeAtGeneration
);
} else if (enableTTS && this.speechInProgress) {
this.sessionLogger.debug("Skipping TTS chunk while speech in progress");
}
// Emit activity log
this.emit({
type: "activity_log",
payload: {
id: uuidv4(),
timestamp: new Date(),
type: "assistant",
content: textBuffer,
},
});
}
textBuffer = "";
};
try {
// Debug: dump conversation before LLM call
await this.dumpConversation();
const openrouterApiKey =
this.openrouterApiKey ?? process.env.OPENROUTER_API_KEY ?? null;
invariant(
openrouterApiKey,
"OpenRouter API key is required (set providers.openrouter.apiKey in config.json or OPENROUTER_API_KEY)"
);
const openrouter = createOpenRouter({
apiKey: openrouterApiKey,
});
// Wait for agent MCP to initialize if needed
if (!this.agentTools) {
this.sessionLogger.debug("Waiting for agent MCP initialization...");
const startTime = Date.now();
while (!this.agentTools && Date.now() - startTime < 5000) {
await new Promise((resolve) => setTimeout(resolve, 100));
}
if (!this.agentTools) {
this.sessionLogger.info("Agent MCP tools unavailable; continuing with default tool set");
}
}
const allTools = getAllTools(this.agentTools ?? undefined);
const result = await streamText({
model: openrouter(this.voiceLlmModel ?? "anthropic/claude-haiku-4.5"),
system: getSystemPrompt(),
providerOptions: {
openrouter: {
transforms: ["middle-out"], // Compress prompts that are > context size.
} as OpenRouterProviderOptions,
},
messages: this.messages,
tools: allTools,
abortSignal: this.abortController.signal,
onFinish: async (event) => {
const newMessages = event.response.messages;
if (newMessages.length > 0) {
this.messages.push(...newMessages);
this.sessionLogger.debug(
{ messageCount: newMessages.length },
`onFinish - saved message with ${newMessages.length} steps`
);
}
},
onChunk: async ({ chunk }) => {
if (chunk.type === "text-delta") {
sawTextDelta = true;
// Accumulate text in buffer
textBuffer += chunk.text;
assistantResponse += chunk.text;
// Emit chunk for UI streaming
this.emit({
type: "assistant_chunk",
payload: { chunk: chunk.text },
});
} else if (chunk.type === "tool-call") {
// Flush accumulated text as a segment before tool call
flushTextBuffer();
// Wait for pending TTS before executing tool
if (pendingTTS) {
this.sessionLogger.debug(
{ toolName: chunk.toolName },
`Waiting for TTS before executing ${chunk.toolName}`
);
await pendingTTS;
}
// Handle present_artifact tool specially
if (chunk.toolName === "present_artifact") {
await this.handlePresentArtifact(
chunk.toolCallId,
chunk.input as PresentArtifactArgs
);
}
// Emit tool call activity log
this.emit({
type: "activity_log",
payload: {
id: chunk.toolCallId,
timestamp: new Date(),
type: "tool_call",
content: `Calling ${chunk.toolName}`,
metadata: {
toolCallId: chunk.toolCallId,
toolName: chunk.toolName,
arguments: chunk.input,
},
},
});
} else if (chunk.type === "tool-result") {
// Check if this is a create_agent result
if (chunk.toolName === "create_agent" && chunk.output) {
const result = chunk.output as any;
if (result.structuredContent?.agentId) {
const agentId = result.structuredContent.agentId;
this.emit({
type: "status",
payload: {
status: "agent_created",
agentId,
},
});
}
}
// Emit tool result event
this.emit({
type: "activity_log",
payload: {
id: chunk.toolCallId,
timestamp: new Date(),
type: "tool_result",
content: `Tool ${chunk.toolName} completed`,
metadata: {
toolCallId: chunk.toolCallId,
toolName: chunk.toolName,
result: chunk.output,
},
},
});
}
},
onError: async (error) => {
this.sessionLogger.error({ err: error }, "Stream error");
this.emit({
type: "activity_log",
payload: {
id: uuidv4(),
timestamp: new Date(),
type: "error",
content: `Stream error: ${
error instanceof Error ? error.message : String(error)
}`,
},
});
},
stopWhen: stepCountIs(10),
});
// Consume the fullStream to handle tool-error chunks
for await (const part of result.fullStream) {
if (part.type === "tool-error") {
this.emit({
type: "activity_log",
payload: {
id: part.toolCallId,
timestamp: new Date(),
type: "error",
content: `Tool ${part.toolName} failed: ${
part.error instanceof Error
? part.error.message
: String(part.error)
}`,
metadata: {
toolCallId: part.toolCallId,
toolName: part.toolName,
error: part.error,
},
},
});
}
}
if (!sawTextDelta) {
let fallbackText = "";
try {
fallbackText = (await result.text).trim();
} catch {
fallbackText = "";
}
if (fallbackText.length > 0) {
textBuffer += fallbackText;
assistantResponse += fallbackText;
this.emit({
type: "assistant_chunk",
payload: { chunk: fallbackText },
});
}
}
// Flush any remaining text at the end
flushTextBuffer();
// Note: Message is saved by onFinish callback with proper tool calls
// Now wait for any pending TTS, but don't fail if it times out
if (pendingTTS) {
try {
await pendingTTS;
} catch (ttsError) {
this.sessionLogger.error(
{ err: ttsError },
"TTS playback failed (message already saved)"
);
}
}
} catch (error) {
// Note: Partial messages are saved by onAbort callback with proper tool calls
// Check if this is an abort error
const isAbortError =
error instanceof Error && error.name === "AbortError";
// Only emit error log for non-abort errors
if (!isAbortError) {
this.emit({
type: "activity_log",
payload: {
id: uuidv4(),
timestamp: new Date(),
type: "error",
content: `Error: ${
error instanceof Error ? error.message : String(error)
}`,
},
});
}
// Don't re-throw abort errors (they're expected during interruptions)
if (isAbortError) {
this.sessionLogger.debug("Stream aborted (partial response saved)");
return;
}
// Re-throw unexpected errors
throw error;
} finally {
// Increment turn index for next LLM call
this.turnIndex++;
// Clear the stream promise tracker
this.currentStreamPromise = null;
}
}
/**
* Handle present_artifact tool execution
*/
private async handlePresentArtifact(
toolCallId: string,
args: PresentArtifactArgs
): Promise<void> {
let content: string;
let isBase64 = false;
try {
if (args.source.type === "file") {
const fileBuffer = await readFile(args.source.path);
content = fileBuffer.toString("base64");
isBase64 = true;
} else if (args.source.type === "command_output") {
const { stdout } = await execAsync(args.source.command, {
encoding: "buffer",
});
content = stdout.toString("base64");
isBase64 = true;
} else if (args.source.type === "text") {
content = args.source.text;
isBase64 = false;
} else {
content = "[Unknown source type]";
isBase64 = false;
}
} catch (error) {
this.sessionLogger.error(
{ err: error },
"Failed to resolve artifact source"
);
content = `[Error: ${
error instanceof Error ? error.message : String(error)
}]`;
isBase64 = false;
}
// Emit artifact message
this.emit({
type: "artifact",
payload: {
type: args.type,
id: toolCallId,
title: args.title,
content,
isBase64,
},
});
// Emit activity log for artifact
this.emit({
type: "activity_log",
payload: {
id: toolCallId,
timestamp: new Date(),
type: "system",
content: `${args.type} artifact: ${args.title}`,
metadata: { artifactId: toolCallId, artifactType: args.type },
},
});
}
/**
* Handle abort request from client
*/
@@ -5445,38 +4949,6 @@ export class Session {
this.onMessage(msg);
}
/**
* Debug helper: dump conversation to disk
*/
private async dumpConversation(): Promise<void> {
try {
const dumpDir = join(process.cwd(), ".debug.conversations");
await mkdir(dumpDir, { recursive: true });
const filename = `${this.sessionId}-${this.turnIndex}.json`;
const filepath = join(dumpDir, filename);
const dump = {
voiceAssistantAgentId: this.voiceAssistantAgentId,
sessionId: this.sessionId,
turnIndex: this.turnIndex,
timestamp: new Date().toISOString(),
messages: this.messages,
};
await writeFile(filepath, inspect(dump, { depth: null }), "utf-8");
this.sessionLogger.debug(
{ filepath },
`Dumped conversation to ${filepath}`
);
} catch (error) {
this.sessionLogger.error(
{ err: error },
"Failed to dump conversation"
);
}
}
/**
* Clean up session resources
*/

2
packages/server/src/server/test-utils/paseo-daemon.ts
View File

@@ -21,7 +21,6 @@ type TestPaseoDaemonOptions = {
cleanup?: boolean;
openai?: PaseoOpenAIConfig;
speech?: PaseoSpeechConfig;
openrouterApiKey?: string | null;
voiceLlmProvider?: PaseoDaemonConfig["voiceLlmProvider"];
voiceLlmProviderExplicit?: boolean;
voiceLlmModel?: string | null;
@@ -82,7 +81,6 @@ export async function createTestPaseoDaemon(
appBaseUrl: "https://app.paseo.sh",
openai: options.openai,
speech: options.speech,
openrouterApiKey: options.openrouterApiKey ?? null,
voiceLlmProvider: options.voiceLlmProvider ?? null,
voiceLlmProviderExplicit: options.voiceLlmProviderExplicit ?? false,
voiceLlmModel: options.voiceLlmModel ?? null,

13
packages/server/src/server/websocket-server.ts
View File

@@ -22,7 +22,6 @@ import { PushService } from "./push/push-service.js";
import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
export type AgentMcpTransportFactory = () => Promise<Transport>;
type VoiceAgentProvider = "claude" | "codex" | "opencode";
type VoiceMcpStdioConfig = {
command: string;
baseArgs: string[];
@@ -79,12 +78,10 @@ export class VoiceAssistantWebSocketServer {
stt?: SpeechToTextProvider | null;
} | null;
private readonly voice: {
openrouterApiKey?: string | null;
voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null;
voiceLlmProvider?: AgentProvider | null;
voiceLlmModeId?: string | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmDefaultProvider?: VoiceAgentProvider | null;
voiceLlmModel?: string | null;
voiceLlmAvailability?: Record<VoiceAgentProvider, boolean> | null;
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
} | null;
private readonly voiceSpeakHandlers = new Map<
@@ -114,12 +111,10 @@ export class VoiceAssistantWebSocketServer {
speech?: { stt: SpeechToTextProvider | null; tts: TextToSpeechProvider | null },
terminalManager?: TerminalManager | null,
voice?: {
openrouterApiKey?: string | null;
voiceLlmProvider?: "openrouter" | "local-agent" | "claude" | "codex" | "opencode" | null;
voiceLlmProvider?: AgentProvider | null;
voiceLlmModeId?: string | null;
voiceLlmProviderExplicit?: boolean;
voiceLlmDefaultProvider?: VoiceAgentProvider | null;
voiceLlmModel?: string | null;
voiceLlmAvailability?: Record<VoiceAgentProvider, boolean> | null;
voiceAgentMcpStdio?: VoiceMcpStdioConfig | null;
},
dictation?: {

53
packages/website/public/schemas/paseo.config.v1.json
View File

@@ -86,8 +86,44 @@
},
"additionalProperties": false
},
"openrouter": {
"$ref": "#/definitions/PaseoConfigV1/properties/providers/properties/openai"
"sherpaOnnx": {
"type": "object",
"properties": {
"modelsDir": {
"type": "string",
"minLength": 1
},
"autoDownload": {
"type": "boolean"
},
"stt": {
"type": "object",
"properties": {
"preset": {
"type": "string",
"minLength": 1
}
},
"additionalProperties": false
},
"tts": {
"type": "object",
"properties": {
"preset": {
"type": "string",
"minLength": 1
},
"speakerId": {
"type": "number"
},
"speed": {
"type": "number"
}
},
"additionalProperties": false
}
},
"additionalProperties": false
}
},
"additionalProperties": false
@@ -104,7 +140,8 @@
"provider": {
"type": "string",
"enum": [
"openai"
"openai",
"local"
]
},
"model": {
@@ -129,7 +166,9 @@
"provider": {
"type": "string",
"enum": [
"openrouter"
"claude",
"codex",
"opencode"
]
},
"model": {
@@ -145,7 +184,8 @@
"provider": {
"type": "string",
"enum": [
"openai"
"openai",
"local"
]
},
"model": {
@@ -161,7 +201,8 @@
"provider": {
"type": "string",
"enum": [
"openai"
"openai",
"local"
]
},
"model": {

14
packages/website/src/routes/docs/configuration.tsx
View File

@@ -67,8 +67,7 @@ function Configuration() {
"$schema": "https://paseo.sh/schemas/paseo.config.v1.json",
"version": 1,
"providers": {
"openai": { "apiKey": "..." },
"openrouter": { "apiKey": "..." }
"openai": { "apiKey": "..." }
},
"daemon": {
"listen": "127.0.0.1:6767",
@@ -92,6 +91,7 @@ function Configuration() {
"features": {
"dictation": { "stt": { "provider": "local" } },
"voiceMode": {
"llm": { "provider": "claude", "model": "haiku" },
"stt": { "provider": "local" },
"tts": { "provider": "local" }
}
@@ -106,6 +106,13 @@ function Configuration() {
}
}`}
</pre>
<p className="text-white/60 leading-relaxed">
Voice LLM orchestration is agents-only. Set{' '}
<code className="font-mono">features.voiceMode.llm.provider</code> to one of{' '}
<code className="font-mono">claude</code>, <code className="font-mono">codex</code>, or{' '}
<code className="font-mono">opencode</code>. If set, startup is strict and fails if unavailable.
If omitted, Paseo picks the first available voice-enabled agent provider.
</p>
<p className="text-white/60 leading-relaxed">
Local voice uses ONNX models (sherpa-onnx + PocketTTS). Default presets are
<code className="font-mono">parakeet-tdt-0.6b-v3-int8</code> for STT and
@@ -152,7 +159,8 @@ function Configuration() {
<li><code className="font-mono">PASEO_HOME</code> — set Paseo home directory</li>
<li><code className="font-mono">PASEO_LISTEN</code> — override <code className="font-mono">daemon.listen</code></li>
<li><code className="font-mono">PASEO_ALLOWED_HOSTS</code> — override/extend <code className="font-mono">daemon.allowedHosts</code></li>
<li><code className="font-mono">OPENAI_API_KEY</code> and <code className="font-mono">OPENROUTER_API_KEY</code> — override provider keys</li>
<li><code className="font-mono">OPENAI_API_KEY</code> — override OpenAI provider key</li>
<li><code className="font-mono">PASEO_VOICE_LLM_PROVIDER</code> — override voice LLM provider (<code className="font-mono">claude</code>, <code className="font-mono">codex</code>, <code className="font-mono">opencode</code>)</li>
<li><code className="font-mono">PASEO_DICTATION_STT_PROVIDER</code>, <code className="font-mono">PASEO_VOICE_STT_PROVIDER</code>, <code className="font-mono">PASEO_VOICE_TTS_PROVIDER</code> — override voice provider selection (<code className="font-mono">local</code> or <code className="font-mono">openai</code>)</li>
<li><code className="font-mono">PASEO_SHERPA_ONNX_MODELS_DIR</code> and <code className="font-mono">PASEO_SHERPA_ONNX_AUTO_DOWNLOAD</code> — control local model directory and download behavior</li>
<li><code className="font-mono">PASEO_SHERPA_STT_PRESET</code> and <code className="font-mono">PASEO_SHERPA_TTS_PRESET</code> — override local STT/TTS model presets</li>