diff --git a/docs/architecture.md b/docs/architecture.md
index 0ed8c46ade..e0e32921cd 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -75,6 +75,8 @@ not retain non-Git directories.
| `server/agent/agent-storage.ts` | File-backed JSON persistence at `$PASEO_HOME/agents/` |
| `server/agent/tools/` | Transport-neutral catalog for workspaces, agents, permissions, and automation |
| `server/agent/mcp-server.ts` | Thin MCP adapter that registers the Paseo tool catalog with the MCP SDK |
+| `server/voice-chat/` | App-level voice calls, provider orchestration, and semantic call events |
+| `server/session/dictation/` | Composer dictation stream lifetime and speech-readiness gating |
| `server/agent/providers/` | Provider adapters (see "Agent providers" below) |
| `server/orchestration-skills/` | Bundled catalog, host selection, convergence, and skill-directory transactions |
| `server/relay-transport.ts` | Outbound relay connection with E2E encryption |
@@ -111,7 +113,25 @@ Cross-platform React Native app that connects to one or more daemons.
- Composer UI and submit/draft behavior live in `packages/app/src/composer/`; screens and panels should integrate it from there instead of dropping composer internals into `components/`, `hooks/`, or `screens/workspace/`
- Timeline reducers in `timeline/session-stream-reducers.ts` handle compaction, gap detection, sequence-based deduplication
- Timeline sync correctness is documented in [docs/timeline-sync.md](timeline-sync.md): live streams are for immediacy, `fetch_agent_timeline_request` is authoritative, and catch-up is paged but complete.
-- Voice features: dictation (STT) and voice agent (realtime)
+- App `VoiceRuntime` owns the app-level call UX, current route context, and semantic transcript. It
+ prepares installed client transports and accepts the server's selected answer without inspecting
+ transport kind. Each transport owns capture, playback, its data plane, mute, volume, and media
+ state; `transports/daemon-audio.ts` is the production transport in this slice. A WebRTC transport
+ adds another factory at app composition and does not change `VoiceRuntime`.
+- Server `VoiceChat` serializes start/stop, cancels startup, routes provider-neutral state/events,
+ forwards opaque transport data, and reacts to the provider's terminal signal. It knows only the
+ `VoiceCallProvider` contract; concrete data-plane validation and serialization stay in the
+ provider and matching client transport. Providers own prompts, credentials, hidden resources,
+ tools, media integration, and cleanup. The manual provider creates a hidden orchestrator through
+ the atomic required-Paseo-tools capability with an authoritative provider prompt, and owns a
+ separate speech runtime configured from the top-level `manualVoice` block. Dictation keeps its own
+ speech runtime. A complete manual orchestrator provider and mode selection is required; partial
+ configuration does not fall back to another provider or mode. Cleanup detaches work delegated by
+ the hidden orchestrator before archiving it.
+- Voice calls end as soon as the final client socket disconnects. The containing client session may
+ remain reconnectable, but calls are not replayed or rehydrated in this protocol slice. The sidebar
+ footer starts the call and hosts its controls; routes and visible agents do not own its lifetime.
+ Dictation remains a separate composer feature.
Consumers request directory or timeline data without choosing memory, cache, or network. The owner
publishes an accepted cache hit and then reconciles it over the existing network path. A miss or an
@@ -264,7 +284,8 @@ optionally carry the same `turnId`. New clients use these fields when present an
status once at the directory boundary rather than maintaining a second activity model.
- Terminal subscribe/input/capture commands
-- Voice/dictation streaming events (`dictation_stream_*`, `assistant_chunk`, `audio_output`, `transcription_result`)
+- App-level voice call commands and events (`voice.call.*`) and independent dictation streams
+ (`dictation_stream_*`)
- Request/response pairs for fetch, list, create, etc., correlated by `requestId`; failures use `rpc_error`
`directory_suggestions_request` is one daemon-owned filesystem search capability. The daemon
diff --git a/packages/app/src/app/_layout.tsx b/packages/app/src/app/_layout.tsx
index c57dba4429..3ea961b166 100644
--- a/packages/app/src/app/_layout.tsx
+++ b/packages/app/src/app/_layout.tsx
@@ -4,7 +4,13 @@ import { PortalProvider } from "@gorhom/portal";
import { QueryClientProvider } from "@tanstack/react-query";
import * as Linking from "expo-linking";
import * as Notifications from "expo-notifications";
-import { Stack, useNavigationContainerRef, usePathname, useRouter } from "expo-router";
+import {
+ Stack,
+ useGlobalSearchParams,
+ useNavigationContainerRef,
+ usePathname,
+ useRouter,
+} from "expo-router";
import {
createContext,
type ReactNode,
@@ -63,7 +69,7 @@ import { HorizontalScrollProvider } from "@/contexts/horizontal-scroll-context";
import { SessionProvider } from "@/contexts/session-context";
import { SidebarCalloutProvider } from "@/contexts/sidebar-callout-context";
import { ToastProvider } from "@/contexts/toast-context";
-import { VoiceProvider } from "@/contexts/voice-context";
+import { useVoiceOptional, VoiceProvider } from "@/contexts/voice-context";
import {
resolveStartupBlocker,
resolveStartupNavigationReady,
@@ -667,11 +673,35 @@ function ProvidersWrapper({ children }: { children: ReactNode }) {
{children}
+
);
}
+function VoiceCallContextSync() {
+ const voice = useVoiceOptional();
+ const params = useGlobalSearchParams<{
+ serverId?: string | string[];
+ workspaceId?: string | string[];
+ agentId?: string | string[];
+ }>();
+ const serverId = Array.isArray(params.serverId) ? params.serverId[0] : params.serverId;
+ const workspaceId = Array.isArray(params.workspaceId)
+ ? params.workspaceId[0]
+ : params.workspaceId;
+ const agentId = Array.isArray(params.agentId) ? params.agentId[0] : params.agentId;
+
+ useEffect(() => {
+ if (!voice?.isVoiceMode || voice.activeServerId !== serverId) return;
+ void voice
+ .updateCallContext({ workspaceId: workspaceId ?? null, agentId: agentId ?? null })
+ .catch((error) => console.warn("[VoiceCall] Failed to update app context", error));
+ }, [agentId, serverId, voice, workspaceId]);
+
+ return null;
+}
+
function DesktopWindowControlsSync() {
const { isLoading } = useAppSettings();
const { theme } = useUnistyles();
diff --git a/packages/app/src/components/left-sidebar.tsx b/packages/app/src/components/left-sidebar.tsx
index 1cdfe83e6d..b664148ff3 100644
--- a/packages/app/src/components/left-sidebar.tsx
+++ b/packages/app/src/components/left-sidebar.tsx
@@ -1,5 +1,13 @@
-import { router } from "expo-router";
-import { FolderPlus, GitBranch, Import, Server, Settings, X } from "lucide-react-native";
+import { router, useGlobalSearchParams } from "expo-router";
+import {
+ AudioLines,
+ FolderPlus,
+ GitBranch,
+ Import,
+ Server,
+ Settings,
+ X,
+} from "lucide-react-native";
import { useTranslation } from "react-i18next";
import { memo, useCallback, useEffect, useMemo, useRef, useState, type RefObject } from "react";
import {
@@ -42,7 +50,9 @@ import { RetainedPanelActivity } from "@/components/retained-panel";
import type { SidebarWorkspaceGroup } from "@/components/sidebar/sidebar-labels";
import type { SidebarProjectIconTarget } from "@/utils/sidebar-project-row-model";
import { type SidebarGroupMode, useSidebarViewStore } from "@/stores/sidebar-view-store";
-import { useHosts } from "@/runtime/host-runtime";
+import { useHostRuntimeIsConnected, useHosts } from "@/runtime/host-runtime";
+import { useVoiceOptional } from "@/contexts/voice-context";
+import { useToast } from "@/contexts/toast-context";
import { usePanelStore } from "@/stores/panel-store";
import { useOwnsWindowChromeCorner, WindowChromeSafeArea } from "@/utils/desktop-window";
import { useCloseAgentListGesture } from "@/mobile-panels/gestures";
@@ -52,6 +62,11 @@ import { openHostOverview } from "@/navigation/settings-navigation";
import { SidebarAgentListSkeleton } from "./sidebar-agent-list-skeleton";
import { SidebarCalloutSlot } from "./sidebar-callout-slot";
import { SidebarWorkspaceList } from "./sidebar-workspace-list";
+import { VoiceCallSurface } from "./voice-call-surface";
+import {
+ useActiveWorkspaceSelection,
+ useLastWorkspaceSelection,
+} from "@/stores/navigation-active-workspace-store";
type SidebarTheme = ReturnType["theme"];
@@ -268,6 +283,7 @@ function sidebarHostOptionTestID(serverId: string): string {
function FooterIconButton({
buttonRef,
onPress,
+ disabled = false,
testID,
label,
icon: Icon,
@@ -276,6 +292,7 @@ function FooterIconButton({
theme,
}: {
onPress: () => void;
+ disabled?: boolean;
testID: string;
label: string;
icon: typeof FolderPlus;
@@ -289,19 +306,20 @@ function FooterIconButton({
{({ hovered }) => (
)}
@@ -313,6 +331,68 @@ function FooterIconButton({
);
}
+function firstSearchParam(value: string | string[] | undefined): string | null {
+ return Array.isArray(value) ? (value[0] ?? null) : (value ?? null);
+}
+
+function SidebarVoiceChatButton({ theme }: { theme: SidebarTheme }) {
+ const { t } = useTranslation();
+ const toast = useToast();
+ const voice = useVoiceOptional();
+ const activeWorkspace = useActiveWorkspaceSelection();
+ const lastWorkspace = useLastWorkspaceSelection();
+ const workspace = activeWorkspace ?? lastWorkspace;
+ const params = useGlobalSearchParams<{
+ serverId?: string | string[];
+ workspaceId?: string | string[];
+ agentId?: string | string[];
+ }>();
+ const isConnected = useHostRuntimeIsConnected(workspace?.serverId ?? "");
+ const isActiveRoute =
+ workspace?.serverId === firstSearchParam(params.serverId) &&
+ workspace?.workspaceId === firstSearchParam(params.workspaceId);
+ const agentId = isActiveRoute ? firstSearchParam(params.agentId) : null;
+ const disabled = !voice || voice.isVoiceSwitching;
+
+ const handlePress = useCallback(() => {
+ if (!voice || disabled) return;
+ if (!workspace) {
+ toast.error("Open a workspace before starting voice chat.");
+ return;
+ }
+ if (!isConnected) {
+ toast.error("Connect to the host before starting voice chat.");
+ return;
+ }
+ void voice
+ .startCall(workspace.serverId, { workspaceId: workspace.workspaceId, agentId })
+ .catch((error) => {
+ console.error("[Sidebar] Failed to start voice chat", error);
+ toast.error(error instanceof Error ? error.message : "Unable to start voice chat");
+ });
+ }, [agentId, disabled, isConnected, toast, voice, workspace]);
+
+ if (voice?.isVoiceMode) return null;
+
+ return (
+
+ );
+}
+
+function VoiceWorkspaceListClearance() {
+ const voice = useVoiceOptional();
+ return voice?.isVoiceMode ? : null;
+}
+
+const voiceWorkspaceListClearance = ;
+
function footerAddProjectButtonStyle({
hovered,
}: PressableStateCallbackType & { hovered?: boolean }) {
@@ -489,6 +569,7 @@ function SidebarFooter({
icon={Import}
theme={theme}
/>
+
)}
+
)}
+
({
paddingVertical: theme.spacing[1],
paddingHorizontal: theme.spacing[1],
},
+ footerIconButtonDisabled: {
+ opacity: theme.opacity[50],
+ },
+ voiceWorkspaceListClearance: {
+ height: theme.spacing[12],
+ },
tooltipRow: {
flexDirection: "row",
alignItems: "center",
diff --git a/packages/app/src/components/realtime-voice-overlay.tsx b/packages/app/src/components/realtime-voice-overlay.tsx
index 88b2e378b4..f4930c1481 100644
--- a/packages/app/src/components/realtime-voice-overlay.tsx
+++ b/packages/app/src/components/realtime-voice-overlay.tsx
@@ -3,24 +3,26 @@ import { useMemo } from "react";
import { useTranslation } from "react-i18next";
import { Pressable, View } from "react-native";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
-import { Mic, MicOff, Square } from "lucide-react-native";
-import { FOOTER_HEIGHT } from "@/constants/layout";
+import { MessageCircle, Mic, MicOff, Square } from "lucide-react-native";
import { useVoiceTelemetry } from "@/contexts/voice-context";
import { VolumeMeter } from "./volume-meter";
interface RealtimeVoiceOverlayProps {
isMuted: boolean;
isSwitching: boolean;
+ isTranscriptOpen: boolean;
+ onToggleTranscript: () => void;
onToggleMute: () => void;
onStop: () => void;
}
-const OVERLAY_BUTTON_SIZE = 44;
-const OVERLAY_VERTICAL_PADDING = (FOOTER_HEIGHT - OVERLAY_BUTTON_SIZE) / 2;
+const OVERLAY_BUTTON_SIZE = 28;
export function RealtimeVoiceOverlay({
isMuted,
isSwitching,
+ isTranscriptOpen,
+ onToggleTranscript,
onToggleMute,
onStop,
}: RealtimeVoiceOverlayProps) {
@@ -42,12 +44,26 @@ export function RealtimeVoiceOverlay({
);
return (
+
+
+
+
@@ -62,9 +78,9 @@ export function RealtimeVoiceOverlay({
style={muteButtonStyle}
>
{isMuted ? (
-
+
) : (
-
+
)}
@@ -79,7 +95,7 @@ export function RealtimeVoiceOverlay({
) : (
({
flexDirection: "row",
alignItems: "center",
width: "100%",
- height: FOOTER_HEIGHT,
- borderRadius: theme.borderRadius["2xl"],
+ minHeight: 40,
+ borderRadius: theme.borderRadius.xl,
justifyContent: "space-between",
- paddingHorizontal: theme.spacing[4],
- paddingVertical: OVERLAY_VERTICAL_PADDING,
+ paddingHorizontal: theme.spacing[2],
+ paddingVertical: theme.spacing[1.5],
backgroundColor: theme.colors.surface1,
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
},
meterContainer: {
flex: 1,
+ minWidth: 0,
alignItems: "center",
justifyContent: "center",
},
actionsContainer: {
flexDirection: "row",
alignItems: "center",
- gap: theme.spacing[2],
+ gap: theme.spacing[1],
},
actionButton: {
width: OVERLAY_BUTTON_SIZE,
@@ -127,6 +144,14 @@ const styles = StyleSheet.create((theme) => ({
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
},
+ transcriptButton: {
+ backgroundColor: theme.colors.surface0,
+ borderWidth: theme.borderWidth[1],
+ borderColor: theme.colors.border,
+ },
+ transcriptButtonOpen: {
+ backgroundColor: theme.colors.interactionHighlight,
+ },
muteButtonMuted: {
backgroundColor: theme.colors.palette.red[600],
borderColor: theme.colors.palette.red[800],
diff --git a/packages/app/src/components/voice-call-surface.tsx b/packages/app/src/components/voice-call-surface.tsx
new file mode 100644
index 0000000000..7167685ca2
--- /dev/null
+++ b/packages/app/src/components/voice-call-surface.tsx
@@ -0,0 +1,149 @@
+import { useCallback, useState } from "react";
+import { Pressable, ScrollView, Text, View } from "react-native";
+import { StyleSheet } from "react-native-unistyles";
+import { RealtimeVoiceOverlay } from "@/components/realtime-voice-overlay";
+import { useVoiceOptional } from "@/contexts/voice-context";
+import type { VoiceCallEvent } from "@getpaseo/protocol/messages";
+
+function VoiceCallEventRow({ event }: { event: VoiceCallEvent }) {
+ switch (event.type) {
+ case "transcript":
+ return (
+
+ {event.speaker === "user" ? "You:" : "Paseo:"}
+ {event.text}
+
+ );
+ case "activity":
+ return (
+
+ {event.label}
+ {event.state}
+
+ );
+ case "notice":
+ return {event.text};
+ case "error":
+ return {event.message};
+ }
+}
+
+export function VoiceCallSurface() {
+ const voice = useVoiceOptional();
+ const [transcriptOpen, setTranscriptOpen] = useState(false);
+ const toggleTranscript = useCallback(() => setTranscriptOpen((open) => !open), []);
+
+ if (!voice || (!voice.isVoiceMode && voice.events.length === 0)) return null;
+
+ const showTranscript = transcriptOpen || !voice.isVoiceMode;
+
+ return (
+
+
+ {showTranscript ? (
+
+ {voice.events.length === 0 ? (
+ Conversation will appear here.
+ ) : (
+ voice.events.map((event) => )
+ )}
+
+ ) : null}
+ {voice.isVoiceMode ? (
+
+ ) : (
+
+ Dismiss
+
+ )}
+
+
+ );
+}
+
+const styles = StyleSheet.create((theme) => ({
+ host: {
+ width: "100%",
+ paddingHorizontal: theme.spacing[2],
+ paddingBottom: theme.spacing[1.5],
+ },
+ surface: {
+ width: "100%",
+ gap: theme.spacing[2],
+ },
+ transcript: {
+ maxHeight: 200,
+ paddingHorizontal: theme.spacing[2],
+ paddingVertical: theme.spacing[1.5],
+ borderRadius: theme.borderRadius.lg,
+ borderWidth: theme.borderWidth[1],
+ borderColor: theme.colors.border,
+ backgroundColor: theme.colors.surface0,
+ },
+ transcriptContent: {
+ gap: theme.spacing[1.5],
+ },
+ empty: {
+ color: theme.colors.foregroundMuted,
+ fontSize: theme.fontSize.sm,
+ },
+ eventText: {
+ color: theme.colors.foreground,
+ fontSize: theme.fontSize.sm,
+ },
+ eventLabel: {
+ color: theme.colors.foregroundMuted,
+ fontWeight: theme.fontWeight.medium,
+ },
+ activityRow: {
+ minWidth: 0,
+ flexDirection: "row",
+ alignItems: "center",
+ justifyContent: "space-between",
+ gap: theme.spacing[2],
+ },
+ activityLabel: {
+ minWidth: 0,
+ flexShrink: 1,
+ color: theme.colors.foregroundMuted,
+ fontSize: theme.fontSize.sm,
+ },
+ activityState: {
+ color: theme.colors.foregroundExtraMuted,
+ fontSize: theme.fontSize.sm,
+ },
+ notice: {
+ color: theme.colors.foregroundMuted,
+ fontSize: theme.fontSize.sm,
+ },
+ error: {
+ color: theme.colors.palette.red[500],
+ fontSize: theme.fontSize.sm,
+ },
+ transcriptButton: {
+ alignSelf: "flex-end",
+ paddingHorizontal: theme.spacing[3],
+ paddingVertical: theme.spacing[2],
+ borderRadius: theme.borderRadius.full,
+ backgroundColor: theme.colors.surface1,
+ borderWidth: theme.borderWidth[1],
+ borderColor: theme.colors.border,
+ },
+ transcriptButtonText: {
+ color: theme.colors.foreground,
+ fontSize: theme.fontSize.sm,
+ fontWeight: "600",
+ },
+}));
diff --git a/packages/app/src/composer/index.tsx b/packages/app/src/composer/index.tsx
index 5a1d51f1fc..7575b03a37 100644
--- a/packages/app/src/composer/index.tsx
+++ b/packages/app/src/composer/index.tsx
@@ -1,11 +1,5 @@
import { LoadingSpinner } from "@/components/ui/loading-spinner";
-import {
- View,
- Pressable,
- Text,
- StyleSheet as RNStyleSheet,
- type PressableStateCallbackType,
-} from "react-native";
+import { View, Pressable, Text, StyleSheet as RNStyleSheet } from "react-native";
import type { TFunction } from "i18next";
import {
useState,
@@ -25,7 +19,6 @@ import {
ArrowUp,
Square,
Pencil,
- AudioLines,
CircleDot,
FileText,
GitPullRequest,
@@ -75,7 +68,6 @@ import {
type QueueWriter,
type QueuedComposerMessage,
} from "@/composer/actions";
-import { useVoiceOptional } from "@/contexts/voice-context";
import { useToast } from "@/contexts/toast-context";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { Shortcut } from "@/components/ui/shortcut";
@@ -182,14 +174,6 @@ function resolveIsComposerLocked(
return submitBehavior === "preserve-and-lock" && isSubmitLoading;
}
-function resolveIsVoiceModeForAgent(
- voice: ReturnType,
- serverId: string,
- agentId: string,
-): boolean {
- return voice?.isVoiceModeForAgent(serverId, agentId) ?? false;
-}
-
function resolveKeyboardPriority(isMessageInputFocused: boolean): number {
return isMessageInputFocused ? 200 : 100;
}
@@ -241,19 +225,6 @@ function buildCancelButtonStyle(isConnected: boolean, isCancellingAgent: boolean
return [styles.cancelButton, disabled].filter((value): value is object => Boolean(value));
}
-function buildRealtimeVoiceButtonStyle(
- hovered: boolean | undefined,
- voiceButtonDisabled: boolean,
- reserveLeadingSpace: boolean,
-): object[] {
- const hoveredStyle = hovered ? styles.iconButtonHovered : undefined;
- const disabledStyle = voiceButtonDisabled ? styles.buttonDisabled : undefined;
- const reserveStyle = reserveLeadingSpace ? styles.realtimeVoiceButtonCompactReserve : undefined;
- return [styles.realtimeVoiceButton, reserveStyle, hoveredStyle, disabledStyle].filter(
- (value): value is object => Boolean(value),
- );
-}
-
function buildAgentStateSelector(serverId: string, agentId: string) {
return (state: ReturnType) => {
const agent = state.sessions[serverId]?.agents?.get(agentId) ?? null;
@@ -487,29 +458,6 @@ function resolveErrorMessage(error: unknown): string | null {
return null;
}
-interface AttemptStartRealtimeVoiceArgs {
- voice: ReturnType;
- isConnected: boolean;
- hasAgent: boolean;
- serverId: string;
- agentId: string;
- toastErrorRef: { current: (message: string) => void };
-}
-
-function attemptStartRealtimeVoice(args: AttemptStartRealtimeVoiceArgs): void {
- const { voice, isConnected, hasAgent, serverId, agentId, toastErrorRef } = args;
- if (!voice || !isConnected || !hasAgent) return;
- if (voice.isVoiceSwitching) return;
- if (voice.isVoiceModeForAgent(serverId, agentId)) return;
- void voice.startVoice(serverId, agentId).catch((error) => {
- console.error("[Composer] Failed to start voice mode", error);
- const message = resolveErrorMessage(error);
- if (message && message.trim().length > 0) {
- toastErrorRef.current(message);
- }
- });
-}
-
function focusMessageInputWithPlatformStrategy(messageInputRef: {
current: MessageInputRef | null;
}): void {
@@ -1043,88 +991,6 @@ function ComposerCancelButton({
);
}
-interface ComposerVoiceModeButtonProps {
- buttonIconSize: number;
- handleToggleRealtimeVoice: () => void;
- isConnected: boolean;
- isVoiceSwitching: boolean;
- realtimeVoiceButtonStyle: (
- state: PressableStateCallbackType & { hovered?: boolean },
- ) => (object | undefined)[];
- voiceToggleKeys: ReturnType;
- t: TFunction;
-}
-
-interface ComposerRightControlsSlotProps extends ComposerVoiceModeButtonProps {
- isVoiceModeForAgent: boolean;
- hasAgent: boolean;
- isAgentRunning: boolean;
- hasSendableContent: boolean;
- isCompact: boolean;
- showVoice: boolean;
-}
-
-function ComposerRightControlsSlot({
- isVoiceModeForAgent,
- hasAgent,
- isAgentRunning,
- hasSendableContent,
- isCompact,
- showVoice,
- ...voiceProps
-}: ComposerRightControlsSlotProps) {
- const hideVoiceForCompactInput = isCompact && hasSendableContent;
- const showVoiceModeButton =
- showVoice && !isVoiceModeForAgent && hasAgent && !isAgentRunning && !hideVoiceForCompactInput;
- if (!showVoiceModeButton) return null;
- return (
-
-
-
- );
-}
-
-function ComposerVoiceModeButton({
- buttonIconSize,
- handleToggleRealtimeVoice,
- isConnected,
- isVoiceSwitching,
- realtimeVoiceButtonStyle,
- voiceToggleKeys,
- t,
-}: ComposerVoiceModeButtonProps) {
- const shortcutNode = voiceToggleKeys ? : null;
- const renderTriggerContent = useCallback(
- ({ hovered }: PressableStateCallbackType & { hovered?: boolean }) => {
- if (isVoiceSwitching) {
- return ;
- }
- const colorMapping = hovered ? iconForegroundMapping : iconForegroundMutedMapping;
- return ;
- },
- [buttonIconSize, isVoiceSwitching],
- );
- return (
-
-
- {renderTriggerContent}
-
-
-
- {t("composer.voice.voiceMode")}
- {shortcutNode}
-
-
-
- );
-}
-
export function Composer({ isPaneFocused, ...props }: ComposerProps) {
return (
@@ -1192,8 +1058,6 @@ function ComposerContentImpl({
const toast = useToast();
const toastErrorRef = useRef(toast.error);
toastErrorRef.current = toast.error;
- const voice = useVoiceOptional();
- const voiceToggleKeys = useShortcutKeys("voice-toggle");
const agentInterruptKeys = useShortcutKeys("agent-interrupt");
const isDictationReady = useIsDictationReady({
serverId,
@@ -1838,19 +1702,6 @@ function ComposerContentImpl({
focusMessageInputWithPlatformStrategy(messageInputRef);
}, []);
- const isVoiceModeForAgent = resolveIsVoiceModeForAgent(voice, serverId, agentId);
-
- const handleToggleRealtimeVoice = useCallback(() => {
- attemptStartRealtimeVoice({
- voice,
- isConnected,
- hasAgent,
- serverId,
- agentId,
- toastErrorRef,
- });
- }, [agentId, hasAgent, isConnected, serverId, voice]);
-
const handleEditQueuedMessage = useCallback(
(id: string) => {
const result = editQueuedComposerMessage({
@@ -1912,8 +1763,6 @@ function ComposerContentImpl({
],
);
- const hasSendableContent = userInput.trim().length > 0 || selectedAttachments.length > 0;
-
// Handle keyboard navigation for command autocomplete.
const handleCommandKeyPress = useCallback(
(event: ComposerKeyPressEvent) => autocompleteOnKeyPressRef.current(event),
@@ -1925,14 +1774,6 @@ function ComposerContentImpl({
[isConnected, isCancellingAgent],
);
- const isVoiceSwitching = voice?.isVoiceSwitching ?? false;
- const voiceButtonDisabled = !isConnected || isVoiceSwitching;
- const realtimeVoiceButtonStyle = useCallback(
- (state: PressableStateCallbackType & { hovered?: boolean }) =>
- buildRealtimeVoiceButtonStyle(state.hovered, voiceButtonDisabled, isCompactLayout),
- [isCompactLayout, voiceButtonDisabled],
- );
-
const activeActionContent = useMemo(
() => (
(
-
- ),
- [
- buttonIconSize,
- handleToggleRealtimeVoice,
- hasAgent,
- hasSendableContent,
- isAgentRunning,
- isConnected,
- isCompactLayout,
- isVoiceModeForAgent,
- isVoiceSwitching,
- mode.showVoice,
- realtimeVoiceButtonStyle,
- t,
- voiceToggleKeys,
- ],
- );
-
const { contextWindowMaxTokens, contextWindowUsedTokens } = resolveContextWindowValues(
agentState.contextWindowMaxTokens,
agentState.contextWindowUsedTokens,
@@ -2362,9 +2168,9 @@ function ComposerContentImpl({
disabled={isSubmitLoading}
leftContent={leftContent}
beforeVoiceContent={beforeVoiceContent}
- rightContent={rightContent}
activeActionContent={activeActionContent}
voiceServerId={serverId}
+ voiceWorkspaceId={workspaceId ?? undefined}
voiceAgentId={agentId}
isAgentRunning={isAgentRunning}
defaultSendBehavior={activeSendBehavior}
@@ -2461,11 +2267,6 @@ const styles = StyleSheet.create((theme: Theme) => ({
justifyContent: "center",
marginLeft: theme.spacing[1],
},
- rightControls: {
- flexDirection: "row",
- alignItems: "center",
- gap: theme.spacing[1],
- },
contextWindowMeterSlot: {
width: 28,
height: 28,
@@ -2473,20 +2274,6 @@ const styles = StyleSheet.create((theme: Theme) => ({
alignItems: "center",
justifyContent: "center",
},
- realtimeVoiceButton: {
- width: 28,
- height: 28,
- borderRadius: theme.borderRadius.full,
- alignItems: "center",
- justifyContent: "center",
- },
- realtimeVoiceButtonCompactReserve: {
- marginLeft: theme.spacing[1],
- },
- realtimeVoiceButtonActive: {
- backgroundColor: theme.colors.palette.green[600],
- borderColor: theme.colors.palette.green[800],
- },
iconButtonHovered: {
backgroundColor: theme.colors.surface2,
},
@@ -2554,7 +2341,6 @@ const ThemedPencil = withUnistyles(Pencil);
const ThemedArrowUp = withUnistyles(ArrowUp);
const ThemedGitPullRequest = withUnistyles(GitPullRequest);
const ThemedCircleDot = withUnistyles(CircleDot);
-const ThemedAudioLines = withUnistyles(AudioLines);
const ThemedPaperclip = withUnistyles(Paperclip);
const ThemedImageIcon = withUnistyles(ImageIcon);
const ThemedClipboardPaste = withUnistyles(ClipboardPaste);
diff --git a/packages/app/src/composer/input/input.tsx b/packages/app/src/composer/input/input.tsx
index fc8a260cfb..411fd2b366 100644
--- a/packages/app/src/composer/input/input.tsx
+++ b/packages/app/src/composer/input/input.tsx
@@ -24,7 +24,6 @@ import { ICON_SIZE, type Theme } from "@/styles/theme";
import { ArrowUp, Mic, MicOff, CornerDownLeft, Plus, Square } from "lucide-react-native";
import { useDictation } from "@/hooks/use-dictation";
import { DictationOverlay } from "@/components/dictation-controls";
-import { RealtimeVoiceOverlay } from "@/components/realtime-voice-overlay";
import type { DaemonClient } from "@getpaseo/client/internal/daemon-client";
import { useSessionStore } from "@/stores/session-store";
import { useVoiceOptional } from "@/contexts/voice-context";
@@ -144,6 +143,7 @@ export interface MessageInputProps {
/** Primary action to render when the agent is active and the composer has no sendable content. */
activeActionContent?: React.ReactNode;
voiceServerId?: string;
+ voiceWorkspaceId?: string;
voiceAgentId?: string;
/** When true and there's sendable content, calls onQueue instead of onSubmit */
isAgentRunning?: boolean;
@@ -594,16 +594,9 @@ function MessageInputOverlay({
/>
);
}
- if (showRealtimeOverlay && voice) {
- return (
-
- );
- }
+ void showRealtimeOverlay;
+ void voice;
+ void onRealtimeVoiceStop;
return null;
}
@@ -830,41 +823,50 @@ interface ToggleRealtimeVoiceContext {
voice:
| {
isVoiceSwitching: boolean;
- isVoiceModeForAgent: (serverId: string, agentId: string) => boolean;
- startVoice: (serverId: string, agentId: string) => Promise;
+ isCallActiveOnServer: (serverId: string) => boolean;
+ startCall: (
+ serverId: string,
+ context: { workspaceId: string | null; agentId: string | null },
+ ) => Promise;
}
| null
| undefined;
voiceServerId: string | undefined;
+ voiceWorkspaceId: string | undefined;
voiceAgentId: string | undefined;
isConnected: boolean;
disabled: boolean;
- isAgentRunning: boolean;
handleStopRealtimeVoice: () => Promise | void;
toast: { error: (msg: string) => void };
- interruptBeforeVoiceMessage: string;
}
function toggleRealtimeVoiceImpl(ctx: ToggleRealtimeVoiceContext): void {
- if (!ctx.voice || !ctx.voiceServerId || !ctx.voiceAgentId || !ctx.isConnected || ctx.disabled) {
+ if (
+ !ctx.voice ||
+ !ctx.voiceServerId ||
+ !ctx.voiceWorkspaceId ||
+ !ctx.isConnected ||
+ ctx.disabled
+ ) {
return;
}
if (ctx.voice.isVoiceSwitching) return;
- if (ctx.voice.isVoiceModeForAgent(ctx.voiceServerId, ctx.voiceAgentId)) {
+ if (ctx.voice.isCallActiveOnServer(ctx.voiceServerId)) {
void ctx.handleStopRealtimeVoice();
return;
}
- if (ctx.isAgentRunning) {
- ctx.toast.error(ctx.interruptBeforeVoiceMessage);
- return;
- }
- void ctx.voice.startVoice(ctx.voiceServerId, ctx.voiceAgentId).catch((error) => {
- console.error("[MessageInput] Failed to start realtime voice", error);
- const message = extractErrorMessage(error);
- if (message && message.trim().length > 0) {
- ctx.toast.error(message);
- }
- });
+ void ctx.voice
+ .startCall(ctx.voiceServerId, {
+ workspaceId: ctx.voiceWorkspaceId,
+ agentId: ctx.voiceAgentId ?? null,
+ })
+ .catch((error) => {
+ console.error("[MessageInput] Failed to start realtime voice", error);
+ const message = extractErrorMessage(error);
+ if (message && message.trim().length > 0) {
+ ctx.toast.error(message);
+ }
+ });
}
interface StartDictationContext {
@@ -959,12 +961,11 @@ function queueMessageImpl(ctx: QueueMessageContext): void {
}
function computeIsRealtimeVoiceForAgent(
- voice: { isVoiceModeForAgent: (serverId: string, agentId: string) => boolean } | null | undefined,
+ voice: { isCallActiveOnServer: (serverId: string) => boolean } | null | undefined,
voiceServerId: string | undefined,
- voiceAgentId: string | undefined,
): boolean {
- if (!voice || !voiceServerId || !voiceAgentId) return false;
- return voice.isVoiceModeForAgent(voiceServerId, voiceAgentId);
+ if (!voice || !voiceServerId) return false;
+ return voice.isCallActiveOnServer(voiceServerId);
}
function computeShouldShowDictationOverlay(
@@ -1068,6 +1069,7 @@ interface ResolvedMessageInputProps {
rightContent: React.ReactNode;
activeActionContent: React.ReactNode;
voiceServerId: string | undefined;
+ voiceWorkspaceId: string | undefined;
voiceAgentId: string | undefined;
isAgentRunning: boolean;
defaultSendBehavior: "interrupt" | "steer" | "queue";
@@ -1115,6 +1117,7 @@ function resolveMessageInputProps(props: MessageInputProps): ResolvedMessageInpu
rightContent: props.rightContent,
activeActionContent: props.activeActionContent,
voiceServerId: props.voiceServerId,
+ voiceWorkspaceId: props.voiceWorkspaceId,
voiceAgentId: props.voiceAgentId,
isAgentRunning: props.isAgentRunning ?? false,
defaultSendBehavior: props.defaultSendBehavior,
@@ -1170,6 +1173,7 @@ export const MessageInput = forwardRef(
rightContent,
activeActionContent,
voiceServerId,
+ voiceWorkspaceId,
voiceAgentId,
isAgentRunning,
defaultSendBehavior,
@@ -1375,11 +1379,7 @@ export const MessageInput = forwardRef(
enableDuration: true,
});
- const isRealtimeVoiceForCurrentAgent = computeIsRealtimeVoiceForAgent(
- voice,
- voiceServerId,
- voiceAgentId,
- );
+ const isRealtimeVoiceForCurrentAgent = computeIsRealtimeVoiceForAgent(voice, voiceServerId);
const showDictationOverlay = computeShouldShowDictationOverlay(
isDictating,
isDictationProcessing,
@@ -1452,9 +1452,6 @@ export const MessageInput = forwardRef(
await stopRealtimeVoice({
voice,
isRealtimeVoiceForCurrentAgent,
- isAgentRunning,
- client,
- voiceAgentId,
});
} catch (error) {
console.error("[MessageInput] Failed to stop realtime voice", error);
@@ -1463,30 +1460,28 @@ export const MessageInput = forwardRef(
toast.error(message);
}
}
- }, [client, isAgentRunning, isRealtimeVoiceForCurrentAgent, toast, voice, voiceAgentId]);
+ }, [isRealtimeVoiceForCurrentAgent, toast, voice]);
const handleToggleRealtimeVoiceShortcut = useCallback(() => {
toggleRealtimeVoiceImpl({
voice,
voiceServerId,
+ voiceWorkspaceId,
voiceAgentId,
isConnected,
disabled,
- isAgentRunning,
handleStopRealtimeVoice,
toast,
- interruptBeforeVoiceMessage: t("composer.voice.interruptBeforeVoice"),
});
}, [
disabled,
handleStopRealtimeVoice,
- isAgentRunning,
isConnected,
- t,
toast,
voice,
voiceAgentId,
voiceServerId,
+ voiceWorkspaceId,
]);
const minimizeInputHeight = useCallback(() => {
diff --git a/packages/app/src/composer/input/state.test.ts b/packages/app/src/composer/input/state.test.ts
index 5503ea1512..5a8a000db4 100644
--- a/packages/app/src/composer/input/state.test.ts
+++ b/packages/app/src/composer/input/state.test.ts
@@ -276,43 +276,14 @@ describe("composer send behavior", () => {
});
describe("stopRealtimeVoice", () => {
- it("keeps voice mode active when the running agent refuses cancellation", async () => {
- const cancellationError = new Error("active run cancellation was not acknowledged");
- const cancelAgent = vi.fn().mockRejectedValue(cancellationError);
+ it("stops the app-level call without touching the visible agent", async () => {
const stopVoice = vi.fn().mockResolvedValue(undefined);
- await expect(
- stopRealtimeVoice({
- voice: { stopVoice },
- isRealtimeVoiceForCurrentAgent: true,
- isAgentRunning: true,
- client: { cancelAgent },
- voiceAgentId: "agent-1",
- }),
- ).rejects.toBe(cancellationError);
-
- expect(stopVoice).not.toHaveBeenCalled();
- });
-
- it("stops voice mode after the running agent acknowledges cancellation", async () => {
- const calls: string[] = [];
-
await stopRealtimeVoice({
- voice: {
- stopVoice: async () => {
- calls.push("stop voice");
- },
- },
+ voice: { stopVoice },
isRealtimeVoiceForCurrentAgent: true,
- isAgentRunning: true,
- client: {
- cancelAgent: async () => {
- calls.push("cancel agent");
- },
- },
- voiceAgentId: "agent-1",
});
- expect(calls).toEqual(["cancel agent", "stop voice"]);
+ expect(stopVoice).toHaveBeenCalledOnce();
});
});
diff --git a/packages/app/src/composer/input/state.ts b/packages/app/src/composer/input/state.ts
index a63aeb02dc..d5cb18d1ba 100644
--- a/packages/app/src/composer/input/state.ts
+++ b/packages/app/src/composer/input/state.ts
@@ -41,9 +41,6 @@ export function resolveComposerSurfacePresentation(
interface StopRealtimeVoiceContext {
voice: { stopVoice: () => Promise } | null | undefined;
isRealtimeVoiceForCurrentAgent: boolean;
- isAgentRunning: boolean;
- client: { cancelAgent: (agentId: string) => Promise } | null;
- voiceAgentId: string | undefined;
}
interface SendActionContext {
@@ -183,12 +180,5 @@ export function runMessageInputKeyboardAction(
export async function stopRealtimeVoice(ctx: StopRealtimeVoiceContext): Promise {
if (!ctx.voice || !ctx.isRealtimeVoiceForCurrentAgent) return;
- if (ctx.isAgentRunning) {
- if (!ctx.client || !ctx.voiceAgentId) {
- throw new Error("Cannot stop the running voice agent while the host is unavailable");
- }
- await ctx.client.cancelAgent(ctx.voiceAgentId);
- }
-
await ctx.voice.stopVoice();
}
diff --git a/packages/app/src/contexts/session-context.tsx b/packages/app/src/contexts/session-context.tsx
index 475eb26f27..275742f3aa 100644
--- a/packages/app/src/contexts/session-context.tsx
+++ b/packages/app/src/contexts/session-context.tsx
@@ -1,5 +1,4 @@
import { useRef, ReactNode, useCallback, useEffect } from "react";
-import { Buffer } from "buffer";
import { AppState } from "react-native";
import { useQueryClient } from "@tanstack/react-query";
import { useTranslation } from "react-i18next";
@@ -34,8 +33,7 @@ import type { AgentSessionConfig } from "@getpaseo/protocol/agent-types";
import type { GitSetupOptions } from "@getpaseo/protocol/messages";
import type { AgentPermissionResponse } from "@getpaseo/protocol/agent-types";
import { getHostRuntimeStore, useHostRuntimeIsConnected } from "@/runtime/host-runtime";
-import { useVoiceAudioEngineOptional, useVoiceRuntimeOptional } from "@/contexts/voice-context";
-import type { AudioPlaybackSource } from "@/voice/audio-engine-types";
+import { useVoiceRuntimeOptional } from "@/contexts/voice-context";
import {
selectAgentTimelineState,
useSessionStore,
@@ -72,50 +70,11 @@ export type {
AgentFileExplorerState,
} from "@/stores/session-store";
-type AudioOutputPayload = Extract["payload"];
-
-interface BufferedAudioChunk {
- chunkIndex: number;
- audio: string;
- format: string;
- id: string;
-}
-
// COMPAT(selectiveAgentTimeline): added in v0.1.106, remove after 2027-01-12.
function getTimelineDeliveryMode(selectiveAgentTimeline?: boolean): TimelineDeliveryMode {
return selectiveAgentTimeline ? "selective" : "legacy";
}
-function decodeBase64Chunk(base64: string): Uint8Array {
- return Buffer.from(base64, "base64");
-}
-
-function buildAudioPlaybackSource(chunks: BufferedAudioChunk[]): AudioPlaybackSource {
- const decodedChunks = chunks.map((chunk) => decodeBase64Chunk(chunk.audio));
- const totalSize = decodedChunks.reduce((sum, chunk) => sum + chunk.length, 0);
- const output = new Uint8Array(totalSize);
- let offset = 0;
- for (const chunk of decodedChunks) {
- output.set(chunk, offset);
- offset += chunk.length;
- }
-
- const format = chunks[0]?.format ?? "pcm";
- let mimeType: string;
- if (format === "pcm") mimeType = "audio/pcm;rate=24000;bits=16";
- else if (format === "mp3") mimeType = "audio/mpeg";
- else mimeType = `audio/${format}`;
-
- const bytes = output.slice();
- return {
- size: bytes.byteLength,
- type: mimeType,
- async arrayBuffer() {
- return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
- },
- };
-}
-
const findLatestAssistantMessageText = (items: StreamItem[]): string | null => {
for (let i = items.length - 1; i >= 0; i -= 1) {
const item = items[i];
@@ -186,15 +145,6 @@ type WorkspaceSetupProgressPayload = Extract<
{ type: "workspace_setup_progress" }
>["payload"];
-function notifyVoiceAbortFailure(
- data: Extract["payload"],
- notifyError: (message: string) => void,
-): void {
- if (data.type === "error" && data.metadata?.voiceAbortFailed === true) {
- notifyError(data.content);
- }
-}
-
interface SessionProviderSharedProps {
children: ReactNode;
serverId: string;
@@ -222,7 +172,6 @@ export function SessionProvider(props: SessionProviderProps) {
function SessionProviderInternal({ children, serverId, client }: SessionProviderClientProps) {
const { t } = useTranslation();
const voiceRuntime = useVoiceRuntimeOptional();
- const voiceAudioEngine = useVoiceAudioEngineOptional();
const queryClient = useQueryClient();
const isConnected = useHostRuntimeIsConnected(serverId);
const toast = useToast();
@@ -255,8 +204,6 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider
const appStateRef = useRef(AppState.currentState);
const forcedTimelineTailReplacements = useRef(new Set());
const viewedTimelineSyncRef = useRef(null);
- const audioOutputBuffersRef = useRef>(new Map());
- const activeAudioGroupsRef = useRef>(new Set());
const isAppVisible = useAppVisible();
useEffect(() => {
@@ -369,29 +316,29 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider
useEffect(() => {
const unregister = voiceRuntime?.registerSession({
serverId,
- setVoiceMode: async (enabled, agentId) => {
+ startCall: async (context, transports) => {
if (!client) {
throw new Error(t("common.errors.daemonUnavailable"));
}
- await client.setVoiceMode(enabled, agentId);
+ return client.startVoiceCall(context, transports);
},
- sendVoiceAudioChunk: async (audioData, mimeType) => {
+ stopCall: async (callId) => {
if (!client) {
throw new Error(t("common.errors.daemonUnavailable"));
}
- await client.sendVoiceAudioChunk(audioData, mimeType);
+ await client.stopVoiceCall(callId);
},
- audioPlayed: async (chunkId) => {
+ updateCallContext: async (callId, context) => {
if (!client) {
throw new Error(t("common.errors.daemonUnavailable"));
}
- await client.audioPlayed(chunkId);
+ await client.updateVoiceCallContext(callId, context);
},
- abortRequest: async () => {
+ sendTransportMessage: (callId, data) => {
if (!client) {
throw new Error(t("common.errors.daemonUnavailable"));
}
- await client.abortRequest();
+ client.sendVoiceCallTransportMessage(callId, data);
},
setAssistantAudioPlaying: (isPlaying) => {
setIsPlayingAudio(serverId, isPlaying);
@@ -532,14 +479,6 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider
const { agentId, event, timestamp, seq, epoch } = message.payload;
const parsedTimestamp = new Date(timestamp);
const streamEvent = event;
- if (
- event.type === "turn_started" ||
- event.type === "turn_completed" ||
- event.type === "turn_failed" ||
- event.type === "turn_canceled"
- ) {
- voiceRuntime?.onTurnEvent(serverId, agentId, event.type);
- }
const turnLiveness = deriveAgentStreamTurnLiveness([
{ event: streamEvent, seq, epoch, timestamp: parsedTimestamp },
]);
@@ -661,94 +600,19 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider
});
});
- const unsubAudioOutput = client.on("audio_output", async (message) => {
- if (message.type !== "audio_output") return;
- if (!voiceAudioEngine) {
- return;
- }
-
- const payload: AudioOutputPayload = message.payload;
- if (payload.isVoiceMode && voiceRuntime) {
- voiceRuntime.handleAudioOutput(serverId, payload);
- return;
- }
-
- const playbackGroupId = payload.groupId ?? payload.id;
- const chunkIndex = payload.chunkIndex ?? 0;
- const isFinalChunk = payload.isLastChunk ?? true;
-
- if (!audioOutputBuffersRef.current.has(playbackGroupId)) {
- audioOutputBuffersRef.current.set(playbackGroupId, []);
- }
-
- const bufferedChunks = audioOutputBuffersRef.current.get(playbackGroupId)!;
- bufferedChunks.push({
- chunkIndex,
- audio: payload.audio,
- format: payload.format,
- id: payload.id,
- });
-
- activeAudioGroupsRef.current.add(playbackGroupId);
- setIsPlayingAudio(serverId, true);
-
- if (!isFinalChunk) {
- return;
- }
-
- bufferedChunks.sort((left, right) => left.chunkIndex - right.chunkIndex);
- const chunkIds = bufferedChunks.map((chunk) => chunk.id);
- const shouldPlay =
- !payload.isVoiceMode || (voiceRuntime?.shouldPlayVoiceAudio(serverId) ?? false);
- const audioBlob = buildAudioPlaybackSource(bufferedChunks);
- function logAudioPlayedError(error: unknown): void {
- console.warn("[Session] Failed to confirm audio playback:", error);
- }
- const confirmAudioPlayed = async () => {
- await Promise.all(
- chunkIds.map((chunkId) => client.audioPlayed(chunkId).catch(logAudioPlayedError)),
- );
- };
-
- let startedVoicePlayback = false;
- try {
- if (shouldPlay) {
- if (payload.isVoiceMode) {
- startedVoicePlayback = true;
- voiceRuntime?.onAssistantAudioStarted(serverId);
- }
- await voiceAudioEngine.play(audioBlob);
- }
- await confirmAudioPlayed();
- } catch (error) {
- console.error("[Session] Audio playback error:", error);
- await confirmAudioPlayed();
- } finally {
- audioOutputBuffersRef.current.delete(playbackGroupId);
- activeAudioGroupsRef.current.delete(playbackGroupId);
- setIsPlayingAudio(serverId, activeAudioGroupsRef.current.size > 0);
-
- if (startedVoicePlayback) {
- voiceRuntime?.onAssistantAudioFinished(serverId);
- }
- }
+ const unsubVoiceCallTransport = client.on("voice.call.transport.server", (message) => {
+ if (message.type !== "voice.call.transport.server" || !voiceRuntime) return;
+ voiceRuntime.handleTransportMessage(serverId, message.payload.callId, message.payload.data);
});
- const unsubActivity = client.on("activity_log", (message) => {
- if (message.type !== "activity_log") return;
- notifyVoiceAbortFailure(message.payload, toast.error);
+ const unsubVoiceCallState = client.on("voice.call.state", (message) => {
+ if (message.type !== "voice.call.state") return;
+ voiceRuntime?.handleCallState(serverId, message.payload);
});
- const unsubTranscription = client.on("transcription_result", (message) => {
- if (message.type !== "transcription_result") return;
-
- const transcriptText = message.payload.text.trim();
- voiceRuntime?.onTranscriptionResult(serverId, transcriptText);
- });
-
- const unsubVoiceInputState = client.on("voice_input_state", (message) => {
- if (message.type !== "voice_input_state") return;
- voiceRuntime?.onServerSpeechStateChanged(serverId, message.payload.isSpeaking);
+ const unsubVoiceCallEvent = client.on("voice.call.event", (message) => {
+ if (message.type !== "voice.call.event") return;
+ voiceRuntime?.handleCallEvent(serverId, message.payload.callId, message.payload.event);
});
const unsubTerminalAttention = client.on("terminal_attention_required", (message) => {
@@ -784,10 +648,9 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider
unsubStatus();
unsubPermissionRequest();
unsubPermissionResolved();
- unsubAudioOutput();
- unsubActivity();
- unsubTranscription();
- unsubVoiceInputState();
+ unsubVoiceCallTransport();
+ unsubVoiceCallState();
+ unsubVoiceCallEvent();
unsubTerminalAttention();
};
}, [
@@ -808,7 +671,6 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider
updateSessionServerInfo,
toast,
voiceRuntime,
- voiceAudioEngine,
]);
const _cancelAgentRun = useCallback(
diff --git a/packages/app/src/contexts/voice-context.tsx b/packages/app/src/contexts/voice-context.tsx
index 7bb0b9b7e1..9d158d7f67 100644
--- a/packages/app/src/contexts/voice-context.tsx
+++ b/packages/app/src/contexts/voice-context.tsx
@@ -8,7 +8,9 @@ import {
type ReactNode,
} from "react";
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
+import type { VoiceAppContext } from "@getpaseo/protocol/messages";
import { useSessionStore } from "@/stores/session-store";
+import { useToast } from "@/contexts/toast-context";
import { createAudioEngine } from "@/voice/audio-engine";
import type { AudioEngine } from "@/voice/audio-engine-types";
import {
@@ -16,12 +18,18 @@ import {
type VoiceRuntime,
type VoiceRuntimeSnapshot,
type VoiceRuntimeTelemetrySnapshot,
-} from "@/voice/voice-runtime";
+} from "@/voice-chat/runtime";
+import {
+ createDaemonAudioTransportFactory,
+ type DaemonAudioTransportFactory,
+} from "@/voice-chat/transports/daemon-audio";
interface VoiceContextValue extends VoiceRuntimeSnapshot {
- startVoice: (serverId: string, agentId: string) => Promise;
+ startCall: (serverId: string, context: VoiceAppContext) => Promise;
stopVoice: () => Promise;
- isVoiceModeForAgent: (serverId: string, agentId: string) => boolean;
+ updateCallContext: (context: VoiceAppContext) => Promise;
+ dismissTranscript: () => void;
+ isCallActiveOnServer: (serverId: string) => boolean;
toggleMute: () => void;
}
@@ -31,7 +39,8 @@ const EMPTY_SNAPSHOT: VoiceRuntimeSnapshot = {
isVoiceSwitching: false,
isMuted: false,
activeServerId: null,
- activeAgentId: null,
+ activeCallId: null,
+ events: [],
};
const EMPTY_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
@@ -73,9 +82,11 @@ export function useVoiceOptional(): VoiceContextValue | null {
}
return {
...snapshot,
- startVoice: runtime.startVoice,
+ startCall: runtime.startCall,
stopVoice: runtime.stopVoice,
- isVoiceModeForAgent: runtime.isVoiceModeForAgent,
+ updateCallContext: runtime.updateCallContext,
+ dismissTranscript: runtime.dismissTranscript,
+ isCallActiveOnServer: runtime.isCallActiveOnServer,
toggleMute: runtime.toggleMute,
};
}, [snapshot, runtime]);
@@ -113,17 +124,24 @@ interface VoiceProviderProps {
}
export function VoiceProvider({ children }: VoiceProviderProps) {
+ const toast = useToast();
const engineRef = useRef(null);
const runtimeRef = useRef(null);
if (!engineRef.current) {
let runtime: VoiceRuntime | null = null;
+ let daemonAudio: DaemonAudioTransportFactory | null = null;
+ const reportMediaError = (error: unknown) => {
+ const message = error instanceof Error ? error.message : "Voice media failed";
+ toast.error(message);
+ void runtime?.reportError(message);
+ };
const engine = createAudioEngine({
onCaptureData: (pcm) => {
- runtime?.handleCapturePcm(pcm);
+ daemonAudio?.handleCapturePcm(pcm);
},
onVolumeLevel: (level) => {
- runtime?.handleCaptureVolume(level);
+ daemonAudio?.handleCaptureVolume(level);
},
onInterruption: () => {
void runtime?.stopVoice().catch((error) => {
@@ -131,12 +149,13 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
});
},
onError: (error) => {
- console.error("[VoiceEngine] Capture error:", error);
+ reportMediaError(error);
},
});
+ daemonAudio = createDaemonAudioTransportFactory({ engine, onError: reportMediaError });
runtime = createVoiceRuntime({
- engine,
+ transports: [daemonAudio],
getServerInfo: (serverId) =>
useSessionStore.getState().getSession(serverId)?.serverInfo ?? null,
activateKeepAwake: async (tag) => {
@@ -156,11 +175,14 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
useEffect(() => {
return () => {
- void runtime.destroy().catch((error) => {
- console.error("[VoiceProvider] Failed to destroy voice runtime", error);
- });
+ void runtime
+ .destroy()
+ .then(() => engine.destroy())
+ .catch((error) => {
+ console.error("[VoiceProvider] Failed to destroy voice runtime", error);
+ });
};
- }, [runtime]);
+ }, [engine, runtime]);
return (
diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts
index e09ef311de..5f083a1ccc 100644
--- a/packages/app/src/i18n/resources/ar.ts
+++ b/packages/app/src/i18n/resources/ar.ts
@@ -1944,6 +1944,7 @@ export const ar: TranslationResources = {
projects: "المشاريع",
connections: "اتصالات",
agents: "Agents",
+ voice: "الصوت",
metadata: "البيانات الوصفية",
workspaces: "Workspaces",
providers: "مقدمي الخدمات",
diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts
index 2516a3f064..0b1a385c7f 100644
--- a/packages/app/src/i18n/resources/en.ts
+++ b/packages/app/src/i18n/resources/en.ts
@@ -1989,6 +1989,7 @@ export const en = {
projects: "Projects",
connections: "Connections",
agents: "Agents",
+ voice: "Voice",
metadata: "Metadata",
workspaces: "Workspaces",
providers: "Providers",
diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts
index 1b7ee5118a..89afe24b16 100644
--- a/packages/app/src/i18n/resources/es.ts
+++ b/packages/app/src/i18n/resources/es.ts
@@ -1990,6 +1990,7 @@ export const es: TranslationResources = {
projects: "Proyectos",
connections: "Conexiones",
agents: "Agents",
+ voice: "Voz",
metadata: "Metadatos",
workspaces: "Workspaces",
providers: "Proveedores",
diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts
index 5807c7e86f..7a9afbaf34 100644
--- a/packages/app/src/i18n/resources/fr.ts
+++ b/packages/app/src/i18n/resources/fr.ts
@@ -1995,6 +1995,7 @@ export const fr: TranslationResources = {
projects: "Projets",
connections: "Relations",
agents: "Agents",
+ voice: "Voix",
metadata: "Métadonnées",
workspaces: "Workspaces",
providers: "Fournisseurs",
diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts
index 1e2e95838d..27794528b3 100644
--- a/packages/app/src/i18n/resources/ja.ts
+++ b/packages/app/src/i18n/resources/ja.ts
@@ -1962,6 +1962,7 @@ export const ja: TranslationResources = {
projects: "プロジェクト",
connections: "接続",
agents: "エージェント",
+ voice: "音声",
metadata: "メタデータ",
workspaces: "ワークスペース",
providers: "プロバイダー",
diff --git a/packages/app/src/i18n/resources/ko.ts b/packages/app/src/i18n/resources/ko.ts
index a37feac7a3..8cdab4c362 100644
--- a/packages/app/src/i18n/resources/ko.ts
+++ b/packages/app/src/i18n/resources/ko.ts
@@ -1954,6 +1954,7 @@ export const ko: TranslationResources = {
projects: "프로젝트",
connections: "연결",
agents: "에이전트",
+ voice: "음성",
metadata: "메타데이터",
workspaces: "워크스페이스",
providers: "프로바이더",
diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts
index f6ee9bdc29..6388f0e694 100644
--- a/packages/app/src/i18n/resources/pt-BR.ts
+++ b/packages/app/src/i18n/resources/pt-BR.ts
@@ -1976,6 +1976,7 @@ export const ptBR: TranslationResources = {
projects: "Projetos",
connections: "Conexões",
agents: "Agentes",
+ voice: "Voz",
metadata: "Metadados",
workspaces: "Workspaces",
providers: "Provedores",
diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts
index eefd8aa147..ee2c9ce783 100644
--- a/packages/app/src/i18n/resources/ru.ts
+++ b/packages/app/src/i18n/resources/ru.ts
@@ -1975,6 +1975,7 @@ export const ru: TranslationResources = {
projects: "Проекты",
connections: "Подключения",
agents: "Агенты",
+ voice: "Голос",
metadata: "Метаданные",
workspaces: "Рабочие пространства",
providers: "Провайдеры",
diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts
index 35e160de59..2bb0651a24 100644
--- a/packages/app/src/i18n/resources/zh-CN.ts
+++ b/packages/app/src/i18n/resources/zh-CN.ts
@@ -1923,6 +1923,7 @@ export const zhCN: TranslationResources = {
projects: "项目",
connections: "连接",
agents: "Agents",
+ voice: "语音",
metadata: "元数据",
workspaces: "Workspaces",
providers: "Providers",
diff --git a/packages/app/src/screens/settings-screen.tsx b/packages/app/src/screens/settings-screen.tsx
index 5e5e9be474..e798486f6d 100644
--- a/packages/app/src/screens/settings-screen.tsx
+++ b/packages/app/src/screens/settings-screen.tsx
@@ -39,6 +39,7 @@ import {
Sparkles,
Blocks,
PanelsTopLeft,
+ AudioLines,
} from "lucide-react-native";
import { DropdownTrigger } from "@/components/ui/dropdown-trigger";
import { ComboboxTrigger } from "@/components/ui/combobox-trigger";
@@ -114,6 +115,7 @@ import {
} from "@/screens/settings/host-page";
import { HostPluginsPage } from "@/screens/settings/plugins-page";
import { MetadataGenerationPage } from "@/screens/settings/metadata-generation-page";
+import { VoiceSettingsPage } from "@/screens/settings/voice-page";
import ProjectsScreen from "@/screens/projects-screen";
import ProjectSettingsScreen from "@/screens/project-settings-screen";
import { SETTINGS_DESKTOP_SIDEBAR_WIDTH, useIsCompactFormFactor } from "@/constants/layout";
@@ -189,6 +191,7 @@ const HOST_SECTION_ITEMS: HostSectionItem[] = [
{ id: "connections", labelKey: "settings.hostSections.connections", icon: Network },
{ id: "pair-device", labelKey: "openProject.tiles.pairDevice.title", icon: Smartphone },
{ id: "agents", labelKey: "settings.hostSections.agents", icon: Bot },
+ { id: "voice", labelKey: "settings.hostSections.voice", icon: AudioLines },
{ id: "metadata", labelKey: "settings.hostSections.metadata", icon: Sparkles },
{ id: "workspaces", labelKey: "settings.hostSections.workspaces", icon: FolderGit2 },
{ id: "providers", labelKey: "settings.hostSections.providers", icon: Boxes },
@@ -210,6 +213,8 @@ function renderHostSettingsContent(
return ;
case "agents":
return ;
+ case "voice":
+ return ;
case "metadata":
return ;
case "workspaces":
diff --git a/packages/app/src/screens/settings/voice-page.tsx b/packages/app/src/screens/settings/voice-page.tsx
new file mode 100644
index 0000000000..26af5154ab
--- /dev/null
+++ b/packages/app/src/screens/settings/voice-page.tsx
@@ -0,0 +1,259 @@
+import { useCallback, useMemo, useState } from "react";
+import { Text, View } from "react-native";
+import type { AgentProvider, ProviderSnapshotEntry } from "@getpaseo/protocol/agent-types";
+import type { MutableManualVoiceOrchestratorConfig } from "@getpaseo/protocol/messages";
+import type { TFunction } from "i18next";
+import { StyleSheet } from "react-native-unistyles";
+import { useTranslation } from "react-i18next";
+import { CombinedModelSelector } from "@/components/combined-model-selector";
+import { LoadingSpinner } from "@/components/ui/loading-spinner";
+import { useToast } from "@/contexts/toast-context";
+import {
+ SelectField,
+ type SelectFieldDisplay,
+ type SelectFieldOption,
+} from "@/components/ui/select-field";
+import { useDaemonConfig } from "@/hooks/use-daemon-config";
+import { useProvidersSnapshot } from "@/hooks/use-providers-snapshot";
+import { buildSelectableProviderSelectorProviders } from "@/provider-selection/provider-selection";
+import type { ProviderSelectorProvider } from "@/provider-selection/provider-selection";
+import { SettingsSection } from "@/screens/settings/settings-section";
+import { settingsStyles } from "@/styles/settings";
+import { resolveVoiceOrchestratorSelection } from "./voice-settings-model";
+
+function findEntry(entries: ProviderSnapshotEntry[], provider: string | null) {
+ return entries.find((entry) => entry.provider === provider) ?? null;
+}
+
+function optionDisplay(
+ options: SelectFieldOption[],
+ value: string | null,
+): SelectFieldDisplay | null {
+ const option = options.find((candidate) => candidate.value === value);
+ return option ? { label: option.label, description: option.description } : null;
+}
+
+export function VoiceSettingsPage({ serverId }: { serverId: string }) {
+ const { t } = useTranslation();
+ const toast = useToast();
+ const { config, isLoading: isConfigLoading, patchConfig } = useDaemonConfig(serverId);
+ const snapshot = useProvidersSnapshot(serverId);
+ const entries = useMemo(() => snapshot.entries ?? [], [snapshot.entries]);
+ const [isSaving, setIsSaving] = useState(false);
+ const orchestrator = config?.manualVoice?.orchestrator;
+ const providers = useMemo(() => buildSelectableProviderSelectorProviders(entries), [entries]);
+ const selectedEntry = findEntry(entries, orchestrator?.provider ?? null);
+ const modeOptions = useMemo[]>(
+ () =>
+ (selectedEntry?.modes ?? []).map((mode) => ({
+ id: mode.id,
+ value: mode.id,
+ label: mode.label,
+ description: mode.description,
+ })),
+ [selectedEntry],
+ );
+ const selectedModel =
+ selectedEntry?.models?.find((model) => model.id === orchestrator?.model) ?? null;
+ const thinkingOptions = useMemo[]>(
+ () =>
+ (selectedModel?.thinkingOptions ?? []).map((option) => ({
+ id: option.id,
+ value: option.id,
+ label: option.label,
+ description: option.description,
+ })),
+ [selectedModel],
+ );
+
+ const save = useCallback(
+ async (next: {
+ provider?: string | null;
+ model?: string | null;
+ modeId?: string | null;
+ thinkingOptionId?: string | null;
+ }) => {
+ setIsSaving(true);
+ try {
+ await patchConfig({ manualVoice: { orchestrator: next } });
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : t("common.errors.unableToSave"));
+ } finally {
+ setIsSaving(false);
+ }
+ },
+ [patchConfig, t, toast],
+ );
+
+ const handleModelSelect = useCallback(
+ (provider: AgentProvider, model: string) => {
+ const selection = resolveVoiceOrchestratorSelection({
+ entries,
+ current: orchestrator,
+ provider,
+ model,
+ });
+ if (!selection) {
+ toast.error("This provider does not offer an agent mode.");
+ return;
+ }
+ void save(selection);
+ },
+ [entries, orchestrator, save, toast],
+ );
+
+ const handleModeSelect = useCallback((modeId: string) => void save({ modeId }), [save]);
+ const handleThinkingSelect = useCallback(
+ (thinkingOptionId: string) => void save({ thinkingOptionId }),
+ [save],
+ );
+ const handleSelectorOpen = useCallback(() => {
+ snapshot.refetchIfStale(orchestrator?.provider ?? undefined);
+ }, [orchestrator?.provider, snapshot]);
+ const handleRetryProvider = useCallback(
+ (provider: AgentProvider) => snapshot.refresh([provider]),
+ [snapshot],
+ );
+
+ if (isConfigLoading || !config) {
+ return (
+
+
+
+ );
+ }
+
+ return (
+
+ );
+}
+
+interface VoiceSettingsFieldsProps {
+ t: TFunction;
+ serverId: string;
+ orchestrator: MutableManualVoiceOrchestratorConfig | undefined;
+ providers: ProviderSelectorProvider[];
+ modeOptions: SelectFieldOption[];
+ thinkingOptions: SelectFieldOption[];
+ isLoadingProviders: boolean;
+ isRefreshingProvider: boolean;
+ isSaving: boolean;
+ onModelSelect(provider: AgentProvider, model: string): void;
+ onModeSelect(modeId: string): void;
+ onThinkingSelect(thinkingOptionId: string): void;
+ onSelectorOpen(): void;
+ onRetryProvider(provider: AgentProvider): void;
+}
+
+function VoiceSettingsFields(props: VoiceSettingsFieldsProps) {
+ const { orchestrator, modeOptions, thinkingOptions } = props;
+ return (
+
+
+ Choose the hidden orchestrator Paseo creates for each manual voice call. It receives Paseo
+ tools independently of ordinary agent MCP injection.
+
+
+
+
+ {props.t("modelSelector.model")}
+
+ Provider and model for the voice orchestrator
+
+
+
+
+
+
+ {props.t("agentControls.mode.title")}
+ Access and approval policy for voice work
+
+ 6}
+ size="sm"
+ field={false}
+ triggerTestID="voice-mode-trigger"
+ />
+
+ {thinkingOptions.length > 0 ? (
+
+
+ {props.t("agentControls.thinking.title")}
+ Reasoning effort for the orchestrator
+
+ 6}
+ size="sm"
+ field={false}
+ triggerTestID="voice-thinking-trigger"
+ />
+
+ ) : null}
+
+
+ );
+}
+
+const styles = StyleSheet.create((theme) => ({
+ description: {
+ color: theme.colors.foregroundMuted,
+ fontSize: theme.fontSize.sm,
+ lineHeight: theme.fontSize.sm * 1.45,
+ marginHorizontal: theme.spacing[1],
+ },
+ loading: {
+ alignItems: "center",
+ justifyContent: "center",
+ minHeight: 180,
+ },
+ spinnerColor: {
+ color: theme.colors.foregroundMuted,
+ },
+}));
diff --git a/packages/app/src/screens/settings/voice-settings-model.test.ts b/packages/app/src/screens/settings/voice-settings-model.test.ts
new file mode 100644
index 0000000000..51517265a9
--- /dev/null
+++ b/packages/app/src/screens/settings/voice-settings-model.test.ts
@@ -0,0 +1,73 @@
+import { describe, expect, it } from "vitest";
+import type { ProviderSnapshotEntry } from "@getpaseo/protocol/agent-types";
+import { resolveVoiceOrchestratorSelection } from "./voice-settings-model";
+
+const entries: ProviderSnapshotEntry[] = [
+ {
+ provider: "codex",
+ enabled: true,
+ status: "ready",
+ defaultModeId: "auto-review",
+ modes: [
+ { id: "auto-review", label: "Agent" },
+ { id: "full-access", label: "Full access" },
+ ],
+ models: [
+ {
+ provider: "codex",
+ id: "gpt-5.4",
+ label: "GPT-5.4",
+ defaultThinkingOptionId: "high",
+ thinkingOptions: [
+ { id: "medium", label: "Medium" },
+ { id: "high", label: "High" },
+ ],
+ },
+ ],
+ },
+];
+
+describe("voice orchestrator settings", () => {
+ it("always resolves an explicit provider mode with a model selection", () => {
+ expect(
+ resolveVoiceOrchestratorSelection({
+ entries,
+ current: undefined,
+ provider: "codex",
+ model: "gpt-5.4",
+ }),
+ ).toEqual({
+ provider: "codex",
+ model: "gpt-5.4",
+ modeId: "auto-review",
+ thinkingOptionId: "high",
+ });
+ });
+
+ it("preserves an explicitly selected valid mode", () => {
+ expect(
+ resolveVoiceOrchestratorSelection({
+ entries,
+ current: {
+ provider: "codex",
+ model: "gpt-5.4",
+ modeId: "full-access",
+ thinkingOptionId: "medium",
+ },
+ provider: "codex",
+ model: "gpt-5.4",
+ }),
+ ).toMatchObject({ modeId: "full-access", thinkingOptionId: "medium" });
+ });
+
+ it("rejects providers that cannot supply a mode", () => {
+ expect(
+ resolveVoiceOrchestratorSelection({
+ entries: [{ provider: "pi", enabled: true, status: "ready", modes: [] }],
+ current: undefined,
+ provider: "pi",
+ model: "",
+ }),
+ ).toBeNull();
+ });
+});
diff --git a/packages/app/src/screens/settings/voice-settings-model.ts b/packages/app/src/screens/settings/voice-settings-model.ts
new file mode 100644
index 0000000000..f0b91d7d0e
--- /dev/null
+++ b/packages/app/src/screens/settings/voice-settings-model.ts
@@ -0,0 +1,58 @@
+import type { AgentProvider, ProviderSnapshotEntry } from "@getpaseo/protocol/agent-types";
+import type { MutableManualVoiceOrchestratorConfig } from "@getpaseo/protocol/messages";
+
+function resolveModeId(input: {
+ entry: ProviderSnapshotEntry | undefined;
+ current: MutableManualVoiceOrchestratorConfig | undefined;
+ provider: AgentProvider;
+}): string | null {
+ const modes = input.entry?.modes ?? [];
+ const selected = modes.find((mode) => mode.id === input.current?.modeId)?.id;
+ if (input.current?.provider === input.provider && selected) return selected;
+ return input.entry?.defaultModeId ?? modes[0]?.id ?? null;
+}
+
+function resolveThinkingOptionId(input: {
+ entry: ProviderSnapshotEntry | undefined;
+ current: MutableManualVoiceOrchestratorConfig | undefined;
+ provider: AgentProvider;
+ model: string | null;
+}): string | null {
+ const definition = input.entry?.models?.find((candidate) => candidate.id === input.model);
+ const selected = definition?.thinkingOptions?.find(
+ (option) => option.id === input.current?.thinkingOptionId,
+ )?.id;
+ if (
+ input.current?.provider === input.provider &&
+ input.current.model === input.model &&
+ selected
+ ) {
+ return selected;
+ }
+ return (
+ definition?.defaultThinkingOptionId ??
+ definition?.thinkingOptions?.find((option) => option.isDefault)?.id ??
+ null
+ );
+}
+
+export function resolveVoiceOrchestratorSelection(input: {
+ entries: ProviderSnapshotEntry[];
+ current: MutableManualVoiceOrchestratorConfig | undefined;
+ provider: AgentProvider;
+ model: string;
+}): MutableManualVoiceOrchestratorConfig | null {
+ const entry = input.entries.find((candidate) => candidate.provider === input.provider);
+ const modeId = resolveModeId({ entry, current: input.current, provider: input.provider });
+ if (!modeId) return null;
+
+ const model = input.model || null;
+ const thinkingOptionId = resolveThinkingOptionId({
+ entry,
+ current: input.current,
+ provider: input.provider,
+ model,
+ });
+
+ return { provider: input.provider, model, modeId, thinkingOptionId };
+}
diff --git a/packages/app/src/utils/host-routes.ts b/packages/app/src/utils/host-routes.ts
index 5b8b6ad8d1..72e8ce49d7 100644
--- a/packages/app/src/utils/host-routes.ts
+++ b/packages/app/src/utils/host-routes.ts
@@ -513,6 +513,7 @@ export const HOST_SECTION_SLUGS = [
"connections",
"pair-device",
"agents",
+ "voice",
"metadata",
"workspaces",
"providers",
diff --git a/packages/app/src/voice-chat/runtime.test.ts b/packages/app/src/voice-chat/runtime.test.ts
new file mode 100644
index 0000000000..19c8faf1e2
--- /dev/null
+++ b/packages/app/src/voice-chat/runtime.test.ts
@@ -0,0 +1,273 @@
+import { describe, expect, it, vi } from "vitest";
+import type { DaemonServerInfo } from "@/stores/session-store";
+import { createVoiceRuntime, type VoiceSessionAdapter } from "@/voice-chat/runtime";
+import type {
+ VoiceClientTransport,
+ VoiceClientTransportFactory,
+ VoiceTransportMediaState,
+} from "@/voice-chat/transport";
+
+function deferred() {
+ let resolve: (value: T) => void = () => undefined;
+ const promise = new Promise((resolvePromise) => {
+ resolve = resolvePromise;
+ });
+ return { promise, resolve };
+}
+
+function createServerInfo(voiceChat = true): DaemonServerInfo {
+ return {
+ serverId: "server-1",
+ hostname: "host",
+ version: "1.0.0",
+ features: { voiceChat },
+ };
+}
+
+function createAdapter(): VoiceSessionAdapter {
+ return {
+ serverId: "server-1",
+ startCall: vi.fn().mockResolvedValue({
+ callId: "call-1",
+ transport: { kind: "webrtc", answerSdp: "answer-sdp" },
+ }),
+ stopCall: vi.fn().mockResolvedValue(undefined),
+ updateCallContext: vi.fn().mockResolvedValue(undefined),
+ sendTransportMessage: vi.fn(),
+ setAssistantAudioPlaying: vi.fn(),
+ };
+}
+
+function createWebRtcTransportProbe() {
+ let publish: ((state: VoiceTransportMediaState) => void) | null = null;
+ const transport: VoiceClientTransport = {
+ start: vi.fn().mockResolvedValue(undefined),
+ stop: vi.fn().mockResolvedValue(undefined),
+ handleServerMessage: vi.fn(),
+ updateCallState: vi.fn(),
+ toggleMute: vi.fn(),
+ };
+ const factory: VoiceClientTransportFactory = {
+ async prepare() {
+ return {
+ offer: { kind: "webrtc", offerSdp: "offer-sdp" },
+ accepts: (answer) => answer.kind === "webrtc",
+ async accept(input) {
+ if (input.answer.kind !== "webrtc") throw new Error("Expected WebRTC answer");
+ publish = input.publish;
+ return transport;
+ },
+ async discard() {},
+ };
+ },
+ };
+ return {
+ factory,
+ transport,
+ publishMedia(state: VoiceTransportMediaState) {
+ if (!publish) throw new Error("Transport was not accepted");
+ publish(state);
+ },
+ };
+}
+
+function setup(voiceChat = true) {
+ const transportProbe = createWebRtcTransportProbe();
+ const runtime = createVoiceRuntime({
+ transports: [transportProbe.factory],
+ getServerInfo: () => createServerInfo(voiceChat),
+ activateKeepAwake: vi.fn().mockResolvedValue(undefined),
+ deactivateKeepAwake: vi.fn().mockResolvedValue(undefined),
+ });
+ const adapter = createAdapter();
+ runtime.registerSession(adapter);
+ return { runtime, adapter, transportProbe };
+}
+
+describe("voice call runtime", () => {
+ it("starts through a WebRTC-shaped client transport without transport logic in runtime", async () => {
+ const { runtime, adapter, transportProbe } = setup();
+ const context = { workspaceId: "workspace-1", agentId: "agent-1" };
+
+ await runtime.startCall("server-1", context);
+
+ expect(adapter.startCall).toHaveBeenCalledWith(context, [
+ { kind: "webrtc", offerSdp: "offer-sdp" },
+ ]);
+ expect(transportProbe.transport.start).toHaveBeenCalledOnce();
+ expect(runtime.getSnapshot()).toMatchObject({
+ phase: "listening",
+ isVoiceMode: true,
+ activeServerId: "server-1",
+ activeCallId: "call-1",
+ });
+ });
+
+ it("derives mute, volume, and playback from transport-owned media state", async () => {
+ const { runtime, transportProbe } = setup();
+ await runtime.startCall("server-1", { workspaceId: null, agentId: null });
+
+ transportProbe.publishMedia({ isMuted: true, volume: 0.4, output: "speaking" });
+
+ expect(runtime.getSnapshot()).toMatchObject({ isMuted: true, phase: "playing" });
+ expect(runtime.getTelemetrySnapshot().volume).toBe(0.4);
+ });
+
+ it("keeps provider events as the lightweight transcript", async () => {
+ const { runtime } = setup();
+ await runtime.startCall("server-1", { workspaceId: null, agentId: null });
+
+ runtime.handleCallEvent("server-1", "call-1", {
+ type: "transcript",
+ id: "utterance-1",
+ speaker: "user",
+ text: "hello",
+ final: false,
+ });
+ runtime.handleCallEvent("server-1", "call-1", {
+ type: "transcript",
+ id: "utterance-1",
+ speaker: "user",
+ text: "hello there",
+ final: true,
+ });
+
+ expect(runtime.getSnapshot().events).toEqual([
+ expect.objectContaining({ id: "utterance-1", text: "hello there", final: true }),
+ ]);
+ });
+
+ it("stops transport and server call exactly once", async () => {
+ const { runtime, adapter, transportProbe } = setup();
+ await runtime.startCall("server-1", { workspaceId: null, agentId: null });
+
+ await runtime.stopVoice();
+
+ expect(adapter.stopCall).toHaveBeenCalledWith("call-1");
+ expect(transportProbe.transport.stop).toHaveBeenCalledOnce();
+ expect(runtime.getSnapshot().phase).toBe("disabled");
+ });
+
+ it("terminates locally on disconnect so reconnect starts clean", async () => {
+ const { runtime, transportProbe } = setup();
+ await runtime.startCall("server-1", { workspaceId: null, agentId: null });
+
+ runtime.updateSessionConnection("server-1", false);
+ await vi.waitFor(() => expect(runtime.getSnapshot().phase).toBe("disabled"));
+
+ expect(transportProbe.transport.stop).toHaveBeenCalledOnce();
+ });
+
+ it("stops a server call that is accepted after local startup cancellation", async () => {
+ const { runtime, adapter, transportProbe } = setup();
+ const started = deferred<{
+ callId: string;
+ transport: { kind: "webrtc"; answerSdp: string };
+ }>();
+ vi.mocked(adapter.startCall).mockReturnValue(started.promise);
+
+ const starting = runtime.startCall("server-1", { workspaceId: null, agentId: null });
+ await vi.waitFor(() => expect(adapter.startCall).toHaveBeenCalledOnce());
+ await runtime.stopVoice();
+ started.resolve({
+ callId: "late-call",
+ transport: { kind: "webrtc", answerSdp: "answer-sdp" },
+ });
+ await starting;
+
+ expect(adapter.stopCall).toHaveBeenCalledWith("late-call");
+ expect(transportProbe.transport.start).not.toHaveBeenCalled();
+ expect(runtime.getSnapshot().phase).toBe("disabled");
+ });
+
+ it("does not let an older startup replace a newer call", async () => {
+ const { runtime, adapter, transportProbe } = setup();
+ const older = deferred<{
+ callId: string;
+ transport: { kind: "webrtc"; answerSdp: string };
+ }>();
+ vi.mocked(adapter.startCall)
+ .mockReturnValueOnce(older.promise)
+ .mockResolvedValueOnce({
+ callId: "newer-call",
+ transport: { kind: "webrtc", answerSdp: "newer-answer" },
+ });
+
+ const olderStart = runtime.startCall("server-1", { workspaceId: null, agentId: null });
+ await vi.waitFor(() => expect(adapter.startCall).toHaveBeenCalledOnce());
+ await runtime.startCall("server-1", { workspaceId: null, agentId: null });
+ older.resolve({
+ callId: "older-call",
+ transport: { kind: "webrtc", answerSdp: "older-answer" },
+ });
+ await olderStart;
+
+ expect(runtime.getSnapshot()).toMatchObject({
+ phase: "listening",
+ activeCallId: "newer-call",
+ });
+ expect(adapter.stopCall).toHaveBeenCalledWith("older-call");
+ expect(transportProbe.transport.start).toHaveBeenCalledOnce();
+ });
+
+ it("forwards provider state and transport messages to the selected transport", async () => {
+ const { runtime, transportProbe } = setup();
+ await runtime.startCall("server-1", { workspaceId: null, agentId: null });
+ const state = {
+ callId: "call-1",
+ lifecycle: "active" as const,
+ input: "speech_detected" as const,
+ error: null,
+ };
+ const data = { type: "candidate", value: "server-data" };
+
+ runtime.handleCallState("server-1", state);
+ runtime.handleTransportMessage("server-1", "call-1", data);
+
+ expect(transportProbe.transport.updateCallState).toHaveBeenCalledWith(state);
+ expect(transportProbe.transport.handleServerMessage).toHaveBeenCalledWith(data);
+ expect(runtime.getTelemetrySnapshot().isSpeaking).toBe(true);
+ });
+
+ it("keeps provider terminal failures in the lightweight transcript", async () => {
+ const { runtime } = setup();
+ await runtime.startCall("server-1", { workspaceId: null, agentId: null });
+
+ runtime.handleCallState("server-1", {
+ callId: "call-1",
+ lifecycle: "failed",
+ input: "idle",
+ error: "Realtime authentication expired",
+ });
+ await vi.waitFor(() => expect(runtime.getSnapshot().phase).toBe("disabled"));
+
+ expect(runtime.getSnapshot().events).toContainEqual({
+ type: "error",
+ id: "call-terminal-call-1",
+ message: "Realtime authentication expired",
+ });
+ });
+
+ it("keeps media failure visible while terminating the call", async () => {
+ const { runtime, adapter } = setup();
+ await runtime.startCall("server-1", { workspaceId: null, agentId: null });
+
+ await runtime.reportError("Microphone disconnected");
+
+ expect(adapter.stopCall).toHaveBeenCalledWith("call-1");
+ expect(runtime.getSnapshot()).toMatchObject({ phase: "disabled" });
+ expect(runtime.getSnapshot().events).toContainEqual({
+ type: "error",
+ id: "media-error-call-1",
+ message: "Microphone disconnected",
+ });
+ });
+
+ it("rejects hosts without the new capability", async () => {
+ const { runtime, adapter } = setup(false);
+ await expect(
+ runtime.startCall("server-1", { workspaceId: null, agentId: null }),
+ ).rejects.toThrow("Update the Paseo host");
+ expect(adapter.startCall).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/app/src/voice-chat/runtime.ts b/packages/app/src/voice-chat/runtime.ts
new file mode 100644
index 0000000000..203046a00e
--- /dev/null
+++ b/packages/app/src/voice-chat/runtime.ts
@@ -0,0 +1,404 @@
+import type {
+ VoiceAppContext,
+ VoiceCallEvent,
+ VoiceCallState,
+ VoiceCallTransport,
+ VoiceCallTransportOffer,
+} from "@getpaseo/protocol/messages";
+import type { DaemonServerInfo } from "@/stores/session-store";
+import {
+ acceptVoiceClientTransport,
+ type PreparedVoiceClientTransport,
+ type VoiceClientTransport,
+ type VoiceClientTransportFactory,
+ type VoiceTransportMediaState,
+ type VoiceTransportSession,
+} from "@/voice-chat/transport";
+
+const KEEP_AWAKE_TAG = "paseo:voice";
+
+export type VoiceRuntimePhase = "disabled" | "starting" | "listening" | "playing" | "stopping";
+
+export interface VoiceRuntimeSnapshot {
+ phase: VoiceRuntimePhase;
+ isVoiceMode: boolean;
+ isVoiceSwitching: boolean;
+ isMuted: boolean;
+ activeServerId: string | null;
+ activeCallId: string | null;
+ events: VoiceCallEvent[];
+}
+
+export interface VoiceRuntimeTelemetrySnapshot {
+ volume: number;
+ isSpeaking: boolean;
+ segmentDuration: number;
+}
+
+export interface VoiceSessionAdapter extends VoiceTransportSession {
+ serverId: string;
+ startCall(
+ context: VoiceAppContext,
+ transports: VoiceCallTransportOffer[],
+ ): Promise<{ callId: string; transport: VoiceCallTransport }>;
+ stopCall(callId: string): Promise;
+ updateCallContext(callId: string, context: VoiceAppContext): Promise;
+}
+
+export interface VoiceRuntimeDeps {
+ transports: readonly VoiceClientTransportFactory[];
+ getServerInfo(serverId: string): DaemonServerInfo | null;
+ activateKeepAwake(tag: string): Promise;
+ deactivateKeepAwake(tag: string): Promise;
+}
+
+interface RuntimeSession {
+ adapter: VoiceSessionAdapter;
+ connected: boolean;
+}
+
+interface ActiveRuntimeCall {
+ serverId: string;
+ callId: string;
+ transport: VoiceClientTransport;
+ media: VoiceTransportMediaState;
+}
+
+const INITIAL_SNAPSHOT: VoiceRuntimeSnapshot = {
+ phase: "disabled",
+ isVoiceMode: false,
+ isVoiceSwitching: false,
+ isMuted: false,
+ activeServerId: null,
+ activeCallId: null,
+ events: [],
+};
+
+const INITIAL_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
+ volume: 0,
+ isSpeaking: false,
+ segmentDuration: 0,
+};
+
+function snapshotsEqual(left: VoiceRuntimeSnapshot, right: VoiceRuntimeSnapshot): boolean {
+ return (
+ left.phase === right.phase &&
+ left.isVoiceMode === right.isVoiceMode &&
+ left.isVoiceSwitching === right.isVoiceSwitching &&
+ left.isMuted === right.isMuted &&
+ left.activeServerId === right.activeServerId &&
+ left.activeCallId === right.activeCallId &&
+ left.events === right.events
+ );
+}
+
+function telemetryEqual(
+ left: VoiceRuntimeTelemetrySnapshot,
+ right: VoiceRuntimeTelemetrySnapshot,
+): boolean {
+ return (
+ left.volume === right.volume &&
+ left.isSpeaking === right.isSpeaking &&
+ left.segmentDuration === right.segmentDuration
+ );
+}
+
+export interface VoiceRuntime {
+ subscribe(listener: () => void): () => void;
+ getSnapshot(): VoiceRuntimeSnapshot;
+ subscribeTelemetry(listener: () => void): () => void;
+ getTelemetrySnapshot(): VoiceRuntimeTelemetrySnapshot;
+ registerSession(adapter: VoiceSessionAdapter): () => void;
+ updateSessionConnection(serverId: string, connected: boolean): void;
+ handleTransportMessage(serverId: string, callId: string, data: unknown): void;
+ handleCallState(serverId: string, callState: VoiceCallState): void;
+ handleCallEvent(serverId: string, callId: string, event: VoiceCallEvent): void;
+ startCall(serverId: string, context: VoiceAppContext): Promise;
+ stopVoice(): Promise;
+ updateCallContext(context: VoiceAppContext): Promise;
+ reportError(message: string): Promise;
+ dismissTranscript(): void;
+ destroy(): Promise;
+ toggleMute(): void;
+ isCallActiveOnServer(serverId: string): boolean;
+}
+
+export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
+ const listeners = new Set<() => void>();
+ const telemetryListeners = new Set<() => void>();
+ const sessions = new Map();
+ let snapshot = INITIAL_SNAPSHOT;
+ let telemetry = INITIAL_TELEMETRY;
+ let activeCall: ActiveRuntimeCall | null = null;
+ let generation = 0;
+ let speechStartedAt: number | null = null;
+ let segmentTimer: ReturnType | null = null;
+
+ function emit(): void {
+ for (const listener of listeners) listener();
+ }
+
+ function emitTelemetry(): void {
+ for (const listener of telemetryListeners) listener();
+ }
+
+ function patchSnapshot(
+ patch:
+ | Partial
+ | ((previous: VoiceRuntimeSnapshot) => VoiceRuntimeSnapshot),
+ ): void {
+ const next = typeof patch === "function" ? patch(snapshot) : { ...snapshot, ...patch };
+ if (snapshotsEqual(next, snapshot)) return;
+ snapshot = next;
+ emit();
+ }
+
+ function patchTelemetry(patch: Partial): void {
+ const next = { ...telemetry, ...patch };
+ if (telemetryEqual(next, telemetry)) return;
+ telemetry = next;
+ emitTelemetry();
+ }
+
+ function clearSegmentTimer(): void {
+ if (!segmentTimer) return;
+ clearInterval(segmentTimer);
+ segmentTimer = null;
+ }
+
+ function publishSpeechState(isSpeaking: boolean): void {
+ speechStartedAt = isSpeaking ? (speechStartedAt ?? Date.now()) : null;
+ patchTelemetry({ isSpeaking, ...(isSpeaking ? {} : { segmentDuration: 0 }) });
+ if (!isSpeaking) {
+ clearSegmentTimer();
+ return;
+ }
+ if (segmentTimer) return;
+ segmentTimer = setInterval(() => {
+ patchTelemetry({ segmentDuration: speechStartedAt ? Date.now() - speechStartedAt : 0 });
+ }, 100);
+ }
+
+ function publishMedia(callGeneration: number, media: VoiceTransportMediaState): void {
+ if (callGeneration !== generation || !activeCall) return;
+ activeCall.media = media;
+ patchTelemetry({ volume: media.volume });
+ patchSnapshot({
+ isMuted: media.isMuted,
+ phase: media.output === "speaking" ? "playing" : "listening",
+ });
+ }
+
+ async function stopLocal(call: ActiveRuntimeCall | null): Promise {
+ activeCall = null;
+ clearSegmentTimer();
+ speechStartedAt = null;
+ await call?.transport.stop().catch(() => undefined);
+ await deps.deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
+ patchTelemetry({ ...INITIAL_TELEMETRY });
+ patchSnapshot({ ...INITIAL_SNAPSHOT, events: snapshot.events });
+ }
+
+ async function prepareTransports(
+ session: RuntimeSession,
+ ): Promise {
+ if (deps.transports.length === 0) throw new Error("No voice transports are installed");
+ return Promise.all(deps.transports.map((factory) => factory.prepare(session.adapter)));
+ }
+
+ const api: VoiceRuntime = {
+ subscribe(listener) {
+ listeners.add(listener);
+ return () => listeners.delete(listener);
+ },
+ getSnapshot() {
+ return snapshot;
+ },
+ subscribeTelemetry(listener) {
+ telemetryListeners.add(listener);
+ return () => telemetryListeners.delete(listener);
+ },
+ getTelemetrySnapshot() {
+ return telemetry;
+ },
+ registerSession(adapter) {
+ sessions.set(adapter.serverId, { adapter, connected: true });
+ return () => {
+ sessions.delete(adapter.serverId);
+ if (snapshot.activeServerId === adapter.serverId && snapshot.phase !== "disabled") {
+ generation += 1;
+ void stopLocal(activeCall);
+ }
+ };
+ },
+ updateSessionConnection(serverId, connected) {
+ const session = sessions.get(serverId);
+ if (!session) return;
+ session.connected = connected;
+ if (!connected && snapshot.activeServerId === serverId && snapshot.phase !== "disabled") {
+ generation += 1;
+ void stopLocal(activeCall);
+ }
+ },
+ handleTransportMessage(serverId, callId, data) {
+ if (activeCall?.serverId !== serverId || activeCall.callId !== callId) return;
+ activeCall.transport.handleServerMessage(data);
+ },
+ handleCallState(serverId, callState) {
+ const call = activeCall;
+ if (!call || call.serverId !== serverId || call.callId !== callState.callId) return;
+ call.transport.updateCallState(callState);
+ publishSpeechState(callState.input === "speech_detected");
+ if (callState.lifecycle === "ended" || callState.lifecycle === "failed") {
+ if (callState.lifecycle === "failed" && callState.error) {
+ const eventId = `call-terminal-${call.callId}`;
+ const errorMessage = callState.error;
+ patchSnapshot((previous) => {
+ if (previous.events.some((event) => event.id === eventId)) return previous;
+ return {
+ ...previous,
+ events: [...previous.events, { type: "error", id: eventId, message: errorMessage }],
+ };
+ });
+ }
+ generation += 1;
+ void stopLocal(call);
+ }
+ },
+ handleCallEvent(serverId, callId, event) {
+ if (activeCall?.serverId !== serverId || activeCall.callId !== callId) return;
+ patchSnapshot((previous) => {
+ const index = previous.events.findIndex((candidate) => candidate.id === event.id);
+ const events =
+ index === -1 ? [...previous.events, event] : previous.events.with(index, event);
+ return { ...previous, events };
+ });
+ },
+ async startCall(serverId, context) {
+ const session = sessions.get(serverId);
+ if (!session) throw new Error(`Voice runtime is not ready for host ${serverId}`);
+ if (!session.connected) throw new Error(`Host ${serverId} is not connected`);
+ if (deps.getServerInfo(serverId)?.features?.voiceChat !== true) {
+ throw new Error("Update the Paseo host to use voice chat.");
+ }
+
+ const callGeneration = ++generation;
+ const previousCall = activeCall;
+ const previousSession = previousCall ? sessions.get(previousCall.serverId) : null;
+ activeCall = null;
+ patchSnapshot({
+ ...snapshot,
+ isVoiceSwitching: true,
+ phase: "starting",
+ activeServerId: serverId,
+ activeCallId: null,
+ events: [],
+ });
+ if (previousCall) {
+ await previousSession?.adapter.stopCall(previousCall.callId).catch(() => undefined);
+ await previousCall.transport.stop().catch(() => undefined);
+ }
+
+ let prepared: PreparedVoiceClientTransport[] = [];
+ let acceptedCallId: string | null = null;
+ let transport: VoiceClientTransport | null = null;
+ try {
+ await deps.activateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
+ prepared = await prepareTransports(session);
+ const started = await session.adapter.startCall(
+ context,
+ prepared.map((candidate) => candidate.offer),
+ );
+ acceptedCallId = started.callId;
+ if (generation !== callGeneration) {
+ await Promise.all(prepared.map((candidate) => candidate.discard()));
+ await session.adapter.stopCall(started.callId).catch(() => undefined);
+ return;
+ }
+ patchSnapshot({ activeCallId: started.callId });
+ transport = await acceptVoiceClientTransport({
+ prepared,
+ callId: started.callId,
+ answer: started.transport,
+ publish: (media) => publishMedia(callGeneration, media),
+ });
+ if (generation !== callGeneration) {
+ await transport.stop();
+ await session.adapter.stopCall(started.callId).catch(() => undefined);
+ return;
+ }
+ const call: ActiveRuntimeCall = {
+ serverId,
+ callId: started.callId,
+ transport,
+ media: { isMuted: false, volume: 0, output: "idle" },
+ };
+ activeCall = call;
+ await transport.start();
+ if (generation !== callGeneration) {
+ if (activeCall === call) activeCall = null;
+ await transport.stop();
+ await session.adapter.stopCall(started.callId).catch(() => undefined);
+ return;
+ }
+ patchSnapshot({
+ isVoiceMode: true,
+ isVoiceSwitching: false,
+ phase: "listening",
+ isMuted: call.media.isMuted,
+ });
+ } catch (error) {
+ await Promise.all(prepared.map((candidate) => candidate.discard().catch(() => undefined)));
+ await transport?.stop().catch(() => undefined);
+ if (acceptedCallId) await session.adapter.stopCall(acceptedCallId).catch(() => undefined);
+ if (generation === callGeneration) await stopLocal(activeCall);
+ throw error;
+ }
+ },
+ async stopVoice() {
+ const call = activeCall;
+ if (!call && snapshot.phase === "disabled") return;
+ const stopGeneration = ++generation;
+ patchSnapshot({ isVoiceSwitching: true, phase: "stopping" });
+ try {
+ const session = call ? sessions.get(call.serverId) : null;
+ if (call && session?.connected) await session.adapter.stopCall(call.callId);
+ } finally {
+ if (stopGeneration === generation) await stopLocal(call);
+ }
+ },
+ async updateCallContext(context) {
+ const call = activeCall;
+ if (!call) return;
+ const session = sessions.get(call.serverId);
+ if (!session?.connected) return;
+ await session.adapter.updateCallContext(call.callId, context);
+ },
+ async reportError(message) {
+ const call = activeCall;
+ if (!call) return;
+ patchSnapshot((previous) => ({
+ ...previous,
+ events: [...previous.events, { type: "error", id: `media-error-${call.callId}`, message }],
+ }));
+ await api.stopVoice();
+ },
+ dismissTranscript() {
+ patchSnapshot((previous) => ({ ...previous, events: [] }));
+ },
+ async destroy() {
+ await api.stopVoice().catch(() => undefined);
+ listeners.clear();
+ telemetryListeners.clear();
+ sessions.clear();
+ },
+ toggleMute() {
+ activeCall?.transport.toggleMute();
+ },
+ isCallActiveOnServer(serverId) {
+ return activeCall?.serverId === serverId && snapshot.isVoiceMode;
+ },
+ };
+
+ return api;
+}
diff --git a/packages/app/src/voice-chat/transport.ts b/packages/app/src/voice-chat/transport.ts
new file mode 100644
index 0000000000..4582c9e0cd
--- /dev/null
+++ b/packages/app/src/voice-chat/transport.ts
@@ -0,0 +1,60 @@
+import type {
+ VoiceCallState,
+ VoiceCallTransport,
+ VoiceCallTransportOffer,
+} from "@getpaseo/protocol/messages";
+
+export interface VoiceTransportMediaState {
+ isMuted: boolean;
+ volume: number;
+ output: "idle" | "speaking";
+}
+
+export interface VoiceTransportSession {
+ sendTransportMessage(callId: string, data: unknown): void;
+ setAssistantAudioPlaying(isPlaying: boolean): void;
+}
+
+export interface VoiceClientTransport {
+ start(): Promise;
+ stop(): Promise;
+ handleServerMessage(data: unknown): void;
+ updateCallState(state: VoiceCallState): void;
+ toggleMute(): void;
+}
+
+export interface PreparedVoiceClientTransport {
+ readonly offer: VoiceCallTransportOffer;
+ accepts(answer: VoiceCallTransport): boolean;
+ accept(input: {
+ callId: string;
+ answer: VoiceCallTransport;
+ publish(state: VoiceTransportMediaState): void;
+ }): Promise;
+ discard(): Promise;
+}
+
+export interface VoiceClientTransportFactory {
+ prepare(session: VoiceTransportSession): Promise;
+}
+
+export async function acceptVoiceClientTransport(input: {
+ prepared: PreparedVoiceClientTransport[];
+ callId: string;
+ answer: VoiceCallTransport;
+ publish(state: VoiceTransportMediaState): void;
+}): Promise {
+ const selected = input.prepared.find((candidate) => candidate.accepts(input.answer));
+ if (!selected) {
+ await Promise.all(input.prepared.map((candidate) => candidate.discard()));
+ throw new Error(`No installed client transport accepted '${input.answer.kind}'`);
+ }
+ const unselected = input.prepared.filter((candidate) => candidate !== selected);
+ const transport = await selected.accept({
+ callId: input.callId,
+ answer: input.answer,
+ publish: input.publish,
+ });
+ await Promise.all(unselected.map((candidate) => candidate.discard()));
+ return transport;
+}
diff --git a/packages/app/src/voice-chat/transports/daemon-audio.test.ts b/packages/app/src/voice-chat/transports/daemon-audio.test.ts
new file mode 100644
index 0000000000..4a7568f95a
--- /dev/null
+++ b/packages/app/src/voice-chat/transports/daemon-audio.test.ts
@@ -0,0 +1,95 @@
+import { expect, test, vi } from "vitest";
+import type { AudioEngine } from "@/voice/audio-engine-types";
+import { createDaemonAudioTransportFactory } from "@/voice-chat/transports/daemon-audio";
+import type { VoiceTransportMediaState } from "@/voice-chat/transport";
+
+function createEngine(): AudioEngine {
+ return {
+ initialize: vi.fn().mockResolvedValue(undefined),
+ destroy: vi.fn().mockResolvedValue(undefined),
+ startCapture: vi.fn().mockResolvedValue(undefined),
+ stopCapture: vi.fn().mockResolvedValue(undefined),
+ toggleMute: vi.fn().mockReturnValue(true),
+ isMuted: vi.fn().mockReturnValue(false),
+ play: vi.fn().mockResolvedValue(0.1),
+ stop: vi.fn(),
+ clearQueue: vi.fn(),
+ isPlaying: vi.fn().mockReturnValue(false),
+ };
+}
+
+test("daemon audio owns capture, playback, acknowledgement, and mute state", async () => {
+ const engine = createEngine();
+ const sent = vi.fn();
+ const playback = vi.fn();
+ const onError = vi.fn();
+ const media: VoiceTransportMediaState[] = [];
+ const factory = createDaemonAudioTransportFactory({ engine, onError });
+ const prepared = await factory.prepare({
+ sendTransportMessage: sent,
+ setAssistantAudioPlaying: playback,
+ });
+ const transport = await prepared.accept({
+ callId: "call-1",
+ answer: { kind: "daemon-audio" },
+ publish: (state) => media.push(state),
+ });
+ await transport.start();
+
+ factory.handleCapturePcm(new Uint8Array([1, 2, 3]));
+ transport.handleServerMessage({
+ type: "audio",
+ outputId: "audio-1",
+ audio: "AQID",
+ format: "pcm",
+ isLastChunk: true,
+ });
+ transport.toggleMute();
+
+ await vi.waitFor(() => {
+ expect(sent).toHaveBeenCalledWith("call-1", { type: "played", outputId: "audio-1" });
+ });
+ expect(sent).toHaveBeenCalledWith("call-1", {
+ type: "append",
+ audio: "AQID",
+ format: "audio/pcm;rate=16000;bits=16",
+ });
+ expect(playback).toHaveBeenNthCalledWith(1, true);
+ expect(playback).toHaveBeenLastCalledWith(false);
+ expect(media).toContainEqual(expect.objectContaining({ isMuted: true }));
+
+ await transport.stop();
+ expect(engine.stopCapture).toHaveBeenCalledOnce();
+});
+
+test("failed playback reports the error without acknowledging success", async () => {
+ const engine = createEngine();
+ vi.mocked(engine.play).mockRejectedValueOnce(new Error("speaker unavailable"));
+ const sent = vi.fn();
+ const onError = vi.fn();
+ const prepared = await createDaemonAudioTransportFactory({ engine, onError }).prepare({
+ sendTransportMessage: sent,
+ setAssistantAudioPlaying: vi.fn(),
+ });
+ const transport = await prepared.accept({
+ callId: "call-1",
+ answer: { kind: "daemon-audio" },
+ publish: vi.fn(),
+ });
+ await transport.start();
+
+ transport.handleServerMessage({
+ type: "audio",
+ outputId: "audio-1",
+ audio: "AQID",
+ format: "pcm",
+ isLastChunk: true,
+ });
+ await vi.waitFor(() => expect(onError).toHaveBeenCalledWith(expect.any(Error)));
+
+ expect(sent).not.toHaveBeenCalledWith("call-1", {
+ type: "played",
+ outputId: "audio-1",
+ });
+ await transport.stop();
+});
diff --git a/packages/app/src/voice-chat/transports/daemon-audio.ts b/packages/app/src/voice-chat/transports/daemon-audio.ts
new file mode 100644
index 0000000000..d4ec4b3a96
--- /dev/null
+++ b/packages/app/src/voice-chat/transports/daemon-audio.ts
@@ -0,0 +1,348 @@
+import { Buffer } from "buffer";
+import { z } from "zod";
+import type { VoiceCallState } from "@getpaseo/protocol/messages";
+import type { AudioEngine } from "@/voice/audio-engine-types";
+import type {
+ PreparedVoiceClientTransport,
+ VoiceClientTransport,
+ VoiceClientTransportFactory,
+ VoiceTransportMediaState,
+ VoiceTransportSession,
+} from "@/voice-chat/transport";
+
+const PCM_MIME_TYPE = "audio/pcm;rate=16000;bits=16";
+const DISPLAY_VOLUME_PUBLISH_INTERVAL_MS = 120;
+const DISPLAY_VOLUME_CHANGE_EPSILON = 0.02;
+const DISPLAY_VOLUME_ATTACK = 0.35;
+const DISPLAY_VOLUME_RELEASE = 0.18;
+
+const DaemonAudioServerMessageSchema = z.object({
+ type: z.literal("audio"),
+ outputId: z.string(),
+ audio: z.string(),
+ format: z.string(),
+ groupId: z.string().optional(),
+ chunkIndex: z.number().int().nonnegative().optional(),
+ isLastChunk: z.boolean().optional(),
+});
+
+interface PlaybackSource {
+ arrayBuffer(): Promise;
+ size: number;
+ type: string;
+}
+
+interface PlaybackChunk {
+ id: string;
+ source: PlaybackSource;
+}
+
+interface PlaybackGroup {
+ groupId: string;
+ chunks: Map;
+ nextChunkToPlay: number;
+ finalChunkIndex: number | null;
+ started: boolean;
+ acknowledged: Set;
+}
+
+interface PlaybackState {
+ groups: Map;
+ orderedGroupIds: string[];
+ activeGroupId: string | null;
+ processing: boolean;
+ generation: number;
+}
+
+export interface DaemonAudioTransportFactory extends VoiceClientTransportFactory {
+ handleCapturePcm(chunk: Uint8Array): void;
+ handleCaptureVolume(level: number): void;
+}
+
+export function createDaemonAudioTransportFactory(input: {
+ engine: AudioEngine;
+ onError(error: unknown): void;
+}): DaemonAudioTransportFactory {
+ let active: DaemonAudioTransport | null = null;
+
+ return {
+ async prepare(session) {
+ return createPreparedDaemonAudioTransport({
+ engine: input.engine,
+ onError: input.onError,
+ session,
+ activate(transport) {
+ active = transport;
+ },
+ deactivate(transport) {
+ if (active === transport) active = null;
+ },
+ });
+ },
+ handleCapturePcm(chunk) {
+ active?.handleCapturePcm(chunk);
+ },
+ handleCaptureVolume(level) {
+ active?.handleCaptureVolume(level);
+ },
+ };
+}
+
+function createPreparedDaemonAudioTransport(input: {
+ engine: AudioEngine;
+ onError(error: unknown): void;
+ session: VoiceTransportSession;
+ activate(transport: DaemonAudioTransport): void;
+ deactivate(transport: DaemonAudioTransport): void;
+}): PreparedVoiceClientTransport {
+ return {
+ offer: { kind: "daemon-audio" },
+ accepts(answer) {
+ return answer.kind === "daemon-audio";
+ },
+ async accept({ callId, answer, publish }) {
+ if (answer.kind !== "daemon-audio") {
+ throw new Error(`Daemon audio cannot accept '${answer.kind}'`);
+ }
+ const transport = new DaemonAudioTransport({
+ callId,
+ engine: input.engine,
+ onError: input.onError,
+ session: input.session,
+ publish,
+ deactivate: input.deactivate,
+ });
+ input.activate(transport);
+ return transport;
+ },
+ async discard() {},
+ };
+}
+
+class DaemonAudioTransport implements VoiceClientTransport {
+ private readonly callId: string;
+ private readonly engine: AudioEngine;
+ private readonly onError: (error: unknown) => void;
+ private readonly session: VoiceTransportSession;
+ private readonly publish: (state: VoiceTransportMediaState) => void;
+ private readonly deactivate: (transport: DaemonAudioTransport) => void;
+ private media: VoiceTransportMediaState = { isMuted: false, volume: 0, output: "idle" };
+ private playback: PlaybackState = {
+ groups: new Map(),
+ orderedGroupIds: [],
+ activeGroupId: null,
+ processing: false,
+ generation: 0,
+ };
+ private started = false;
+ private stopped = false;
+ private lastVolumePublishMs = 0;
+
+ constructor(input: {
+ callId: string;
+ engine: AudioEngine;
+ onError(error: unknown): void;
+ session: VoiceTransportSession;
+ publish(state: VoiceTransportMediaState): void;
+ deactivate(transport: DaemonAudioTransport): void;
+ }) {
+ this.callId = input.callId;
+ this.engine = input.engine;
+ this.onError = input.onError;
+ this.session = input.session;
+ this.publish = input.publish;
+ this.deactivate = input.deactivate;
+ }
+
+ async start(): Promise {
+ if (this.started) return;
+ await this.engine.initialize();
+ await this.engine.startCapture();
+ this.started = true;
+ this.patchMedia({ isMuted: this.engine.isMuted() });
+ }
+
+ async stop(): Promise {
+ if (this.stopped) return;
+ this.stopped = true;
+ this.resetPlayback();
+ this.engine.stop();
+ this.engine.clearQueue();
+ this.session.setAssistantAudioPlaying(false);
+ await this.engine.stopCapture().catch(() => undefined);
+ this.deactivate(this);
+ this.patchMedia({ volume: 0, output: "idle" });
+ }
+
+ handleCapturePcm(chunk: Uint8Array): void {
+ if (!this.started || this.stopped || this.media.isMuted || chunk.byteLength === 0) return;
+ this.session.sendTransportMessage(this.callId, {
+ type: "append",
+ audio: Buffer.from(chunk).toString("base64"),
+ format: PCM_MIME_TYPE,
+ });
+ }
+
+ handleCaptureVolume(level: number): void {
+ const now = Date.now();
+ const displayed = this.media.isMuted ? 0 : level;
+ const smoothing =
+ displayed >= this.media.volume ? DISPLAY_VOLUME_ATTACK : DISPLAY_VOLUME_RELEASE;
+ const next = Math.max(
+ 0,
+ Math.min(1, this.media.volume + (displayed - this.media.volume) * smoothing),
+ );
+ const enoughTime = now - this.lastVolumePublishMs >= DISPLAY_VOLUME_PUBLISH_INTERVAL_MS;
+ const enoughChange = Math.abs(next - this.media.volume) >= DISPLAY_VOLUME_CHANGE_EPSILON;
+ if (!enoughTime && !enoughChange) return;
+ this.lastVolumePublishMs = now;
+ this.patchMedia({ volume: Number(next.toFixed(3)) });
+ }
+
+ handleServerMessage(data: unknown): void {
+ if (this.stopped) return;
+ const parsed = DaemonAudioServerMessageSchema.safeParse(data);
+ if (!parsed.success) return;
+ const payload = parsed.data;
+ const groupId = payload.groupId ?? payload.outputId;
+ const chunkIndex = payload.chunkIndex ?? 0;
+ let group = this.playback.groups.get(groupId);
+ if (!group) {
+ group = {
+ groupId,
+ chunks: new Map(),
+ nextChunkToPlay: 0,
+ finalChunkIndex: null,
+ started: false,
+ acknowledged: new Set(),
+ };
+ this.playback.groups.set(groupId, group);
+ this.playback.orderedGroupIds.push(groupId);
+ this.playback.activeGroupId ??= groupId;
+ }
+ group.chunks.set(chunkIndex, {
+ id: payload.outputId,
+ source: toPlaybackSource(Buffer.from(payload.audio, "base64"), payload.format),
+ });
+ if (payload.isLastChunk) group.finalChunkIndex = chunkIndex;
+ void this.processPlayback();
+ }
+
+ updateCallState(state: VoiceCallState): void {
+ if (state.callId !== this.callId || state.input !== "speech_detected") return;
+ const hadPlayback = this.playback.groups.size > 0;
+ this.resetPlayback();
+ if (hadPlayback) {
+ this.engine.stop();
+ this.engine.clearQueue();
+ this.session.setAssistantAudioPlaying(false);
+ this.patchMedia({ output: "idle" });
+ }
+ }
+
+ toggleMute(): void {
+ const isMuted = this.engine.toggleMute();
+ this.patchMedia({ isMuted, ...(isMuted ? { volume: 0 } : {}) });
+ }
+
+ private patchMedia(patch: Partial): void {
+ this.media = { ...this.media, ...patch };
+ this.publish(this.media);
+ }
+
+ private resetPlayback(): void {
+ this.playback.generation += 1;
+ this.playback.groups.clear();
+ this.playback.orderedGroupIds = [];
+ this.playback.activeGroupId = null;
+ this.playback.processing = false;
+ }
+
+ private activateNextGroup(): void {
+ while (this.playback.orderedGroupIds.length > 0) {
+ const groupId = this.playback.orderedGroupIds[0];
+ if (this.playback.groups.has(groupId)) {
+ this.playback.activeGroupId = groupId;
+ return;
+ }
+ this.playback.orderedGroupIds.shift();
+ }
+ this.playback.activeGroupId = null;
+ }
+
+ private retireGroup(group: PlaybackGroup): void {
+ this.playback.groups.delete(group.groupId);
+ this.playback.orderedGroupIds = this.playback.orderedGroupIds.filter(
+ (groupId) => groupId !== group.groupId,
+ );
+ if (group.started && this.playback.groups.size === 0) {
+ this.session.setAssistantAudioPlaying(false);
+ this.patchMedia({ output: "idle" });
+ }
+ }
+
+ private async processPlayback(): Promise {
+ if (this.playback.processing) return;
+ this.playback.processing = true;
+ const generation = this.playback.generation;
+ try {
+ while (this.playback.activeGroupId && generation === this.playback.generation) {
+ const group = this.playback.groups.get(this.playback.activeGroupId);
+ if (!group) {
+ this.activateNextGroup();
+ continue;
+ }
+ const chunk = group.chunks.get(group.nextChunkToPlay);
+ if (!chunk) {
+ const complete =
+ group.finalChunkIndex !== null && group.nextChunkToPlay > group.finalChunkIndex;
+ if (!complete) return;
+ this.retireGroup(group);
+ this.activateNextGroup();
+ continue;
+ }
+ group.chunks.delete(group.nextChunkToPlay);
+ if (!group.started) {
+ group.started = true;
+ this.session.setAssistantAudioPlaying(true);
+ this.patchMedia({ output: "speaking" });
+ }
+ try {
+ await this.engine.play(chunk.source);
+ } catch (error) {
+ if (generation !== this.playback.generation) return;
+ this.onError(error);
+ return;
+ }
+ if (generation !== this.playback.generation) return;
+ if (!group.acknowledged.has(chunk.id)) {
+ group.acknowledged.add(chunk.id);
+ this.session.sendTransportMessage(this.callId, {
+ type: "played",
+ outputId: chunk.id,
+ });
+ }
+ group.nextChunkToPlay += 1;
+ }
+ } finally {
+ if (generation === this.playback.generation) this.playback.processing = false;
+ }
+ }
+}
+
+function toPlaybackSource(bytes: Uint8Array, format: string): PlaybackSource {
+ const type = resolveMimeType(format);
+ return {
+ size: bytes.byteLength,
+ type,
+ async arrayBuffer() {
+ return Uint8Array.from(bytes).buffer;
+ },
+ };
+}
+
+function resolveMimeType(format: string): string {
+ if (format === "pcm") return "audio/pcm;rate=24000;bits=16";
+ if (format === "mp3") return "audio/mpeg";
+ return `audio/${format}`;
+}
diff --git a/packages/app/src/voice/voice-runtime.test.ts b/packages/app/src/voice/voice-runtime.test.ts
deleted file mode 100644
index aa79d6b245..0000000000
--- a/packages/app/src/voice/voice-runtime.test.ts
+++ /dev/null
@@ -1,479 +0,0 @@
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import type { DaemonServerInfo } from "@/stores/session-store";
-import type { AudioEngine } from "@/voice/audio-engine-types";
-import { createVoiceRuntime, type VoiceSessionAdapter } from "@/voice/voice-runtime";
-import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
-
-function createAudioEngineMock(): AudioEngine {
- return {
- initialize: vi.fn().mockResolvedValue(undefined),
- destroy: vi.fn().mockResolvedValue(undefined),
- startCapture: vi.fn().mockResolvedValue(undefined),
- stopCapture: vi.fn().mockResolvedValue(undefined),
- toggleMute: vi.fn().mockReturnValue(true),
- isMuted: vi.fn().mockReturnValue(false),
- play: vi.fn().mockResolvedValue(0.1),
- stop: vi.fn(),
- clearQueue: vi.fn(),
- isPlaying: vi.fn().mockReturnValue(false),
- };
-}
-
-function createSessionAdapter(serverId = "server-1"): VoiceSessionAdapter {
- return {
- serverId,
- setVoiceMode: vi.fn().mockResolvedValue(undefined),
- sendVoiceAudioChunk: vi.fn().mockResolvedValue(undefined),
- audioPlayed: vi.fn().mockResolvedValue(undefined),
- abortRequest: vi.fn().mockResolvedValue(undefined),
- setAssistantAudioPlaying: vi.fn(),
- };
-}
-
-function createAudioPayload(args: {
- id: string;
- groupId: string;
- chunkIndex: number;
- isLastChunk: boolean;
- isVoiceMode?: boolean;
-}) {
- return {
- id: args.id,
- groupId: args.groupId,
- chunkIndex: args.chunkIndex,
- isLastChunk: args.isLastChunk,
- isVoiceMode: args.isVoiceMode ?? true,
- format: "pcm",
- audio: Buffer.from(`chunk-${args.chunkIndex}`).toString("base64"),
- } as const;
-}
-
-function createServerInfo(): DaemonServerInfo {
- return {
- serverId: "server-1",
- hostname: "host",
- version: "1.0.0",
- capabilities: {
- voice: {
- dictation: { enabled: true, reason: "" },
- voice: { enabled: true, reason: "" },
- },
- },
- };
-}
-
-function createRuntime(options?: {
- engine?: AudioEngine;
- getServerInfo?: (serverId: string) => DaemonServerInfo | null;
-}) {
- const engine = options?.engine ?? createAudioEngineMock();
- const runtime = createVoiceRuntime({
- engine,
- getServerInfo: options?.getServerInfo ?? (() => createServerInfo()),
- activateKeepAwake: vi.fn().mockResolvedValue(undefined),
- deactivateKeepAwake: vi.fn().mockResolvedValue(undefined),
- });
-
- return { runtime, engine };
-}
-
-describe("voice runtime", () => {
- beforeEach(() => {
- vi.useFakeTimers();
- });
-
- afterEach(() => {
- vi.useRealTimers();
- });
-
- it("starts voice when adapter is ready", async () => {
- const adapter = createSessionAdapter();
- const { runtime, engine } = createRuntime();
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
-
- expect(engine.initialize).toHaveBeenCalled();
- expect(adapter.setVoiceMode).toHaveBeenCalledWith(true, "agent-1");
- expect(engine.startCapture).toHaveBeenCalled();
- expect(runtime.getSnapshot()).toMatchObject({
- phase: "listening",
- isVoiceMode: true,
- activeServerId: "server-1",
- activeAgentId: "agent-1",
- });
- });
-
- it("streams continuous PCM chunks and waits after receiving a transcript", async () => {
- const adapter = createSessionAdapter();
- const { runtime } = createRuntime();
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
- runtime.handleCapturePcm(new Uint8Array(32000));
- expect(runtime.getSnapshot().phase).toBe("listening");
-
- expect(adapter.sendVoiceAudioChunk).toHaveBeenLastCalledWith(
- expect.any(String),
- "audio/pcm;rate=16000;bits=16",
- );
-
- runtime.onTranscriptionResult("server-1", "hello");
- expect(runtime.getSnapshot().phase).toBe("waiting");
- });
-
- it("moves from listening to playing on the first assistant audio", async () => {
- const adapter = createSessionAdapter();
- const { runtime, engine: _engine } = createRuntime();
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
- runtime.onAssistantAudioStarted("server-1");
-
- expect(runtime.getSnapshot().phase).toBe("playing");
- expect(adapter.setAssistantAudioPlaying).toHaveBeenCalledWith(true);
- });
-
- it("starts playback on the first chunk before isLastChunk arrives", async () => {
- const adapter = createSessionAdapter();
- let resolvePlay!: (duration: number) => void;
- const engine = createAudioEngineMock();
- vi.mocked(engine.play).mockImplementation(
- () =>
- new Promise((resolve) => {
- resolvePlay = resolve;
- }),
- );
- const { runtime } = createRuntime({ engine });
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
- runtime.onTurnEvent("server-1", "agent-1", "turn_started");
- vi.mocked(engine.play).mockClear();
-
- runtime.handleAudioOutput(
- "server-1",
- createAudioPayload({
- id: "chunk-0",
- groupId: "group-1",
- chunkIndex: 0,
- isLastChunk: false,
- }),
- );
-
- await vi.waitFor(() => {
- expect(engine.play).toHaveBeenCalledTimes(1);
- expect(runtime.getSnapshot().phase).toBe("playing");
- expect(adapter.audioPlayed).not.toHaveBeenCalled();
- });
-
- runtime.handleAudioOutput(
- "server-1",
- createAudioPayload({
- id: "chunk-1",
- groupId: "group-1",
- chunkIndex: 1,
- isLastChunk: true,
- }),
- );
-
- expect(engine.play).toHaveBeenCalledTimes(1);
- resolvePlay(0.1);
-
- await vi.waitFor(() => {
- expect(adapter.audioPlayed).toHaveBeenCalledWith("chunk-0");
- expect(engine.play).toHaveBeenCalledTimes(2);
- });
- });
-
- it("acknowledges chunks only after they are consumed and finishes after the final chunk", async () => {
- const adapter = createSessionAdapter();
- const playResolvers: Array<(duration: number) => void> = [];
- const engine = createAudioEngineMock();
- vi.mocked(engine.play).mockImplementation(
- () =>
- new Promise((resolve) => {
- playResolvers.push(resolve);
- }),
- );
- const { runtime } = createRuntime({ engine });
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
- runtime.onTurnEvent("server-1", "agent-1", "turn_started");
- vi.mocked(engine.play).mockClear();
- playResolvers.length = 0;
-
- runtime.handleAudioOutput(
- "server-1",
- createAudioPayload({
- id: "chunk-0",
- groupId: "group-1",
- chunkIndex: 0,
- isLastChunk: false,
- }),
- );
- runtime.handleAudioOutput(
- "server-1",
- createAudioPayload({
- id: "chunk-1",
- groupId: "group-1",
- chunkIndex: 1,
- isLastChunk: true,
- }),
- );
-
- await vi.waitFor(() => {
- expect(engine.play).toHaveBeenCalledTimes(1);
- });
- expect(adapter.audioPlayed).not.toHaveBeenCalled();
-
- playResolvers.shift()!(0.1);
- await vi.waitFor(() => {
- expect(adapter.audioPlayed).toHaveBeenCalledWith("chunk-0");
- expect(engine.play).toHaveBeenCalledTimes(2);
- });
-
- playResolvers.shift()!(0.1);
- await vi.waitFor(() => {
- expect(adapter.audioPlayed).toHaveBeenCalledWith("chunk-1");
- expect(runtime.getSnapshot().phase).toBe("waiting");
- });
- });
-
- it("leaves playback phase unchanged after assistant playback while the turn is still active", async () => {
- const adapter = createSessionAdapter();
- const { runtime, engine } = createRuntime();
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
- runtime.onTurnEvent("server-1", "agent-1", "turn_started");
- runtime.onAssistantAudioStarted("server-1");
- runtime.onAssistantAudioFinished("server-1");
-
- expect(runtime.getSnapshot().phase).toBe("waiting");
- expect(engine.play).toHaveBeenCalled();
- });
-
- it("starts the thinking tone when an agent turn begins before playback", async () => {
- const adapter = createSessionAdapter();
- const { runtime, engine } = createRuntime();
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
- runtime.onTurnEvent("server-1", "agent-1", "turn_started");
-
- expect(runtime.getSnapshot().phase).toBe("waiting");
- expect(engine.play).toHaveBeenCalled();
- });
-
- it("does not restart the thinking tone from local detection jitter while waiting", async () => {
- const adapter = createSessionAdapter();
- let resolvePlay!: (duration: number) => void;
- const engine = createAudioEngineMock();
- vi.mocked(engine.play).mockImplementation(
- () =>
- new Promise((resolve) => {
- resolvePlay = resolve;
- }),
- );
- const { runtime } = createRuntime({ engine });
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
- runtime.onTurnEvent("server-1", "agent-1", "turn_started");
-
- expect(runtime.getSnapshot().phase).toBe("waiting");
- expect(engine.play).toHaveBeenCalledTimes(1);
- vi.mocked(engine.stop).mockClear();
- vi.mocked(engine.clearQueue).mockClear();
-
- runtime.handleCaptureVolume(REALTIME_VOICE_VAD_CONFIG.volumeThreshold + 0.05);
- runtime.handleCaptureVolume(0);
-
- expect(engine.stop).not.toHaveBeenCalled();
- expect(engine.clearQueue).not.toHaveBeenCalled();
- expect(engine.play).toHaveBeenCalledTimes(1);
-
- runtime.onServerSpeechStateChanged("server-1", true);
-
- expect(engine.stop).toHaveBeenCalledTimes(1);
- expect(engine.clearQueue).toHaveBeenCalledTimes(1);
-
- resolvePlay(0.1);
- });
-
- it("returns to listening after assistant playback once the turn is complete", async () => {
- const adapter = createSessionAdapter();
- const { runtime, engine } = createRuntime();
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
- runtime.onTurnEvent("server-1", "agent-1", "turn_started");
- runtime.onAssistantAudioStarted("server-1");
- runtime.onTurnEvent("server-1", "agent-1", "turn_completed");
- runtime.onAssistantAudioFinished("server-1");
-
- expect(runtime.getSnapshot().phase).toBe("listening");
- expect(engine.play).toHaveBeenCalled();
- });
-
- it("keeps local volume alone non-authoritative for playback interruption", async () => {
- const adapter = createSessionAdapter();
- const { runtime, engine } = createRuntime();
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
- runtime.onTurnEvent("server-1", "agent-1", "turn_started");
- runtime.onAssistantAudioStarted("server-1");
- vi.mocked(engine.stop).mockClear();
- vi.mocked(engine.clearQueue).mockClear();
-
- runtime.handleCaptureVolume(0.5);
- expect(runtime.getTelemetrySnapshot().isSpeaking).toBe(false);
- expect(adapter.abortRequest).not.toHaveBeenCalled();
- expect(runtime.getSnapshot().phase).toBe("playing");
- });
-
- it("keeps the meter white state driven by server speech detection", async () => {
- const adapter = createSessionAdapter();
- const { runtime } = createRuntime();
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
- runtime.handleCaptureVolume(0.7);
-
- expect(runtime.getTelemetrySnapshot()).toMatchObject({
- volume: expect.any(Number),
- isSpeaking: false,
- });
-
- runtime.onServerSpeechStateChanged("server-1", true);
- expect(runtime.getTelemetrySnapshot().isSpeaking).toBe(true);
-
- runtime.onServerSpeechStateChanged("server-1", false);
- expect(runtime.getTelemetrySnapshot().isSpeaking).toBe(false);
- });
-
- it("stops assistant playback immediately when server speech starts", async () => {
- const adapter = createSessionAdapter();
- const { runtime, engine } = createRuntime();
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
- runtime.onAssistantAudioStarted("server-1");
-
- runtime.onServerSpeechStateChanged("server-1", true);
-
- expect(engine.stop).toHaveBeenCalled();
- expect(engine.clearQueue).toHaveBeenCalled();
- expect(adapter.setAssistantAudioPlaying).toHaveBeenCalledWith(false);
- expect(runtime.getTelemetrySnapshot().isSpeaking).toBe(true);
- });
-
- it("drops queued voice chunks that arrive after server speech interrupts playback", async () => {
- const adapter = createSessionAdapter();
- const { runtime, engine } = createRuntime();
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
- runtime.onTurnEvent("server-1", "agent-1", "turn_started");
- vi.mocked(engine.play).mockClear();
-
- runtime.handleAudioOutput(
- "server-1",
- createAudioPayload({
- id: "chunk-0",
- groupId: "group-1",
- chunkIndex: 0,
- isLastChunk: false,
- }),
- );
- await vi.waitFor(() => {
- expect(engine.play).toHaveBeenCalledTimes(1);
- });
-
- runtime.onServerSpeechStateChanged("server-1", true);
- runtime.handleAudioOutput(
- "server-1",
- createAudioPayload({
- id: "chunk-1",
- groupId: "group-1",
- chunkIndex: 1,
- isLastChunk: true,
- }),
- );
-
- expect(engine.stop).toHaveBeenCalled();
- expect(engine.clearQueue).toHaveBeenCalled();
- expect(vi.mocked(adapter.audioPlayed).mock.calls.flat()).not.toContain("chunk-1");
- });
-
- it("authoritatively stops and suppresses later voice audio", async () => {
- const adapter = createSessionAdapter();
- const { runtime, engine } = createRuntime();
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
- await runtime.stopVoice();
-
- expect(adapter.setVoiceMode).toHaveBeenLastCalledWith(false);
- expect(engine.stopCapture).toHaveBeenCalled();
- expect(runtime.getSnapshot().phase).toBe("disabled");
- expect(runtime.shouldPlayVoiceAudio("server-1")).toBe(false);
- });
-
- it("rolls daemon voice mode back when capture fails after enabling it", async () => {
- const adapter = createSessionAdapter();
- const engine = createAudioEngineMock();
- vi.mocked(engine.startCapture).mockRejectedValue(new Error("audio focus unavailable"));
- const { runtime } = createRuntime({ engine });
- runtime.registerSession(adapter);
-
- await expect(runtime.startVoice("server-1", "agent-1")).rejects.toThrow(
- "audio focus unavailable",
- );
-
- expect(adapter.setVoiceMode).toHaveBeenNthCalledWith(1, true, "agent-1");
- expect(adapter.setVoiceMode).toHaveBeenNthCalledWith(2, false);
- expect(runtime.getSnapshot().phase).toBe("disabled");
- });
-
- it("returns an explicit not-ready error when the adapter is missing", async () => {
- const { runtime } = createRuntime();
- await expect(runtime.startVoice("server-1", "agent-1")).rejects.toThrow(
- "Voice runtime is not ready for host server-1",
- );
- });
-
- it("resyncs voice mode after connection recovers", async () => {
- const adapter = createSessionAdapter();
- const { runtime } = createRuntime();
- runtime.registerSession(adapter);
-
- await runtime.startVoice("server-1", "agent-1");
- vi.mocked(adapter.setVoiceMode).mockClear();
-
- runtime.updateSessionConnection("server-1", false);
- runtime.updateSessionConnection("server-1", true);
- await Promise.resolve();
-
- expect(adapter.setVoiceMode).toHaveBeenCalledWith(true, "agent-1");
- });
-
- it("does not emit when the snapshot is unchanged", async () => {
- const adapter = createSessionAdapter();
- const { runtime } = createRuntime();
- runtime.registerSession(adapter);
- await runtime.startVoice("server-1", "agent-1");
-
- const listener = vi.fn();
- const unsubscribe = runtime.subscribe(listener);
-
- runtime.handleCaptureVolume(0);
- runtime.handleCaptureVolume(0);
-
- expect(listener).not.toHaveBeenCalled();
- unsubscribe();
- });
-});
diff --git a/packages/app/src/voice/voice-runtime.ts b/packages/app/src/voice/voice-runtime.ts
deleted file mode 100644
index 6bb42499e4..0000000000
--- a/packages/app/src/voice/voice-runtime.ts
+++ /dev/null
@@ -1,960 +0,0 @@
-import { Buffer } from "buffer";
-import type { AgentStreamEventPayload, SessionOutboundMessage } from "@getpaseo/protocol/messages";
-import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
-import type { DaemonServerInfo } from "@/stores/session-store";
-import type { AudioEngine } from "@/voice/audio-engine-types";
-import {
- THINKING_TONE_NATIVE_PCM_BASE64,
- THINKING_TONE_NATIVE_PCM_DURATION_MS,
-} from "@/utils/thinking-tone.native-pcm";
-
-const PCM_MIME_TYPE = "audio/pcm;rate=16000;bits=16";
-const KEEP_AWAKE_TAG = "paseo:voice";
-const THINKING_TONE_REPEAT_GAP_MS = 350;
-const DISPLAY_VOLUME_PUBLISH_INTERVAL_MS = 120;
-const DISPLAY_VOLUME_CHANGE_EPSILON = 0.02;
-const DISPLAY_VOLUME_ATTACK = 0.35;
-const DISPLAY_VOLUME_RELEASE = 0.18;
-
-type TurnEventType = Extract<
- AgentStreamEventPayload["type"],
- "turn_started" | "turn_completed" | "turn_failed" | "turn_canceled"
->;
-
-export type VoiceRuntimePhase =
- | "disabled"
- | "starting"
- | "listening"
- | "submitting"
- | "waiting"
- | "playing"
- | "stopping";
-
-export interface VoiceRuntimeSnapshot {
- phase: VoiceRuntimePhase;
- isVoiceMode: boolean;
- isVoiceSwitching: boolean;
- isMuted: boolean;
- activeServerId: string | null;
- activeAgentId: string | null;
-}
-
-export interface VoiceRuntimeTelemetrySnapshot {
- volume: number;
- isSpeaking: boolean;
- segmentDuration: number;
-}
-
-export interface VoiceSessionAdapter {
- serverId: string;
- setVoiceMode(enabled: boolean, agentId?: string): Promise;
- sendVoiceAudioChunk(audioData: string, mimeType: string): Promise;
- audioPlayed(chunkId: string): Promise;
- abortRequest(): Promise;
- setAssistantAudioPlaying(isPlaying: boolean): void;
-}
-
-export interface VoiceRuntimeDeps {
- engine: AudioEngine;
- getServerInfo(serverId: string): DaemonServerInfo | null;
- activateKeepAwake(tag: string): Promise;
- deactivateKeepAwake(tag: string): Promise;
-}
-
-interface RuntimeSessionState {
- adapter: VoiceSessionAdapter;
- connected: boolean;
-}
-
-interface ContinuousVoiceUploader {
- reset(): void;
- pushPcmChunk(chunk: Uint8Array): void;
-}
-
-interface RuntimeState {
- snapshot: VoiceRuntimeSnapshot;
- telemetry: VoiceRuntimeTelemetrySnapshot;
- turnInProgress: boolean;
- serverSpeechDetected: boolean;
- transportReady: boolean;
- generation: number;
- segmentDurationTimer: ReturnType | null;
- lastDisplayVolumePublishMs: number;
- serverSpeechStartedAt: number | null;
-}
-
-type AudioOutputPayload = Extract["payload"];
-
-interface StreamingPlaybackChunk {
- id: string;
- chunkIndex: number;
- source: { arrayBuffer(): Promise; size: number; type: string };
-}
-
-interface StreamingPlaybackGroup {
- groupId: string;
- isVoiceMode: boolean;
- shouldPlay: boolean;
- chunks: Map;
- nextChunkToPlay: number;
- finalChunkIndex: number | null;
- started: boolean;
- ackedChunkIds: Set;
-}
-
-interface RuntimePlaybackState {
- groups: Map;
- orderedGroupIds: string[];
- activeGroupId: string | null;
- processing: boolean;
- generation: number;
-}
-
-interface CueState {
- active: boolean;
- token: number;
- timeout: ReturnType | null;
- playing: boolean;
-}
-
-const INITIAL_SNAPSHOT: VoiceRuntimeSnapshot = {
- phase: "disabled",
- isVoiceMode: false,
- isVoiceSwitching: false,
- isMuted: false,
- activeServerId: null,
- activeAgentId: null,
-};
-
-const INITIAL_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
- volume: 0,
- isSpeaking: false,
- segmentDuration: 0,
-};
-
-let nextVoiceRuntimeInstanceId = 1;
-
-function snapshotsEqual(left: VoiceRuntimeSnapshot, right: VoiceRuntimeSnapshot): boolean {
- return (
- left.phase === right.phase &&
- left.isVoiceMode === right.isVoiceMode &&
- left.isVoiceSwitching === right.isVoiceSwitching &&
- left.isMuted === right.isMuted &&
- left.activeServerId === right.activeServerId &&
- left.activeAgentId === right.activeAgentId
- );
-}
-
-function telemetryEqual(
- left: VoiceRuntimeTelemetrySnapshot,
- right: VoiceRuntimeTelemetrySnapshot,
-): boolean {
- return (
- left.volume === right.volume &&
- left.isSpeaking === right.isSpeaking &&
- left.segmentDuration === right.segmentDuration
- );
-}
-
-export interface VoiceRuntime {
- subscribe(listener: () => void): () => void;
- getSnapshot(): VoiceRuntimeSnapshot;
- subscribeTelemetry(listener: () => void): () => void;
- getTelemetrySnapshot(): VoiceRuntimeTelemetrySnapshot;
- registerSession(adapter: VoiceSessionAdapter): () => void;
- updateSessionConnection(serverId: string, connected: boolean): void;
- handleCapturePcm(chunk: Uint8Array): void;
- handleCaptureVolume(level: number): void;
- handleAudioOutput(serverId: string, payload: AudioOutputPayload): void;
- startVoice(serverId: string, agentId: string): Promise;
- stopVoice(): Promise;
- destroy(): Promise;
- toggleMute(): void;
- isVoiceModeForAgent(serverId: string, agentId: string): boolean;
- shouldPlayVoiceAudio(serverId: string): boolean;
- onAssistantAudioStarted(serverId: string): void;
- onAssistantAudioFinished(serverId: string): void;
- onTranscriptionResult(serverId: string, text: string): void;
- onServerSpeechStateChanged(serverId: string, isSpeaking: boolean): void;
- onTurnEvent(serverId: string, agentId: string, eventType: TurnEventType): void;
-}
-
-export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
- const instanceId = nextVoiceRuntimeInstanceId++;
- const listeners = new Set<() => void>();
- const telemetryListeners = new Set<() => void>();
- const sessions = new Map();
- const state: RuntimeState = {
- snapshot: INITIAL_SNAPSHOT,
- telemetry: INITIAL_TELEMETRY,
- turnInProgress: false,
- serverSpeechDetected: false,
- transportReady: false,
- generation: 0,
- segmentDurationTimer: null,
- lastDisplayVolumePublishMs: 0,
- serverSpeechStartedAt: null,
- };
- const playback: RuntimePlaybackState = {
- groups: new Map(),
- orderedGroupIds: [],
- activeGroupId: null,
- processing: false,
- generation: 0,
- };
- const cue: CueState = {
- active: false,
- token: 0,
- timeout: null,
- playing: false,
- };
- const cuePcm16 = Uint8Array.from(Buffer.from(THINKING_TONE_NATIVE_PCM_BASE64, "base64"));
- const cueSource = {
- size: cuePcm16.byteLength,
- type: "audio/pcm;rate=16000;bits=16",
- async arrayBuffer() {
- return cuePcm16.buffer.slice(cuePcm16.byteOffset, cuePcm16.byteOffset + cuePcm16.byteLength);
- },
- };
- function emit(): void {
- for (const listener of listeners) {
- listener();
- }
- }
-
- function emitTelemetry(): void {
- for (const listener of telemetryListeners) {
- listener();
- }
- }
-
- function patchSnapshot(
- patch:
- | Partial
- | ((previous: VoiceRuntimeSnapshot) => VoiceRuntimeSnapshot),
- ): void {
- const next =
- typeof patch === "function" ? patch(state.snapshot) : { ...state.snapshot, ...patch };
- if (snapshotsEqual(next, state.snapshot)) {
- return;
- }
- state.snapshot = next;
- emit();
- }
-
- function patchTelemetry(
- patch:
- | Partial
- | ((previous: VoiceRuntimeTelemetrySnapshot) => VoiceRuntimeTelemetrySnapshot),
- ): void {
- const next =
- typeof patch === "function" ? patch(state.telemetry) : { ...state.telemetry, ...patch };
- if (telemetryEqual(next, state.telemetry)) {
- return;
- }
- state.telemetry = next;
- emitTelemetry();
- }
-
- function getActiveSession(): RuntimeSessionState | null {
- if (!state.snapshot.activeServerId) {
- return null;
- }
- return sessions.get(state.snapshot.activeServerId) ?? null;
- }
-
- function decodeAudioChunk(base64: string): Uint8Array {
- return Buffer.from(base64, "base64");
- }
-
- function toPlaybackSource(
- bytes: Uint8Array,
- format: string,
- ): { arrayBuffer(): Promise; size: number; type: string } {
- let mimeType: string;
- if (format === "pcm") mimeType = "audio/pcm;rate=24000;bits=16";
- else if (format === "mp3") mimeType = "audio/mpeg";
- else mimeType = `audio/${format}`;
-
- return {
- size: bytes.byteLength,
- type: mimeType,
- async arrayBuffer() {
- return Uint8Array.from(bytes).buffer;
- },
- };
- }
-
- function resetPlaybackState(): void {
- playback.generation += 1;
- playback.groups.clear();
- playback.orderedGroupIds = [];
- playback.activeGroupId = null;
- playback.processing = false;
- }
-
- function activateNextPlaybackGroup(): void {
- while (playback.orderedGroupIds.length > 0) {
- const groupId = playback.orderedGroupIds[0];
- if (playback.groups.has(groupId)) {
- playback.activeGroupId = groupId;
- return;
- }
- playback.orderedGroupIds.shift();
- }
- playback.activeGroupId = null;
- }
-
- function retireFinishedGroup(
- group: { groupId: string; started: boolean; isVoiceMode: boolean },
- serverId: string,
- ): void {
- playback.groups.delete(group.groupId);
- if (playback.orderedGroupIds[0] === group.groupId) {
- playback.orderedGroupIds.shift();
- } else {
- playback.orderedGroupIds = playback.orderedGroupIds.filter(
- (value) => value !== group.groupId,
- );
- }
- if (group.started && group.isVoiceMode) {
- api.onAssistantAudioFinished(serverId);
- }
- }
-
- async function acknowledgeChunk(chunkId: string): Promise {
- const activeSession = getActiveSession();
- if (!activeSession) {
- return;
- }
- await activeSession.adapter.audioPlayed(chunkId);
- }
-
- async function processPlaybackQueue(serverId: string): Promise {
- if (playback.processing) {
- return;
- }
-
- playback.processing = true;
- const generation = playback.generation;
- try {
- while (playback.activeGroupId) {
- if (generation !== playback.generation) {
- return;
- }
-
- const group = playback.groups.get(playback.activeGroupId);
- if (!group) {
- activateNextPlaybackGroup();
- continue;
- }
-
- const nextChunk = group.chunks.get(group.nextChunkToPlay);
- if (!nextChunk) {
- const groupIsFinished =
- group.finalChunkIndex !== null && group.nextChunkToPlay > group.finalChunkIndex;
- if (!groupIsFinished) {
- return;
- }
- retireFinishedGroup(group, serverId);
- activateNextPlaybackGroup();
- continue;
- }
-
- group.chunks.delete(group.nextChunkToPlay);
-
- if (group.shouldPlay && !group.started && group.isVoiceMode) {
- group.started = true;
- api.onAssistantAudioStarted(serverId);
- }
-
- try {
- if (group.shouldPlay) {
- await deps.engine.play(nextChunk.source);
- }
- } catch (error) {
- if (generation !== playback.generation) {
- return;
- }
- console.error(`[VoiceRuntime] play error chunk=${group.nextChunkToPlay}:`, error);
- }
-
- if (generation !== playback.generation) {
- return;
- }
-
- if (!group.ackedChunkIds.has(nextChunk.id)) {
- group.ackedChunkIds.add(nextChunk.id);
- void acknowledgeChunk(nextChunk.id).catch((error) => {
- console.warn("[VoiceRuntime] Failed to confirm audio playback:", error);
- });
- }
-
- group.nextChunkToPlay += 1;
- }
- } finally {
- if (generation === playback.generation) {
- playback.processing = false;
- }
- }
- }
-
- function clearSegmentDurationTimer(): void {
- if (state.segmentDurationTimer) {
- clearInterval(state.segmentDurationTimer);
- state.segmentDurationTimer = null;
- }
- }
-
- function reconcileSegmentDurationTimer(): void {
- if (!state.serverSpeechDetected) {
- clearSegmentDurationTimer();
- patchTelemetry((prev) => ({ ...prev, segmentDuration: 0 }));
- return;
- }
-
- if (state.segmentDurationTimer) {
- return;
- }
-
- state.segmentDurationTimer = setInterval(() => {
- const startedAt = state.serverSpeechStartedAt;
- patchTelemetry((prev) => ({
- ...prev,
- segmentDuration: startedAt ? Date.now() - startedAt : 0,
- }));
- }, 100);
- }
-
- function canPlayCue(): boolean {
- return (
- state.snapshot.isVoiceMode &&
- state.snapshot.phase === "waiting" &&
- !state.telemetry.isSpeaking
- );
- }
-
- function stopCue(): void {
- const hadActive = cue.active || cue.timeout !== null || cue.playing;
- cue.active = false;
- cue.token += 1;
- if (cue.timeout) {
- clearTimeout(cue.timeout);
- cue.timeout = null;
- }
- cue.playing = false;
- if (hadActive) {
- deps.engine.stop();
- deps.engine.clearQueue();
- }
- }
-
- function resetCaptureTelemetry(): void {
- clearSegmentDurationTimer();
- state.serverSpeechStartedAt = null;
- patchTelemetry({ ...INITIAL_TELEMETRY });
- }
-
- function reconcileCue(): void {
- if (!canPlayCue()) {
- stopCue();
- return;
- }
- if (cue.active) {
- return;
- }
- cue.active = true;
- cue.token += 1;
- const token = cue.token;
-
- const playNext = () => {
- if (!cue.active || cue.token !== token) {
- return;
- }
- cue.playing = true;
- void deps.engine
- .play(cueSource)
- .catch((error) => {
- if (cue.token !== token) {
- return;
- }
- console.warn(`[VoiceRuntime#${instanceId}] Cue playback failed:`, error);
- })
- .finally(() => {
- cue.playing = false;
- if (!cue.active || cue.token !== token) {
- return;
- }
- cue.timeout = setTimeout(
- playNext,
- THINKING_TONE_NATIVE_PCM_DURATION_MS + THINKING_TONE_REPEAT_GAP_MS,
- );
- });
- };
-
- playNext();
- }
-
- const uploader: ContinuousVoiceUploader = {
- reset() {},
- pushPcmChunk(chunk) {
- const activeSession = getActiveSession();
- if (
- !activeSession ||
- !state.transportReady ||
- !state.snapshot.isVoiceMode ||
- chunk.byteLength === 0
- ) {
- return;
- }
-
- const base64 = Buffer.from(chunk).toString("base64");
-
- void activeSession.adapter.sendVoiceAudioChunk(base64, PCM_MIME_TYPE).catch((error) => {
- console.error(`[VoiceRuntime#${instanceId}] Failed to send audio chunk:`, error);
- });
- },
- };
-
- function resetToDisabledState(): void {
- state.transportReady = false;
- state.turnInProgress = false;
- state.serverSpeechDetected = false;
- state.lastDisplayVolumePublishMs = 0;
- uploader.reset();
- resetCaptureTelemetry();
- patchSnapshot({ ...INITIAL_SNAPSHOT });
- }
-
- function publishDisplayVolume(level: number, nowMs: number): void {
- const previousVolume = state.telemetry.volume;
- const smoothing = level >= previousVolume ? DISPLAY_VOLUME_ATTACK : DISPLAY_VOLUME_RELEASE;
- const nextVolume = Math.max(
- 0,
- Math.min(1, previousVolume + (level - previousVolume) * smoothing),
- );
- const enoughTimeElapsed =
- nowMs - state.lastDisplayVolumePublishMs >= DISPLAY_VOLUME_PUBLISH_INTERVAL_MS;
- const enoughChange = Math.abs(nextVolume - previousVolume) >= DISPLAY_VOLUME_CHANGE_EPSILON;
-
- if (!enoughTimeElapsed && !enoughChange) {
- return;
- }
-
- state.lastDisplayVolumePublishMs = nowMs;
- patchTelemetry((prev) => ({
- ...prev,
- volume: Number(nextVolume.toFixed(3)),
- }));
- }
-
- async function performLocalStop(): Promise {
- stopCue();
- uploader.reset();
- resetPlaybackState();
- deps.engine.stop();
- deps.engine.clearQueue();
- await deps.engine.stopCapture().catch(() => undefined);
- await deps.deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
- getActiveSession()?.adapter.setAssistantAudioPlaying(false);
- resetToDisabledState();
- }
-
- async function resyncVoiceMode(serverId: string): Promise {
- if (
- !state.snapshot.isVoiceMode ||
- state.snapshot.activeServerId !== serverId ||
- !state.snapshot.activeAgentId
- ) {
- return;
- }
-
- const activeSession = getActiveSession();
- if (!activeSession || !activeSession.connected) {
- return;
- }
-
- patchSnapshot((prev) => ({ ...prev, isVoiceSwitching: true }));
- try {
- await activeSession.adapter.setVoiceMode(true, state.snapshot.activeAgentId);
- state.transportReady = true;
- } finally {
- patchSnapshot((prev) => ({ ...prev, isVoiceSwitching: false }));
- }
- }
-
- const api: VoiceRuntime = {
- subscribe(listener) {
- listeners.add(listener);
- return () => {
- listeners.delete(listener);
- };
- },
-
- getSnapshot() {
- return state.snapshot;
- },
-
- subscribeTelemetry(listener) {
- telemetryListeners.add(listener);
- return () => {
- telemetryListeners.delete(listener);
- };
- },
-
- getTelemetrySnapshot() {
- return state.telemetry;
- },
-
- registerSession(adapter) {
- sessions.set(adapter.serverId, {
- adapter,
- connected: true,
- });
-
- return () => {
- const activeServerId = state.snapshot.activeServerId;
- sessions.delete(adapter.serverId);
- if (activeServerId === adapter.serverId) {
- void performLocalStop();
- }
- };
- },
-
- updateSessionConnection(serverId, connected) {
- const session = sessions.get(serverId);
- if (!session) {
- return;
- }
- session.connected = connected;
- if (state.snapshot.activeServerId !== serverId) {
- return;
- }
- if (!connected) {
- state.transportReady = false;
- return;
- }
- void resyncVoiceMode(serverId);
- },
-
- handleCapturePcm(chunk) {
- if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
- return;
- }
- uploader.pushPcmChunk(chunk);
- },
-
- handleCaptureVolume(level) {
- const nowMs = Date.now();
- const displayLevel = state.snapshot.isMuted ? 0 : level;
- publishDisplayVolume(displayLevel, nowMs);
- if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
- patchTelemetry((prev) => ({
- ...prev,
- isSpeaking: false,
- segmentDuration: 0,
- }));
- return;
- }
-
- patchTelemetry((prev) => ({
- ...prev,
- isSpeaking: state.serverSpeechDetected,
- }));
- reconcileSegmentDurationTimer();
- reconcileCue();
- },
-
- handleAudioOutput(serverId, payload) {
- if (
- serverId !== state.snapshot.activeServerId ||
- !state.snapshot.isVoiceMode ||
- !payload.isVoiceMode
- ) {
- return;
- }
-
- const groupId = payload.groupId ?? payload.id;
- const chunkIndex = payload.chunkIndex ?? 0;
- const decoded = decodeAudioChunk(payload.audio);
-
- let group = playback.groups.get(groupId);
- if (!group) {
- group = {
- groupId,
- isVoiceMode: payload.isVoiceMode,
- shouldPlay: api.shouldPlayVoiceAudio(serverId),
- chunks: new Map(),
- nextChunkToPlay: 0,
- finalChunkIndex: null,
- started: false,
- ackedChunkIds: new Set(),
- };
- playback.groups.set(groupId, group);
- playback.orderedGroupIds.push(groupId);
- if (!playback.activeGroupId) {
- playback.activeGroupId = groupId;
- }
- }
-
- group.chunks.set(chunkIndex, {
- id: payload.id,
- chunkIndex,
- source: toPlaybackSource(decoded, payload.format),
- });
- if (payload.isLastChunk) {
- group.finalChunkIndex = chunkIndex;
- }
-
- void processPlaybackQueue(serverId);
- },
-
- async startVoice(serverId, agentId) {
- const session = sessions.get(serverId);
- if (!session) {
- throw new Error(`Voice runtime is not ready for host ${serverId}`);
- }
- if (!session.connected) {
- throw new Error(`Host ${serverId} is not connected`);
- }
-
- const serverInfo = deps.getServerInfo(serverId);
- const unavailableMessage = resolveVoiceUnavailableMessage({
- serverInfo,
- mode: "voice",
- });
- if (unavailableMessage) {
- throw new Error(unavailableMessage);
- }
-
- const previousServerId = state.snapshot.activeServerId;
- const previousAgentId = state.snapshot.activeAgentId;
- const generation = state.generation + 1;
- let enabledCurrentVoiceMode = false;
- state.generation = generation;
- state.transportReady = false;
- patchSnapshot((prev) => ({
- ...prev,
- isVoiceSwitching: true,
- phase: "starting",
- activeServerId: serverId,
- activeAgentId: agentId,
- }));
-
- try {
- if (
- state.snapshot.isVoiceMode &&
- previousServerId &&
- (previousServerId !== serverId || previousAgentId !== agentId)
- ) {
- const previousSession = sessions.get(previousServerId);
- if (previousSession) {
- previousSession.adapter.setAssistantAudioPlaying(false);
- await previousSession.adapter.setVoiceMode(false);
- }
- }
-
- await deps.activateKeepAwake(KEEP_AWAKE_TAG).catch((error) => {
- console.warn("[VoiceRuntime] Failed to activate keep-awake:", error);
- });
-
- await deps.engine.initialize();
- await session.adapter.setVoiceMode(true, agentId);
- enabledCurrentVoiceMode = true;
- await deps.engine.startCapture();
- if (state.generation !== generation) {
- return;
- }
-
- state.transportReady = true;
- state.turnInProgress = false;
- uploader.reset();
- resetCaptureTelemetry();
- patchSnapshot((prev) => ({
- ...prev,
- isVoiceMode: true,
- isVoiceSwitching: false,
- phase: "listening",
- isMuted: deps.engine.isMuted(),
- }));
- } catch (error) {
- if (enabledCurrentVoiceMode) {
- await session.adapter.setVoiceMode(false).catch(() => undefined);
- }
- await performLocalStop();
- throw error;
- }
- },
-
- async stopVoice() {
- const activeSession = getActiveSession();
- const generation = state.generation + 1;
- state.generation = generation;
- patchSnapshot((prev) => ({
- ...prev,
- isVoiceSwitching: true,
- phase: "stopping",
- }));
-
- try {
- stopCue();
- uploader.reset();
- state.transportReady = false;
- resetPlaybackState();
- deps.engine.stop();
- deps.engine.clearQueue();
- activeSession?.adapter.setAssistantAudioPlaying(false);
- if (activeSession) {
- await activeSession.adapter.setVoiceMode(false);
- }
- await deps.engine.stopCapture();
- await deps.deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
- } finally {
- if (state.generation === generation) {
- resetToDisabledState();
- }
- }
- },
-
- async destroy() {
- await this.stopVoice().catch(() => undefined);
- await deps.engine.destroy();
- listeners.clear();
- telemetryListeners.clear();
- sessions.clear();
- },
-
- toggleMute() {
- const nextMuted = deps.engine.toggleMute();
- if (nextMuted) {
- uploader.reset();
- resetCaptureTelemetry();
- patchSnapshot((prev) => ({
- ...prev,
- isMuted: true,
- }));
- reconcileCue();
- return;
- }
-
- patchSnapshot((prev) => ({ ...prev, isMuted: false }));
- },
-
- isVoiceModeForAgent(serverId, agentId) {
- return (
- state.snapshot.isVoiceMode &&
- state.snapshot.activeServerId === serverId &&
- state.snapshot.activeAgentId === agentId
- );
- },
-
- shouldPlayVoiceAudio(serverId) {
- return (
- state.snapshot.isVoiceMode &&
- state.snapshot.activeServerId === serverId &&
- state.snapshot.phase !== "stopping" &&
- state.snapshot.phase !== "disabled"
- );
- },
-
- onAssistantAudioStarted(serverId) {
- if (!state.snapshot.isVoiceMode || state.snapshot.activeServerId !== serverId) {
- return;
- }
- stopCue();
- getActiveSession()?.adapter.setAssistantAudioPlaying(true);
- patchSnapshot((prev) => ({ ...prev, phase: "playing" }));
- },
-
- onAssistantAudioFinished(serverId) {
- if (state.snapshot.activeServerId !== serverId) {
- return;
- }
-
- getActiveSession()?.adapter.setAssistantAudioPlaying(false);
- if (!state.snapshot.isVoiceMode) {
- return;
- }
-
- if (state.turnInProgress) {
- patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
- reconcileCue();
- return;
- }
-
- patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
- reconcileCue();
- },
-
- onTranscriptionResult(serverId, text) {
- if (serverId !== state.snapshot.activeServerId || !state.snapshot.isVoiceMode) {
- return;
- }
-
- if (text.trim()) {
- state.turnInProgress = true;
- patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
- reconcileCue();
- return;
- }
-
- state.turnInProgress = false;
- patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
- stopCue();
- },
-
- onServerSpeechStateChanged(serverId, isSpeaking) {
- if (serverId !== state.snapshot.activeServerId || !state.snapshot.isVoiceMode) {
- return;
- }
-
- state.serverSpeechDetected = isSpeaking;
- state.serverSpeechStartedAt = isSpeaking ? (state.serverSpeechStartedAt ?? Date.now()) : null;
- if (isSpeaking) {
- const shouldInterruptPlayback =
- state.snapshot.phase === "playing" || playback.groups.size > 0;
- const hadCue = cue.active || cue.timeout !== null || cue.playing;
- resetPlaybackState();
- stopCue();
- if (shouldInterruptPlayback && !hadCue) {
- deps.engine.stop();
- deps.engine.clearQueue();
- }
- getActiveSession()?.adapter.setAssistantAudioPlaying(false);
- }
- patchTelemetry((prev) => ({
- ...prev,
- isSpeaking,
- }));
- reconcileSegmentDurationTimer();
- reconcileCue();
- },
-
- onTurnEvent(serverId, agentId, eventType) {
- if (
- !state.snapshot.isVoiceMode ||
- state.snapshot.activeServerId !== serverId ||
- state.snapshot.activeAgentId !== agentId
- ) {
- return;
- }
-
- if (eventType === "turn_started") {
- state.turnInProgress = true;
- if (state.snapshot.phase !== "playing") {
- patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
- reconcileCue();
- }
- return;
- }
-
- state.turnInProgress = false;
- if (state.snapshot.phase !== "playing") {
- patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
- }
- stopCue();
- },
- };
-
- return api;
-}
diff --git a/packages/client/src/daemon-client.ts b/packages/client/src/daemon-client.ts
index 4b3e4d7e3c..b918c9dc1e 100644
--- a/packages/client/src/daemon-client.ts
+++ b/packages/client/src/daemon-client.ts
@@ -111,6 +111,10 @@ import type {
AgentSkillSelection,
AgentSkillsStatus,
AgentSkillsSaveResult,
+ VoiceAppContext,
+ VoiceCallClientTransportMessage,
+ VoiceCallTransport,
+ VoiceCallTransportOffer,
} from "@getpaseo/protocol/messages";
import type {
AgentPermissionRequest,
@@ -3395,6 +3399,84 @@ export class DaemonClient {
// Audio / Voice
// ============================================================================
+ async startVoiceCall(
+ context: VoiceAppContext,
+ transports: VoiceCallTransportOffer[],
+ ): Promise<{ callId: string; transport: VoiceCallTransport }> {
+ const requestId = this.createRequestId();
+ const message = SessionInboundMessageSchema.parse({
+ type: "voice.call.start.request",
+ requestId,
+ context,
+ transports,
+ });
+ const response = await this.sendRequest({
+ requestId,
+ message,
+ select: (candidate) => {
+ if (candidate.type !== "voice.call.start.response") return null;
+ if (candidate.payload.requestId !== requestId) return null;
+ return candidate.payload;
+ },
+ });
+ if (!response.accepted || !response.callId || !response.transport) {
+ throw new Error(response.error ?? "Failed to start voice chat");
+ }
+ return { callId: response.callId, transport: response.transport };
+ }
+
+ async stopVoiceCall(callId: string): Promise {
+ const requestId = this.createRequestId();
+ const message = SessionInboundMessageSchema.parse({
+ type: "voice.call.stop.request",
+ requestId,
+ callId,
+ });
+ const response = await this.sendRequest({
+ requestId,
+ message,
+ select: (candidate) => {
+ if (candidate.type !== "voice.call.stop.response") return null;
+ if (candidate.payload.requestId !== requestId) return null;
+ return candidate.payload;
+ },
+ });
+ if (!response.accepted) {
+ throw new Error(response.error ?? "Failed to stop voice chat");
+ }
+ }
+
+ async updateVoiceCallContext(callId: string, context: VoiceAppContext): Promise {
+ const requestId = this.createRequestId();
+ const message = SessionInboundMessageSchema.parse({
+ type: "voice.call.context.update.request",
+ requestId,
+ callId,
+ context,
+ });
+ const response = await this.sendRequest({
+ requestId,
+ message,
+ select: (candidate) => {
+ if (candidate.type !== "voice.call.context.update.response") return null;
+ if (candidate.payload.requestId !== requestId) return null;
+ return candidate.payload;
+ },
+ });
+ if (!response.accepted) {
+ throw new Error(response.error ?? "Failed to update voice chat context");
+ }
+ }
+
+ sendVoiceCallTransportMessage(callId: string, data: unknown): void {
+ const message: VoiceCallClientTransportMessage = {
+ type: "voice.call.transport.client",
+ callId,
+ data,
+ };
+ this.sendSessionMessageStrict(message);
+ }
+
async setVoiceMode(enabled: boolean, agentId?: string): Promise {
const requestId = this.createRequestId();
const message = SessionInboundMessageSchema.parse({
diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts
index 6c9fd5f810..35c84a5cf7 100644
--- a/packages/protocol/src/messages.ts
+++ b/packages/protocol/src/messages.ts
@@ -60,6 +60,19 @@ import {
BrowserAutomationExecuteResponseSchema,
} from "./browser-automation/rpc-schemas.js";
import { BrowserAutomationHostCapabilitySchema } from "./browser-automation/capabilities.js";
+import {
+ VoiceCallClientTransportMessageSchema,
+ VoiceCallContextUpdateRequestSchema,
+ VoiceCallContextUpdateResponseSchema,
+ VoiceCallEventMessageSchema,
+ VoiceCallStartRequestSchema,
+ VoiceCallStartResponseSchema,
+ VoiceCallStateMessageSchema,
+ VoiceCallStopRequestSchema,
+ VoiceCallStopResponseSchema,
+ VoiceCallServerTransportMessageSchema,
+} from "./voice-chat.js";
+export * from "./voice-chat.js";
import {
PaseoConfigRawSchema,
PaseoLifecycleCommandRawSchema,
@@ -211,6 +224,18 @@ export const AgentSkillSelectionSchema = z.discriminatedUnion("mode", [
]);
export type AgentSkillSelection = z.infer;
+export const MutableManualVoiceOrchestratorConfigSchema = z
+ .object({
+ provider: z.string().nullable(),
+ model: z.string().nullable(),
+ modeId: z.string().nullable(),
+ thinkingOptionId: z.string().nullable(),
+ })
+ .strict();
+export type MutableManualVoiceOrchestratorConfig = z.infer<
+ typeof MutableManualVoiceOrchestratorConfigSchema
+>;
+
export const MutableDaemonConfigSchema = z
.object({
// COMPAT(relayConfig): added in v0.2.6, remove after 2027-01-31 when old daemons are unsupported.
@@ -246,6 +271,10 @@ export const MutableDaemonConfigSchema = z
terminalProfiles: z.array(TerminalProfileSchema).optional(),
agentProfiles: z.array(AgentProfileSchema).optional(),
skills: z.object({ selection: AgentSkillSelectionSchema.optional() }).strict().optional(),
+ manualVoice: z
+ .object({ orchestrator: MutableManualVoiceOrchestratorConfigSchema })
+ .strict()
+ .optional(),
pluginsEnabled: z.boolean().optional(),
plugins: z.record(PluginIdSchema, PluginSourceSchema).optional(),
})
@@ -266,6 +295,10 @@ export const MutableDaemonConfigPatchSchema = z
appendSystemPrompt: z.string().optional(),
terminalProfiles: z.array(TerminalProfileSchema).optional(),
agentProfiles: z.array(AgentProfileSchema).optional(),
+ manualVoice: z
+ .object({ orchestrator: MutableManualVoiceOrchestratorConfigSchema.partial() })
+ .strict()
+ .optional(),
pluginsEnabled: z.boolean().optional(),
plugins: z.record(PluginIdSchema, PluginSourceSchema).optional(),
})
@@ -3040,6 +3073,10 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
HubExecutionAgentValidateRequestSchema,
HubExecutionControlRequestSchema,
BrowserAutomationExecuteResponseSchema,
+ VoiceCallStartRequestSchema,
+ VoiceCallStopRequestSchema,
+ VoiceCallContextUpdateRequestSchema,
+ VoiceCallClientTransportMessageSchema,
VoiceAudioChunkMessageSchema,
AbortRequestMessageSchema,
AudioPlayedMessageSchema,
@@ -3399,6 +3436,8 @@ export const ServerInfoStatusPayloadSchema = z
// COMPAT(providersSnapshot): added in v0.1.48, remove gating when all clients use snapshot
features: z
.object({
+ // COMPAT(voiceChat): added in v0.7.0, remove gate after 2027-08-29.
+ voiceChat: z.boolean().optional(),
providersSnapshot: z.boolean().optional(),
// COMPAT(providersSnapshotCwd): added in v0.3.2, remove gate after 2027-02-10.
providersSnapshotCwd: z.boolean().optional(),
@@ -6402,6 +6441,12 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
HubExecutionAgentUpdateSchema,
HubExecutionAgentStreamSchema,
BrowserAutomationExecuteRequestSchema,
+ VoiceCallStartResponseSchema,
+ VoiceCallStopResponseSchema,
+ VoiceCallContextUpdateResponseSchema,
+ VoiceCallStateMessageSchema,
+ VoiceCallEventMessageSchema,
+ VoiceCallServerTransportMessageSchema,
PluginCatalogGetResponseSchema,
PluginListResponseSchema,
PluginLogsGetResponseSchema,
diff --git a/packages/protocol/src/voice-chat.test.ts b/packages/protocol/src/voice-chat.test.ts
new file mode 100644
index 0000000000..caa4214241
--- /dev/null
+++ b/packages/protocol/src/voice-chat.test.ts
@@ -0,0 +1,94 @@
+import { describe, expect, it } from "vitest";
+import { SessionInboundMessageSchema, SessionOutboundMessageSchema } from "./messages.js";
+import {
+ VoiceCallEventMessageSchema,
+ VoiceCallClientTransportMessageSchema,
+ VoiceCallServerTransportMessageSchema,
+ VoiceCallStartRequestSchema,
+ VoiceCallStartResponseSchema,
+ VoiceCallStateMessageSchema,
+} from "./voice-chat.js";
+
+describe("voice chat protocol", () => {
+ it("accepts a fresh call start through the session protocol", () => {
+ const message = {
+ type: "voice.call.start.request",
+ requestId: "request-1",
+ context: { workspaceId: "workspace-1", agentId: null },
+ transports: [{ kind: "webrtc", offerSdp: "offer" }],
+ };
+
+ expect(VoiceCallStartRequestSchema.parse(message)).toEqual(message);
+ expect(SessionInboundMessageSchema.parse(message)).toEqual(message);
+ });
+
+ it("negotiates transport independently of the selected provider", () => {
+ const message = {
+ type: "voice.call.start.response",
+ payload: {
+ requestId: "request-1",
+ accepted: true,
+ callId: "call-1",
+ transport: { kind: "webrtc", answerSdp: "answer" },
+ error: null,
+ },
+ };
+
+ expect(VoiceCallStartResponseSchema.parse(message)).toEqual(message);
+ expect(SessionOutboundMessageSchema.parse(message)).toEqual(message);
+ });
+
+ it("keeps server state limited to lifecycle and provider-owned input", () => {
+ const message = {
+ type: "voice.call.state",
+ payload: {
+ callId: "call-1",
+ lifecycle: "active",
+ input: "speech_detected",
+ error: null,
+ },
+ };
+
+ expect(VoiceCallStateMessageSchema.parse(message)).toEqual(message);
+ expect(SessionOutboundMessageSchema.parse(message)).toEqual(message);
+ });
+
+ it("carries provider-owned transport data through opaque call envelopes", () => {
+ const clientMessage = {
+ type: "voice.call.transport.client",
+ callId: "call-1",
+ data: { type: "candidate", value: "client-data" },
+ };
+ const serverMessage = {
+ type: "voice.call.transport.server",
+ payload: {
+ callId: "call-1",
+ data: { type: "candidate", value: "server-data" },
+ },
+ };
+
+ expect(VoiceCallClientTransportMessageSchema.parse(clientMessage)).toEqual(clientMessage);
+ expect(SessionInboundMessageSchema.parse(clientMessage)).toEqual(clientMessage);
+ expect(VoiceCallServerTransportMessageSchema.parse(serverMessage)).toEqual(serverMessage);
+ expect(SessionOutboundMessageSchema.parse(serverMessage)).toEqual(serverMessage);
+ });
+
+ it("accepts provider-neutral partial transcript events", () => {
+ const message = {
+ type: "voice.call.event",
+ payload: {
+ callId: "call-1",
+ event: {
+ type: "transcript",
+ id: "transcript-1",
+ speaker: "user",
+ text: "Start an agent",
+ final: false,
+ },
+ },
+ };
+
+ expect(VoiceCallEventMessageSchema.parse(message)).toEqual(message);
+ expect(SessionOutboundMessageSchema.parse(message)).toEqual(message);
+ });
+});
diff --git a/packages/protocol/src/voice-chat.ts b/packages/protocol/src/voice-chat.ts
new file mode 100644
index 0000000000..ef41a0598d
--- /dev/null
+++ b/packages/protocol/src/voice-chat.ts
@@ -0,0 +1,166 @@
+import { z } from "zod";
+
+export const VoiceAppContextSchema = z.object({
+ workspaceId: z.string().nullable(),
+ agentId: z.string().nullable(),
+});
+
+export const VoiceCallStartRequestSchema = z.object({
+ type: z.literal("voice.call.start.request"),
+ requestId: z.string(),
+ context: VoiceAppContextSchema,
+ // COMPAT(voiceCallTransportOffers): added in v0.3, optional so old clients still parse.
+ transports: z
+ .array(
+ z.discriminatedUnion("kind", [
+ z.object({ kind: z.literal("daemon-audio") }),
+ z.object({ kind: z.literal("webrtc"), offerSdp: z.string().min(1) }),
+ ]),
+ )
+ .optional(),
+});
+
+export const VoiceCallTransportSchema = z.discriminatedUnion("kind", [
+ z.object({ kind: z.literal("daemon-audio") }),
+ z.object({ kind: z.literal("webrtc"), answerSdp: z.string().min(1) }),
+]);
+
+export const VoiceCallStartResponseSchema = z.object({
+ type: z.literal("voice.call.start.response"),
+ payload: z.object({
+ requestId: z.string(),
+ accepted: z.boolean(),
+ callId: z.string().nullable(),
+ transport: VoiceCallTransportSchema.optional(),
+ error: z.string().nullable(),
+ }),
+});
+
+export const VoiceCallStopRequestSchema = z.object({
+ type: z.literal("voice.call.stop.request"),
+ requestId: z.string(),
+ callId: z.string(),
+});
+
+export const VoiceCallStopResponseSchema = z.object({
+ type: z.literal("voice.call.stop.response"),
+ payload: z.object({
+ requestId: z.string(),
+ callId: z.string(),
+ accepted: z.boolean(),
+ error: z.string().nullable(),
+ }),
+});
+
+export const VoiceCallContextUpdateRequestSchema = z.object({
+ type: z.literal("voice.call.context.update.request"),
+ requestId: z.string(),
+ callId: z.string(),
+ context: VoiceAppContextSchema,
+});
+
+export const VoiceCallContextUpdateResponseSchema = z.object({
+ type: z.literal("voice.call.context.update.response"),
+ payload: z.object({
+ requestId: z.string(),
+ callId: z.string(),
+ accepted: z.boolean(),
+ error: z.string().nullable(),
+ }),
+});
+
+export const VoiceCallClientTransportMessageSchema = z.object({
+ type: z.literal("voice.call.transport.client"),
+ callId: z.string(),
+ data: z.unknown(),
+});
+
+export const VoiceCallInboundMessageSchema = z.discriminatedUnion("type", [
+ VoiceCallStartRequestSchema,
+ VoiceCallStopRequestSchema,
+ VoiceCallContextUpdateRequestSchema,
+ VoiceCallClientTransportMessageSchema,
+]);
+
+export const VoiceCallStateSchema = z.object({
+ callId: z.string(),
+ lifecycle: z.enum(["active", "ended", "failed"]),
+ input: z.enum(["idle", "listening", "speech_detected"]),
+ error: z.string().nullable(),
+});
+
+export const VoiceCallStateMessageSchema = z.object({
+ type: z.literal("voice.call.state"),
+ payload: VoiceCallStateSchema,
+});
+
+export const VoiceCallTranscriptEventSchema = z.object({
+ type: z.literal("transcript"),
+ id: z.string(),
+ speaker: z.enum(["user", "assistant"]),
+ text: z.string(),
+ final: z.boolean(),
+});
+
+export const VoiceCallActivityEventSchema = z.object({
+ type: z.literal("activity"),
+ id: z.string(),
+ label: z.string(),
+ state: z.enum(["running", "completed", "failed"]),
+});
+
+export const VoiceCallNoticeEventSchema = z.object({
+ type: z.literal("notice"),
+ id: z.string(),
+ text: z.string(),
+});
+
+export const VoiceCallErrorEventSchema = z.object({
+ type: z.literal("error"),
+ id: z.string(),
+ message: z.string(),
+});
+
+export const VoiceCallEventSchema = z.discriminatedUnion("type", [
+ VoiceCallTranscriptEventSchema,
+ VoiceCallActivityEventSchema,
+ VoiceCallNoticeEventSchema,
+ VoiceCallErrorEventSchema,
+]);
+
+export const VoiceCallEventMessageSchema = z.object({
+ type: z.literal("voice.call.event"),
+ payload: z.object({
+ callId: z.string(),
+ event: VoiceCallEventSchema,
+ }),
+});
+
+export const VoiceCallServerTransportMessageSchema = z.object({
+ type: z.literal("voice.call.transport.server"),
+ payload: z.object({
+ callId: z.string(),
+ data: z.unknown(),
+ }),
+});
+
+export const VoiceCallOutboundMessageSchema = z.discriminatedUnion("type", [
+ VoiceCallStartResponseSchema,
+ VoiceCallStopResponseSchema,
+ VoiceCallContextUpdateResponseSchema,
+ VoiceCallStateMessageSchema,
+ VoiceCallEventMessageSchema,
+ VoiceCallServerTransportMessageSchema,
+]);
+
+export type VoiceAppContext = z.infer;
+export type VoiceCallInboundMessage = z.infer;
+export type VoiceCallOutboundMessage = z.infer;
+export type VoiceCallState = z.infer;
+export type VoiceCallEvent = z.infer;
+export type VoiceCallTransport = z.infer;
+export type VoiceCallTransportOffer = NonNullable<
+ z.infer["transports"]
+>[number];
+export type VoiceCallClientTransportMessage = z.infer;
+export type VoiceCallServerTransportMessage = z.infer;
diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts
index f2c8fc7d05..5600f4e063 100644
--- a/packages/server/src/server/agent/agent-manager.test.ts
+++ b/packages/server/src/server/agent/agent-manager.test.ts
@@ -3110,6 +3110,141 @@ test("keeps the global Paseo-tools gate outside provider policy and MCP injectio
rmSync(workdir, { recursive: true, force: true });
});
+test.each([true, false])(
+ "createAgent injects required Paseo tools with ordinary tools enabled=%s",
+ async (paseoToolsEnabled) => {
+ const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
+ const storage = new AgentStorage(join(workdir, "agents"), logger);
+
+ class CaptureClient extends TestAgentClient {
+ lastConfig: AgentSessionConfig | null = null;
+
+ override async createSession(config: AgentSessionConfig): Promise {
+ this.lastConfig = config;
+ return new McpCapableTestAgentSession(config);
+ }
+ }
+
+ const client = new CaptureClient();
+ const manager = new AgentManager({
+ clients: { codex: client },
+ registry: storage,
+ logger,
+ paseoToolsEnabled,
+ resolvePaseoToolPolicy: () => ({ enabled: false }),
+ requiredPaseoToolsBaseUrl: "http://127.0.0.1:6767/mcp/agents",
+ mcpAuthToken: "voice-capability-token",
+ idFactory: () => "00000000-0000-4000-8000-000000000117",
+ });
+
+ try {
+ const snapshot = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, {
+ workspaceId: undefined,
+ paseoTools: "required",
+ });
+
+ expect(client.lastConfig?.mcpServers?.paseo).toEqual({
+ type: "http",
+ url: `http://127.0.0.1:6767/mcp/agents?callerAgentId=${snapshot.id}`,
+ headers: { Authorization: "Bearer voice-capability-token" },
+ });
+ expect(snapshot.config.mcpServers).toBeUndefined();
+ expect(manager.getPaseoToolPolicy(snapshot.id)).toBeUndefined();
+ } finally {
+ rmSync(workdir, { recursive: true, force: true });
+ }
+ },
+);
+
+test("internal-agent capability owns tool context and preserves delegated children on cleanup", async () => {
+ const workdir = mkdtempSync(join(tmpdir(), "agent-manager-internal-agent-"));
+ const storage = new AgentStorage(join(workdir, "agents"), logger);
+ const parentId = "00000000-0000-4000-8000-000000000119";
+ const childId = "00000000-0000-4000-8000-000000000120";
+ const client = new McpCapableTestAgentClient();
+ const manager = new AgentManager({
+ clients: { codex: client },
+ registry: storage,
+ logger,
+ appendSystemPrompt: "Ordinary daemon instructions.",
+ requiredPaseoToolsBaseUrl: "http://127.0.0.1:6767/mcp/agents",
+ });
+ const tools = {
+ caller: { allowCustomCwd: false },
+ extension: { tools: [] },
+ };
+
+ try {
+ const internal = await manager.launchInternalAgentWithRequiredPaseoTools({
+ agentId: parentId,
+ config: {
+ provider: "codex",
+ cwd: workdir,
+ internal: true,
+ systemPrompt: "Authoritative internal instructions.",
+ },
+ workspaceId: undefined,
+ tools,
+ systemPromptBehavior: "authoritative",
+ });
+ await manager.createAgent({ provider: "codex", cwd: workdir }, childId, {
+ workspaceId: undefined,
+ labels: { [PARENT_AGENT_ID_LABEL]: parentId },
+ });
+
+ expect(manager.getInternalAgentPaseoTools(parentId)).toBe(tools);
+ expect(client.createdConfigs[0]?.systemPrompt).toBe("Authoritative internal instructions.");
+ expect(client.createdConfigs[0]).not.toHaveProperty("daemonAppendSystemPrompt");
+ await Promise.all([internal.dispose(), internal.dispose()]);
+
+ expect(manager.getInternalAgentPaseoTools(parentId)).toBeNull();
+ expect((await storage.get(parentId))?.archivedAt).toBeTruthy();
+ const child = await storage.get(childId);
+ expect(child?.archivedAt).toBeUndefined();
+ expect(child?.labels[PARENT_AGENT_ID_LABEL]).toBeUndefined();
+ } finally {
+ rmSync(workdir, { recursive: true, force: true });
+ }
+});
+
+test("createAgent rejects a voice orchestrator that cannot receive required Paseo tools", async () => {
+ const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
+ const storage = new AgentStorage(join(workdir, "agents"), logger);
+ let sessionClosed = false;
+
+ class RequiredToolsSession extends TestAgentSession {
+ override async close(): Promise {
+ sessionClosed = true;
+ }
+ }
+
+ class RequiredToolsClient extends TestAgentClient {
+ override async createSession(config: AgentSessionConfig): Promise {
+ return new RequiredToolsSession(config);
+ }
+ }
+
+ const manager = new AgentManager({
+ clients: { codex: new RequiredToolsClient() },
+ registry: storage,
+ logger,
+ requiredPaseoToolsBaseUrl: "http://127.0.0.1:6767/mcp/agents",
+ idFactory: () => "00000000-0000-4000-8000-000000000118",
+ });
+
+ try {
+ await expect(
+ manager.createAgent({ provider: "codex", cwd: workdir }, undefined, {
+ workspaceId: undefined,
+ paseoTools: "required",
+ }),
+ ).rejects.toThrow("cannot receive required Paseo tools");
+ expect(sessionClosed).toBe(true);
+ } finally {
+ rmSync(workdir, { recursive: true, force: true });
+ }
+});
+
test("resumeAgentFromPersistence replaces stored internal paseo MCP with current runtime URL", async () => {
const workdir = mkdtempSync(join(tmpdir(), "agent-manager-test-"));
const storagePath = join(workdir, "agents");
diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts
index edda3204e4..29d2a1bb7d 100644
--- a/packages/server/src/server/agent/agent-manager.ts
+++ b/packages/server/src/server/agent/agent-manager.ts
@@ -76,7 +76,11 @@ import { invokeRewindCapability, type RewindMode } from "./rewind/rewind.js";
import { isSystemInjectedEnvelope } from "./agent-prompt.js";
import { stripInternalPaseoMcpServer, withRuntimePaseoMcpServer } from "./runtime-mcp-config.js";
import { resolveCreateAgentTitles } from "./create-agent-title.js";
-import type { PaseoToolCatalogFactory } from "./tools/types.js";
+import type {
+ PaseoCallerContext,
+ PaseoToolCatalogFactory,
+ PaseoToolExtension,
+} from "./tools/types.js";
import { isPaseoToolPolicyEnabled } from "./paseo-tool-policy.js";
import {
ProviderSubagentStore,
@@ -283,6 +287,28 @@ export interface CreateAgentOptions {
// undefined is an explicit decision: the agent never appears in the sidebar.
workspaceId: string | undefined;
owner?: AgentOwner;
+ paseoTools?: "required";
+ systemPromptBehavior?: "append-daemon" | "authoritative";
+}
+
+export interface InternalAgentPaseoTools {
+ caller: PaseoCallerContext;
+ extension?: PaseoToolExtension;
+}
+
+export interface LaunchInternalAgentOptions {
+ agentId?: string;
+ config: AgentSessionConfig;
+ workspaceId: string | undefined;
+ labels?: Record;
+ initialTitle?: string | null;
+ tools: InternalAgentPaseoTools;
+ systemPromptBehavior: "authoritative";
+}
+
+export interface InternalAgentHandle {
+ agent: ManagedAgent;
+ dispose(): Promise;
}
export interface AgentManagerOptions {
@@ -295,6 +321,7 @@ export interface AgentManagerOptions {
durableTimelineStore?: AgentTimelineStore;
terminalManager?: TerminalManager | null;
mcpBaseUrl?: string;
+ requiredPaseoToolsBaseUrl?: string;
mcpAuthToken?: string;
paseoToolsEnabled?: boolean;
paseoToolCatalogFactory?: PaseoToolCatalogFactory;
@@ -309,6 +336,10 @@ export interface AgentManagerOptions {
logger: Logger;
}
+function resolveRequiredPaseoToolsBaseUrl(options: AgentManagerOptions): string | null {
+ return options.requiredPaseoToolsBaseUrl ?? null;
+}
+
export type ActiveTurnSteerDispatchResult =
| { status: "inactive" | "steered" }
| { status: "replaced"; iterator: AsyncGenerator };
@@ -707,6 +738,7 @@ export class AgentManager {
private readonly lifecycleMutationTails = new Map>();
private readonly agentStreamCoalescer: AgentStreamCoalescer;
private mcpBaseUrl: string | null;
+ private requiredPaseoToolsBaseUrl: string | null;
private readonly mcpAuthToken: string | null;
private paseoToolsEnabled = true;
private paseoToolCatalogFactory: PaseoToolCatalogFactory | null = null;
@@ -714,6 +746,7 @@ export class AgentManager {
private readonly resolvePaseoToolPolicy: (
provider: AgentProvider,
) => ProviderPaseoToolsPolicy | undefined;
+ private readonly internalAgentTools = new Map();
private appendSystemPrompt: string;
private onAgentAttention?: AgentAttentionCallback;
private onAgentArchived?: AgentArchivedCallback;
@@ -730,6 +763,7 @@ export class AgentManager {
this.onAgentAttention = options?.onAgentAttention;
this.onWorkspaceStateMayHaveChanged = options?.onWorkspaceStateMayHaveChanged;
this.mcpBaseUrl = options?.mcpBaseUrl ?? null;
+ this.requiredPaseoToolsBaseUrl = resolveRequiredPaseoToolsBaseUrl(options);
this.mcpAuthToken = options?.mcpAuthToken ?? null;
this.configurePaseoTools(options);
this.resolvePaseoToolPolicy = options.resolvePaseoToolPolicy ?? (() => undefined);
@@ -802,6 +836,10 @@ export class AgentManager {
this.mcpBaseUrl = url;
}
+ setRequiredPaseoToolsBaseUrl(url: string | null): void {
+ this.requiredPaseoToolsBaseUrl = url;
+ }
+
prepareForShutdown(): void {
this.acceptingAgentRegistrations = false;
}
@@ -818,6 +856,10 @@ export class AgentManager {
return this.paseoToolPolicies.get(agentId);
}
+ getInternalAgentPaseoTools(agentId: string): InternalAgentPaseoTools | null {
+ return this.internalAgentTools.get(agentId) ?? null;
+ }
+
/**
* Capability token the daemon's own MCP clients must present to the Agent MCP
* endpoint when a daemon password is configured. Read by the per-client
@@ -1182,6 +1224,49 @@ export class AgentManager {
return this.trackAgentRegistrationOperation(this.createAgentInternal(config, agentId, options));
}
+ async launchInternalAgentWithRequiredPaseoTools(
+ options: LaunchInternalAgentOptions,
+ ): Promise {
+ const agentId = validateAgentId(options.agentId ?? this.idFactory(), "launchInternalAgent");
+ if (this.internalAgentTools.has(agentId)) {
+ throw new Error(`Internal agent tool context already exists for ${agentId}`);
+ }
+ this.internalAgentTools.set(agentId, options.tools);
+ let agent: ManagedAgent;
+ try {
+ agent = await this.createAgent(options.config, agentId, {
+ workspaceId: options.workspaceId,
+ labels: options.labels,
+ initialTitle: options.initialTitle,
+ paseoTools: "required",
+ systemPromptBehavior: options.systemPromptBehavior,
+ });
+ } catch (error) {
+ this.internalAgentTools.delete(agentId);
+ throw error;
+ }
+
+ let disposed = false;
+ let disposal: Promise | null = null;
+ return {
+ agent,
+ dispose: async () => {
+ if (disposed) return;
+ disposal ??= (async () => {
+ await this.detachDelegatedChildren(agentId);
+ if (this.agents.has(agentId)) await this.archiveAgent(agentId);
+ this.internalAgentTools.delete(agentId);
+ disposed = true;
+ })();
+ try {
+ await disposal;
+ } finally {
+ if (!disposed) disposal = null;
+ }
+ },
+ };
+ }
+
private async createAgentInternal(
config: AgentSessionConfig,
agentId: string | undefined,
@@ -1194,6 +1279,8 @@ export class AgentManager {
config,
resolvedAgentId,
options?.env,
+ options.paseoTools,
+ options.systemPromptBehavior,
);
this.requireEnabledProvider(storedConfig.provider);
const client = await this.requireAvailableClient({
@@ -1206,10 +1293,17 @@ export class AgentManager {
storedConfig.cwd,
paseoToolPolicy,
options?.env,
+ options.paseoTools,
);
const providerLaunchConfig = this.resolveProviderLaunchConfig(launchConfig, launchContext);
const createOptions = this.buildCreateSessionOptions(options);
const session = await client.createSession(providerLaunchConfig, launchContext, createOptions);
+ await this.requirePaseoToolsSupport(
+ session,
+ providerLaunchConfig,
+ launchContext,
+ options.paseoTools,
+ );
await this.requireExternalMcpSupport(session, storedConfig);
return this.registerSession(session, storedConfig, resolvedAgentId, {
labels: options.labels,
@@ -1704,6 +1798,15 @@ export class AgentManager {
}
}
+ private async detachDelegatedChildren(parentAgentId: string): Promise {
+ if (!this.registry) return;
+ const records = await this.registry.list();
+ for (const record of records) {
+ if (record.archivedAt || record.labels?.[PARENT_AGENT_ID_LABEL] !== parentAgentId) continue;
+ await this.detachAgent(record.id);
+ }
+ }
+
private async markRecordArchived(record: StoredAgentRecord): Promise {
const registry = this.requireRegistry();
const archivedAt = new Date().toISOString();
@@ -3338,6 +3441,18 @@ export class AgentManager {
throw new Error(`Provider '${storedConfig.provider}' does not support MCP servers`);
}
+ private async requirePaseoToolsSupport(
+ session: AgentSession,
+ launchConfig: AgentSessionConfig,
+ launchContext: AgentLaunchContext,
+ requirement: CreateAgentOptions["paseoTools"],
+ ): Promise {
+ if (requirement !== "required" || launchContext.paseoTools) return;
+ if (launchConfig.mcpServers?.paseo && session.capabilities.supportsMcpServers === true) return;
+ await this.closeUnregisteredSession(session);
+ throw new Error(`Provider '${launchConfig.provider}' cannot receive required Paseo tools`);
+ }
+
private async initializeAgentTimelineForRegister(params: {
agentId: string;
now: Date;
@@ -4883,22 +4998,27 @@ export class AgentManager {
config: AgentSessionConfig,
agentId: string,
env?: Record,
+ paseoTools?: CreateAgentOptions["paseoTools"],
+ systemPromptBehavior: CreateAgentOptions["systemPromptBehavior"] = "append-daemon",
): Promise {
const storedConfig = await this.normalizeConfig(stripInternalPaseoMcpServer(config), { env });
- const paseoToolPolicy = this.paseoToolsEnabled
+ let paseoToolPolicy = this.paseoToolsEnabled
? this.resolvePaseoToolPolicy(storedConfig.provider)
: { enabled: false };
- const launchConfig = this.applyDaemonAppendSystemPrompt(
- withRuntimePaseoMcpServer({
- config: storedConfig,
- agentId,
- mcpBaseUrl:
- this.paseoToolsEnabled && isPaseoToolPolicyEnabled(paseoToolPolicy)
- ? this.mcpBaseUrl
- : null,
- mcpAuthToken: this.mcpAuthToken,
- }),
- );
+ if (paseoTools === "required") paseoToolPolicy = undefined;
+ const toolsEnabled = isPaseoToolPolicyEnabled(paseoToolPolicy);
+ let mcpBaseUrl = toolsEnabled ? this.mcpBaseUrl : null;
+ if (paseoTools === "required") mcpBaseUrl = this.requiredPaseoToolsBaseUrl;
+ const runtimeConfig = withRuntimePaseoMcpServer({
+ config: storedConfig,
+ agentId,
+ mcpBaseUrl,
+ mcpAuthToken: this.mcpAuthToken,
+ });
+ const launchConfig =
+ systemPromptBehavior === "authoritative"
+ ? runtimeConfig
+ : this.applyDaemonAppendSystemPrompt(runtimeConfig);
return { storedConfig, launchConfig, paseoToolPolicy };
}
@@ -4921,6 +5041,7 @@ export class AgentManager {
cwd: string,
paseoToolPolicy: ProviderPaseoToolsPolicy | undefined,
env?: Record,
+ paseoTools?: CreateAgentOptions["paseoTools"],
): Promise {
const context: AgentLaunchContext = {
agentId,
@@ -4931,7 +5052,7 @@ export class AgentManager {
},
};
if (
- this.paseoToolsEnabled &&
+ (this.paseoToolsEnabled || paseoTools === "required") &&
isPaseoToolPolicyEnabled(paseoToolPolicy) &&
client.capabilities.supportsNativePaseoTools &&
this.paseoToolCatalogFactory
diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts
index c5b97ce978..930a7b57f9 100644
--- a/packages/server/src/server/agent/mcp-server.test.ts
+++ b/packages/server/src/server/agent/mcp-server.test.ts
@@ -5175,51 +5175,40 @@ describe("provider MCP tools", () => {
});
});
-describe("speak MCP tool", () => {
+describe("agent tool extensions", () => {
const logger = createTestLogger();
- it("invokes registered speak handler for caller agent", async () => {
+ it("registers provider-owned tools for the caller agent", async () => {
const { agentManager, agentStorage } = createTestDeps();
- const speak = vi.fn().mockResolvedValue(undefined);
+ const handler = vi.fn().mockResolvedValue({ content: [], structuredContent: { ok: true } });
const server = await createAgentMcpServer({
agentManager,
agentStorage,
providerSnapshotManager: createOpenCodeManager().manager,
callerAgentId: "voice-agent-1",
- enableVoiceTools: true,
- resolveSpeakHandler: () => speak,
+ resolveToolExtension: () => ({
+ tools: [
+ {
+ name: "speak",
+ description: "Speak",
+ inputSchema: z.object({ text: z.string() }),
+ handler,
+ },
+ ],
+ }),
logger,
});
const tool = registeredTool(server, "speak");
expect(tool).toBeDefined();
await tool.handler({ text: "Hello from voice agent." });
- expect(speak).toHaveBeenCalledWith(
- expect.objectContaining({
- text: "Hello from voice agent.",
- callerAgentId: "voice-agent-1",
- }),
+ expect(handler).toHaveBeenCalledWith(
+ { text: "Hello from voice agent." },
+ expect.objectContaining({}),
);
});
- it("fails when no speak handler exists", async () => {
- const { agentManager, agentStorage } = createTestDeps();
- const server = await createAgentMcpServer({
- agentManager,
- agentStorage,
- providerSnapshotManager: createOpenCodeManager().manager,
- callerAgentId: "voice-agent-2",
- enableVoiceTools: true,
- resolveSpeakHandler: () => null,
- logger,
- });
- const tool = registeredTool(server, "speak");
- await expect(tool.handler({ text: "Hello." })).rejects.toThrow(
- "No speak handler registered for your session",
- );
- });
-
- it("does not register speak tool unless voice tools are enabled", async () => {
+ it("does not register provider tools without an extension", async () => {
const { agentManager, agentStorage } = createTestDeps();
const server = await createAgentMcpServer({
agentManager,
diff --git a/packages/server/src/server/agent/tools/paseo-tools.ts b/packages/server/src/server/agent/tools/paseo-tools.ts
index 2ddadaa1a5..6d0dd9867a 100644
--- a/packages/server/src/server/agent/tools/paseo-tools.ts
+++ b/packages/server/src/server/agent/tools/paseo-tools.ts
@@ -32,7 +32,6 @@ import {
type ArchiveDependencies,
} from "../../workspace-archive-service.js";
import { createAgentCommand, type CreateAgentFromMcpInput } from "../create-agent/create.js";
-import type { VoiceCallerContext, VoiceSpeakHandler } from "../../voice-types.js";
import type { FirstAgentContext } from "../../messages.js";
import { everyMsToFiveFieldCron } from "@getpaseo/protocol/schedule/cadence";
import { expandUserPath, isSameOrDescendantPath, resolvePathFromBase } from "../../path-utils.js";
@@ -86,10 +85,12 @@ import {
import { registerBrowserTools } from "../../browser-tools/tools.js";
import type { BrowserToolsBroker } from "../../browser-tools/broker.js";
import type {
+ PaseoCallerContext,
PaseoToolCatalog,
PaseoToolConfig,
PaseoToolDefinition,
PaseoToolExecutionContext,
+ PaseoToolExtension,
PaseoToolResult,
} from "./types.js";
import type { ProviderPaseoToolsPolicy } from "@getpaseo/protocol/provider-config";
@@ -138,14 +139,8 @@ export interface PaseoToolHostDependencies {
* Used for cwd/mode inheritance when agents spawn child agents.
*/
callerAgentId?: string;
- /**
- * Optional resolver for session-bound speak handlers.
- * Used by hidden voice agents to narrate through daemon-managed TTS.
- */
- resolveSpeakHandler?: (callerAgentId: string) => VoiceSpeakHandler | null;
- resolveCallerContext?: (callerAgentId: string) => VoiceCallerContext | null;
- enableVoiceTools?: boolean;
- voiceOnly?: boolean;
+ resolveCallerContext?: (callerAgentId: string) => PaseoCallerContext | null;
+ resolveToolExtension?: (callerAgentId: string) => PaseoToolExtension | null;
logger: Logger;
}
@@ -551,8 +546,8 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase
providerSnapshotManager,
daemonConfigStore,
callerAgentId,
- resolveSpeakHandler,
resolveCallerContext,
+ resolveToolExtension,
logger,
} = options;
const childLogger = logger.child({ module: "agent", component: "paseo-tool-catalog" });
@@ -609,6 +604,13 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase
},
});
+ if (callerAgentId) {
+ const extension = resolveToolExtension?.(callerAgentId) ?? null;
+ for (const tool of extension?.tools ?? []) {
+ tools.set(tool.name, tool);
+ }
+ }
+
const buildCronScheduleCadence = (input: {
cron: string | undefined;
timezone?: string;
@@ -1162,49 +1164,6 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase
type TopLevelCreateAgentArgs = z.infer;
type LegacyTopLevelCreateAgentArgs = z.infer;
- if (options.voiceOnly || options.enableVoiceTools || callerContext?.enableVoiceTools) {
- registerTool(
- "speak",
- {
- title: "Speak",
- description:
- "Speak text to the user via daemon-managed voice output. Blocks until playback completes.",
- inputSchema: {
- text: z
- .string()
- .trim()
- .min(1, "text is required")
- .max(4000, "text must be 4000 characters or fewer"),
- },
- outputSchema: {
- ok: z.boolean(),
- },
- },
- async (args, context) => {
- if (!callerAgentId) {
- throw new Error("speak is only available to agent-scoped tool sessions");
- }
- const handler = resolveSpeakHandler?.(callerAgentId) ?? null;
- if (!handler) {
- throw new Error(`No speak handler registered for your session '${callerAgentId}'`);
- }
- await handler({
- text: args.text,
- callerAgentId,
- signal: context?.signal,
- });
- return {
- content: [],
- structuredContent: ensureValidJson({ ok: true }),
- };
- },
- );
- }
-
- if (options.voiceOnly) {
- return toCatalog();
- }
-
if (options.browserToolsEnabled && options.browserToolsBroker) {
registerBrowserTools({
registerTool,
diff --git a/packages/server/src/server/agent/tools/types.ts b/packages/server/src/server/agent/tools/types.ts
index 66a4a23da2..3b9d5c6667 100644
--- a/packages/server/src/server/agent/tools/types.ts
+++ b/packages/server/src/server/agent/tools/types.ts
@@ -35,11 +35,19 @@ export interface PaseoToolCatalog {
): Promise;
}
+export interface PaseoCallerContext {
+ childAgentDefaultLabels?: Record;
+ lockedCwd?: string;
+ allowCustomCwd?: boolean;
+}
+
+export interface PaseoToolExtension {
+ tools: readonly PaseoToolDefinition[];
+}
+
export interface PaseoToolRuntimeContext {
callerAgentId?: string;
paseoToolPolicy?: ProviderPaseoToolsPolicy;
- enableVoiceTools?: boolean;
- voiceOnly?: boolean;
}
export type PaseoToolCatalogFactory = (
diff --git a/packages/server/src/server/auth.ts b/packages/server/src/server/auth.ts
index 619bb238e0..409784a293 100644
--- a/packages/server/src/server/auth.ts
+++ b/packages/server/src/server/auth.ts
@@ -148,15 +148,18 @@ export async function isAgentMcpRequestAuthorized(input: {
if (!input.password) {
return true;
}
+ if (isAgentMcpCapabilityToken(input)) return true;
const token = extractHttpBearerToken(input.authorizationHeader);
- if (input.capabilityToken !== null && token !== null) {
- // Constant-time compare; length-guard first because timingSafeEqual throws
- // on differing buffer lengths.
- const provided = Buffer.from(token);
- const expected = Buffer.from(input.capabilityToken);
- if (provided.length === expected.length && timingSafeEqual(provided, expected)) {
- return true;
- }
- }
return isBearerTokenValidAsync({ password: input.password, token });
}
+
+export function isAgentMcpCapabilityToken(input: {
+ capabilityToken: string | null;
+ authorizationHeader: string | undefined;
+}): boolean {
+ const token = extractHttpBearerToken(input.authorizationHeader);
+ if (input.capabilityToken === null || token === null) return false;
+ const provided = Buffer.from(token);
+ const expected = Buffer.from(input.capabilityToken);
+ return provided.length === expected.length && timingSafeEqual(provided, expected);
+}
diff --git a/packages/server/src/server/authorization/operation-permissions.ts b/packages/server/src/server/authorization/operation-permissions.ts
index 95995bc65e..52d253e3bf 100644
--- a/packages/server/src/server/authorization/operation-permissions.ts
+++ b/packages/server/src/server/authorization/operation-permissions.ts
@@ -181,6 +181,10 @@ const INBOUND_PERMISSION = {
unsubscribe_terminals_request: "workspace.read",
update_agent_request: "workspace.write",
validate_branch_request: "workspace.read",
+ "voice.call.context.update.request": "workspace.write",
+ "voice.call.start.request": "workspace.write",
+ "voice.call.stop.request": "workspace.write",
+ "voice.call.transport.client": "workspace.write",
voice_audio_chunk: "workspace.write",
wait_for_finish_request: "workspace.read",
"workspace.clear_attention.request": "workspace.write",
@@ -391,6 +395,12 @@ const OUTBOUND_PERMISSION = {
transcription_result: "workspace.write",
update_agent_response: "workspace.write",
validate_branch_response: "workspace.read",
+ "voice.call.context.update.response": "workspace.write",
+ "voice.call.event": "workspace.write",
+ "voice.call.start.response": "workspace.write",
+ "voice.call.state": "workspace.write",
+ "voice.call.stop.response": "workspace.write",
+ "voice.call.transport.server": "workspace.write",
voice_input_state: "workspace.write",
wait_for_finish_response: "workspace.read",
"workspace.clear_attention.response": "workspace.write",
diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts
index 1cfea1c99d..d0b566899c 100644
--- a/packages/server/src/server/bootstrap.ts
+++ b/packages/server/src/server/bootstrap.ts
@@ -128,6 +128,12 @@ import type { OpenAiSpeechProviderConfig } from "./speech/providers/openai/confi
import type { LocalSpeechProviderConfig } from "./speech/providers/local/config.js";
import type { RequestedSpeechProviders } from "./speech/speech-types.js";
import { createSpeechService } from "./speech/speech-runtime.js";
+import { createVoiceChat, type VoiceChat } from "./voice-chat/index.js";
+import { createManualVoiceProvider } from "./voice-chat/providers/manual/provider.js";
+import {
+ resolveManualVoiceConfig,
+ type ManualVoiceConfig,
+} from "./voice-chat/providers/manual/config.js";
import { AgentManager } from "./agent/agent-manager.js";
import { AgentStorage } from "./agent/agent-storage.js";
import { attachAgentStoragePersistence } from "./persistence-hooks.js";
@@ -179,6 +185,7 @@ import type {
PluginSource,
TerminalProfile,
} from "@getpaseo/protocol/messages";
+import { MutableManualVoiceOrchestratorConfigSchema } from "@getpaseo/protocol/messages";
import type {
AgentProviderRuntimeSettingsMap,
ProviderOverride,
@@ -201,6 +208,7 @@ import { terminateWithTreeKill } from "../utils/tree-kill.js";
import { isHostnameAllowed, type HostnamesConfig } from "./hostnames.js";
import {
createRequireBearerMiddleware,
+ isAgentMcpCapabilityToken,
isAgentMcpRequestAuthorized,
type DaemonAuthConfig,
} from "./auth.js";
@@ -430,9 +438,7 @@ export interface PaseoDaemonConfig {
auth?: DaemonAuthConfig;
openai?: PaseoOpenAIConfig;
speech?: PaseoSpeechConfig;
- voiceLlmProvider?: AgentProvider | null;
- voiceLlmProviderExplicit?: boolean;
- voiceLlmModel?: string | null;
+ manualVoice?: ManualVoiceConfig;
dictationFinalTimeoutMs?: number;
downloadTokenTtlMs?: number;
agentProviderSettings?: AgentProviderRuntimeSettingsMap;
@@ -523,8 +529,24 @@ function resolveExpressTrustProxySetting(config: PaseoDaemonConfig): true | stri
return config.trustedProxies ?? ["loopback"];
}
+function isAgentMcpEndpointAvailable(enabled: boolean, hasInternalCapability: boolean): boolean {
+ return enabled || hasInternalCapability;
+}
+
+function resolveDaemonManualVoiceConfig(config: PaseoDaemonConfig): ManualVoiceConfig {
+ return (
+ config.manualVoice ??
+ resolveManualVoiceConfig({
+ paseoHome: config.paseoHome,
+ persisted: undefined,
+ providers: undefined,
+ })
+ );
+}
+
function createInitialMutableDaemonConfig(config: PaseoDaemonConfig): MutableDaemonConfig {
const providers = config.providerOverrides ?? {};
+ const manualVoice = resolveDaemonManualVoiceConfig(config);
const initialConfig: MutableDaemonConfig = {
relay: { enabled: config.relayEnabled ?? true },
@@ -551,6 +573,9 @@ function createInitialMutableDaemonConfig(config: PaseoDaemonConfig): MutableDae
pluginsEnabled: config.pluginsEnabled ?? false,
plugins: config.plugins ?? {},
skills: { selection: config.skillSelection },
+ manualVoice: {
+ orchestrator: manualVoice.orchestrator,
+ },
};
if (config.terminalProfiles !== undefined) {
@@ -1343,11 +1368,10 @@ export async function createPaseoDaemon(
{ elapsed: elapsed() },
`Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand`,
);
- logger.info(
- "Voice mode configured for agent-scoped resume flow (no dedicated voice assistant provider)",
- );
logger.info({ elapsed: elapsed() }, "Preparing voice and MCP runtime");
+ let voiceChat: VoiceChat | null = null;
+
const createAgentToolHostDependencies = (
runtime: PaseoToolRuntimeContext,
): PaseoToolHostDependencies => ({
@@ -1407,10 +1431,10 @@ export async function createPaseoDaemon(
paseoHome: config.paseoHome,
worktreesRoot: config.worktreesRoot,
callerAgentId: runtime.callerAgentId,
- enableVoiceTools: runtime.enableVoiceTools,
- voiceOnly: runtime.voiceOnly,
- resolveSpeakHandler: (agentId) => wsServer?.resolveVoiceSpeakHandler(agentId) ?? null,
- resolveCallerContext: (agentId) => wsServer?.resolveVoiceCallerContext(agentId) ?? null,
+ resolveCallerContext: (agentId) =>
+ agentManager.getInternalAgentPaseoTools(agentId)?.caller ?? null,
+ resolveToolExtension: (agentId) =>
+ agentManager.getInternalAgentPaseoTools(agentId)?.extension ?? null,
logger,
});
const createAgentToolCatalog = (runtime: PaseoToolRuntimeContext) =>
@@ -1462,7 +1486,11 @@ export async function createPaseoDaemon(
req: express.Request,
res: express.Response,
): Promise => {
- if (!mcpEnabled) {
+ const hasInternalCapability = isAgentMcpCapabilityToken({
+ capabilityToken: agentMcpAuthToken,
+ authorizationHeader: req.header("authorization"),
+ });
+ if (!isAgentMcpEndpointAvailable(mcpEnabled, hasInternalCapability)) {
res.status(404).json({ error: "Agent MCP endpoint disabled" });
return;
}
@@ -1550,11 +1578,25 @@ export async function createPaseoDaemon(
logger.info({ route: agentMcpRoute, enabled: mcpEnabled }, "Agent MCP route mounted");
}
- const speechService = createSpeechService({
+ const dictationSpeechService = createSpeechService({
logger,
openaiConfig: config.openai,
speechConfig: config.speech,
});
+ const manualVoiceProvider = createManualVoiceProvider({
+ config: resolveDaemonManualVoiceConfig(config),
+ agentManager,
+ agentStorage,
+ workspaceRegistry,
+ logger,
+ });
+ daemonConfigStore.onFieldChange("manualVoice.orchestrator", (value) => {
+ const orchestrator = MutableManualVoiceOrchestratorConfigSchema.parse(value);
+ manualVoiceProvider.configureOrchestrator(orchestrator);
+ });
+ voiceChat = createVoiceChat({
+ provider: manualVoiceProvider,
+ });
logger.info({ elapsed: elapsed() }, "Speech service created");
logger.info({ elapsed: elapsed() }, "Bootstrap complete, ready to start listening");
@@ -1589,6 +1631,7 @@ export async function createPaseoDaemon(
const logAndResolve = async () => {
boundListenTarget = resolveBoundListenTarget(listenTarget, httpServer);
const mcpBaseUrl = createAgentMcpBaseUrl(boundListenTarget);
+ agentManager.setRequiredPaseoToolsBaseUrl(mcpBaseUrl);
agentMcpBaseUrl =
!mcpEnabled || config.mcpInjectIntoAgents === false ? null : mcpBaseUrl;
agentManager.setMcpBaseUrl(agentMcpBaseUrl);
@@ -1656,7 +1699,7 @@ export async function createPaseoDaemon(
},
workspaceAutoName,
config.auth,
- speechService,
+ dictationSpeechService,
terminalManager,
{
finalTimeoutMs: config.dictationFinalTimeoutMs,
@@ -1706,6 +1749,7 @@ export async function createPaseoDaemon(
pluginRuntime,
orchestrationSkills,
workspaceLabelService,
+ voiceChat,
);
pluginRuntime.bindPaseoSessionHost(wsServer);
await pluginRuntime.start();
@@ -1749,7 +1793,8 @@ export async function createPaseoDaemon(
// Start speech service after listening so synchronous Sherpa native
// model loading doesn't block the server from accepting connections.
- speechService.start();
+ dictationSpeechService.start();
+ manualVoiceProvider.startHarness();
scriptHealthMonitor.start();
} catch (error) {
unsubscribePluginProviders();
@@ -1779,7 +1824,9 @@ export async function createPaseoDaemon(
await agentStorage.flush().catch(() => undefined);
await agentProviderRuntime.shutdown();
terminalManager.killAll();
- await speechService.stop();
+ await voiceChat?.close();
+ await manualVoiceProvider.stopHarness();
+ await dictationSpeechService.stop();
await scheduleService.stop().catch(() => undefined);
await relayRuntime?.stop().catch(() => undefined);
if (wsServer) {
diff --git a/packages/server/src/server/config.test.ts b/packages/server/src/server/config.test.ts
index 80045a4454..d268a68ee4 100644
--- a/packages/server/src/server/config.test.ts
+++ b/packages/server/src/server/config.test.ts
@@ -69,7 +69,6 @@ describe("server config", () => {
PASEO_TRUSTED_PROXIES: "true",
PASEO_WEB_UI_ENABLED: "true",
PASEO_LOG_FILE_PATH: "custom.log",
- PASEO_VOICE_LLM_PROVIDER: "codex",
},
cli: { relayUseTls: false },
});
@@ -80,67 +79,73 @@ describe("server config", () => {
"daemon.relay.endpoint",
"daemon.relay.useTls",
"daemon.trustedProxies",
- "features.voiceMode.llm.provider",
"features.webUi.enabled",
"log.file.path",
]);
expect(config.listen).toBe("127.0.0.1:7000");
expect(config.trustedProxies).toBe(true);
expect(config.log?.file?.path).toBe("custom.log");
- expect(config.voiceLlmProvider).toBe("codex");
});
- test.each([
- {
- name: "local speech providers",
- providers: { dictation: "local", voiceStt: "local", voiceTts: "local" },
- expected: [
- "features.dictation.stt.model",
- "features.voiceMode.stt.model",
- "features.voiceMode.tts.model",
- ],
- },
- {
- name: "OpenAI speech providers",
- providers: { dictation: "openai", voiceStt: "openai", voiceTts: "openai" },
- expected: [
- "features.dictation.stt.confidenceThreshold",
- "features.dictation.stt.model",
- "features.voiceMode.stt.model",
- "features.voiceMode.tts.model",
- "features.voiceMode.tts.voice",
- ],
- },
- {
- name: "mixed local and OpenAI speech providers",
- providers: { dictation: "local", voiceStt: "openai", voiceTts: "local" },
- expected: [
- "features.dictation.stt.confidenceThreshold",
- "features.dictation.stt.model",
- "features.voiceMode.stt.model",
- "features.voiceMode.tts.model",
- ],
- },
- ])("classifies speech overrides for $name", ({ providers, expected }) => {
+ test("resolves the complete manual voice orchestrator selection", async () => {
+ const paseoHome = await mkdtemp(path.join(os.tmpdir(), "paseo-config-voice-agent-"));
+ roots.push(paseoHome);
+ await writeFile(
+ path.join(paseoHome, "config.json"),
+ JSON.stringify({
+ manualVoice: {
+ orchestrator: {
+ provider: "codex",
+ model: "gpt-5.4",
+ modeId: "full-access",
+ thinkingOptionId: "high",
+ },
+ stt: { provider: "openai", model: "gpt-4o-mini-transcribe", language: "de" },
+ turnDetection: { provider: "local" },
+ tts: { provider: "openai", model: "tts-1-hd", voice: "nova" },
+ },
+ providers: {
+ openai: { apiKey: "manual-speech-key" },
+ },
+ }),
+ );
+
+ const config = loadConfig(paseoHome, { env: {} });
+
+ expect(config.manualVoice.orchestrator).toEqual({
+ provider: "codex",
+ model: "gpt-5.4",
+ modeId: "full-access",
+ thinkingOptionId: "high",
+ });
+ expect(config.manualVoice.speech.providers.voiceStt.provider).toBe("openai");
+ expect(config.manualVoice.speech.providers.voiceTurnDetection.provider).toBe("local");
+ expect(config.manualVoice.speech.providers.voiceTts.provider).toBe("openai");
+ expect(config.manualVoice.speech.sttLanguages.voice).toBe("de");
+ expect(config.manualVoice.openai?.stt).toMatchObject({
+ apiKey: "manual-speech-key",
+ model: "gpt-4o-mini-transcribe",
+ });
+ expect(config.manualVoice.openai?.tts).toMatchObject({
+ apiKey: "manual-speech-key",
+ model: "tts-1-hd",
+ voice: "nova",
+ });
+ });
+
+ test("classifies only dictation speech overrides in shared config", () => {
const config = resolveConfigFromPersisted(
"/tmp/paseo-speech-override-classification",
{
version: 1,
features: {
- dictation: { enabled: true, stt: { provider: providers.dictation } },
- voiceMode: {
- enabled: true,
- stt: { provider: providers.voiceStt },
- tts: { provider: providers.voiceTts },
- },
+ dictation: { enabled: true, stt: { provider: "local" } },
},
},
{
env: {
OPENAI_API_KEY: "test-api-key",
PASEO_DICTATION_LOCAL_STT_MODEL: "parakeet-tdt-0.6b-v2-int8",
- PASEO_VOICE_LOCAL_STT_MODEL: "parakeet-tdt-0.6b-v2-int8",
- PASEO_VOICE_LOCAL_TTS_MODEL: "kokoro-en-v0_19",
STT_CONFIDENCE_THRESHOLD: "0.5",
STT_MODEL: "whisper-1",
TTS_MODEL: "tts-1",
@@ -149,7 +154,7 @@ describe("server config", () => {
},
);
- expect(config.configReload?.overrideControlledPaths).toEqual(expected);
+ expect(config.configReload?.overrideControlledPaths).toEqual(["features.dictation.stt.model"]);
});
test("resolves bundled web UI path from source-tree modules", () => {
diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts
index 7ed5f16519..fe35ed944f 100644
--- a/packages/server/src/server/config.ts
+++ b/packages/server/src/server/config.ts
@@ -12,7 +12,6 @@ import {
LogLevelSchema,
type PersistedConfig,
} from "./persisted-config.js";
-import type { AgentProvider } from "./agent/agent-sdk-types.js";
import type {
AgentProviderRuntimeSettingsMap,
ProviderOverride,
@@ -24,6 +23,7 @@ import { resolveSpeechConfig } from "./speech/speech-config-resolver.js";
import type { RequestedSpeechProviders } from "./speech/speech-types.js";
import { mergeHostnames, parseHostnamesEnv, type HostnamesConfig } from "./hostnames.js";
import { resolveGitProcessPolicy } from "../utils/git-process-scheduler.js";
+import { resolveManualVoiceConfig } from "./voice-chat/providers/manual/config.js";
const DEFAULT_PORT = 6767;
const DEFAULT_RELAY_ENDPOINT = "relay.paseo.sh:443";
@@ -193,18 +193,6 @@ function parsePositiveIntegerEnv(value: string | undefined): number | undefined
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
}
-const OptionalVoiceLlmProviderSchema = z
- .union([z.string(), z.null(), z.undefined()])
- .transform((value): string | null =>
- typeof value === "string" ? value.trim().toLowerCase() : null,
- )
- .pipe(z.union([AgentProviderSchema, z.null()]));
-
-function parseOptionalVoiceLlmProvider(value: unknown): AgentProvider | null {
- const parsed = OptionalVoiceLlmProviderSchema.safeParse(value);
- return parsed.success ? parsed.data : null;
-}
-
function extractProviderOverrides(
providers: Record | undefined,
): Record | undefined {
@@ -326,12 +314,6 @@ function resolveRelayConfig(input: ResolveRelayInput): ResolvedRelay {
};
}
-interface ResolvedVoiceLlm {
- provider: AgentProvider | null;
- providerExplicit: boolean;
- model: string | null;
-}
-
function resolveServiceProxyPublicBaseUrl(value: string | null): string | null {
if (value === null) {
return null;
@@ -394,21 +376,6 @@ function resolveWebUiConfig(
};
}
-function resolveVoiceLlmConfig(
- env: NodeJS.ProcessEnv,
- persisted: ReturnType,
-): ResolvedVoiceLlm {
- const envVoiceLlmProvider = parseOptionalVoiceLlmProvider(env.PASEO_VOICE_LLM_PROVIDER);
- const persistedVoiceLlmProvider = parseOptionalVoiceLlmProvider(
- persisted.features?.voiceMode?.llm?.provider,
- );
- return {
- provider: envVoiceLlmProvider ?? persistedVoiceLlmProvider ?? null,
- providerExplicit: envVoiceLlmProvider !== null || persistedVoiceLlmProvider !== null,
- model: persisted.features?.voiceMode?.llm?.model ?? null,
- };
-}
-
function resolveCorsAllowedOrigins(
env: NodeJS.ProcessEnv,
persisted: ReturnType,
@@ -589,7 +556,11 @@ export function resolveConfigFromPersisted(
persisted,
});
- const voiceLlm = resolveVoiceLlmConfig(env, persisted);
+ const manualVoice = resolveManualVoiceConfig({
+ paseoHome,
+ persisted: persisted.manualVoice,
+ providers: persisted.providers,
+ });
const providerOverrides = extractProviderOverrides(
persisted.agents?.providers as Record | undefined,
);
@@ -633,9 +604,7 @@ export function resolveConfigFromPersisted(
auth: resolveAuthConfig(env, persisted),
openai,
speech,
- voiceLlmProvider: voiceLlm.provider,
- voiceLlmProviderExplicit: voiceLlm.providerExplicit,
- voiceLlmModel: voiceLlm.model,
+ manualVoice,
agentProviderSettings: extractAgentProviderSettings(providerOverrides),
providerCatalogRefreshTimeoutMs: persisted.agents?.catalogRefreshTimeoutMs,
metadataGeneration: persisted.agents?.metadataGeneration,
@@ -808,42 +777,13 @@ function resolveSpeechOverrideControlledPaths(
paths.push("features.dictation.stt.model");
}
add("PASEO_DICTATION_LANGUAGE", "features.dictation.stt.language");
- add("PASEO_VOICE_MODE_ENABLED", "features.voiceMode.enabled");
- add("PASEO_VOICE_LLM_PROVIDER", "features.voiceMode.llm.provider");
- add("PASEO_VOICE_STT_PROVIDER", "features.voiceMode.stt.provider");
- if (
- env.PASEO_VOICE_LOCAL_STT_MODEL !== undefined &&
- isEnabledSpeechProvider(providers.voiceStt, "local")
- ) {
- paths.push("features.voiceMode.stt.model");
- }
- add("PASEO_VOICE_LANGUAGE", "features.voiceMode.stt.language");
- add("PASEO_VOICE_TURN_DETECTION_PROVIDER", "features.voiceMode.turnDetection.provider");
- add("PASEO_VOICE_TTS_PROVIDER", "features.voiceMode.tts.provider");
- if (
- env.PASEO_VOICE_LOCAL_TTS_MODEL !== undefined &&
- isEnabledSpeechProvider(providers.voiceTts, "local")
- ) {
- paths.push("features.voiceMode.tts.model");
- }
- add("PASEO_VOICE_LOCAL_TTS_SPEAKER_ID", "features.voiceMode.tts.speakerId");
- add("PASEO_VOICE_LOCAL_TTS_SPEED", "features.voiceMode.tts.speed");
add("PASEO_LOCAL_MODELS_DIR", "providers.local.modelsDir");
const openAiDictationStt = isEnabledSpeechProvider(providers.dictationStt, "openai");
- const openAiVoiceStt = isEnabledSpeechProvider(providers.voiceStt, "openai");
- if (env.STT_CONFIDENCE_THRESHOLD !== undefined && (openAiDictationStt || openAiVoiceStt)) {
+ if (env.STT_CONFIDENCE_THRESHOLD !== undefined && openAiDictationStt) {
paths.push("features.dictation.stt.confidenceThreshold");
}
if (env.STT_MODEL !== undefined) {
if (openAiDictationStt) paths.push("features.dictation.stt.model");
- if (openAiVoiceStt) paths.push("features.voiceMode.stt.model");
- }
- if (isEnabledSpeechProvider(providers.voiceTts, "openai")) {
- add("TTS_MODEL", "features.voiceMode.tts.model");
- add("TTS_VOICE", "features.voiceMode.tts.voice");
- }
- if (env.PASEO_DICTATION_LANGUAGE !== undefined && env.PASEO_VOICE_LANGUAGE === undefined) {
- paths.push("features.voiceMode.stt.language");
}
return paths;
}
diff --git a/packages/server/src/server/daemon-config-store.test.ts b/packages/server/src/server/daemon-config-store.test.ts
index 3629a0fd18..d688e91522 100644
--- a/packages/server/src/server/daemon-config-store.test.ts
+++ b/packages/server/src/server/daemon-config-store.test.ts
@@ -120,6 +120,55 @@ describe("DaemonConfigStore", () => {
expect(loadPersistedConfig(paseoHome).daemon?.relay?.enabled).toBe(true);
});
+ test("patch persists and emits the complete voice orchestrator selection", () => {
+ const paseoHome = mkdtempSync(path.join(tmpdir(), "paseo-daemon-config-store-"));
+ tempDirs.push(paseoHome);
+ const store = new DaemonConfigStore(paseoHome, {
+ mcp: { injectIntoAgents: false },
+ browserTools: { enabled: false },
+ providers: {},
+ metadataGeneration: { providers: [] },
+ autoArchiveAfterMerge: false,
+ enableTerminalAgentHooks: false,
+ appendSystemPrompt: "",
+ manualVoice: {
+ orchestrator: {
+ provider: null,
+ model: null,
+ modeId: null,
+ thinkingOptionId: null,
+ },
+ },
+ });
+ const changes: unknown[] = [];
+ store.onFieldChange("manualVoice.orchestrator", (value) => changes.push(value));
+
+ const next = store.patch({
+ manualVoice: {
+ orchestrator: {
+ provider: "codex",
+ model: "gpt-5.4",
+ modeId: "full-access",
+ thinkingOptionId: "high",
+ },
+ },
+ });
+
+ expect(next.manualVoice?.orchestrator).toEqual({
+ provider: "codex",
+ model: "gpt-5.4",
+ modeId: "full-access",
+ thinkingOptionId: "high",
+ });
+ expect(changes).toEqual([next.manualVoice?.orchestrator]);
+ expect(loadPersistedConfig(paseoHome).manualVoice?.orchestrator).toEqual({
+ provider: "codex",
+ model: "gpt-5.4",
+ modeId: "full-access",
+ thinkingOptionId: "high",
+ });
+ });
+
test("patch round-trips agent profiles through the strictly-parsed persisted config", () => {
const paseoHome = mkdtempSync(path.join(tmpdir(), "paseo-daemon-config-store-"));
tempDirs.push(paseoHome);
diff --git a/packages/server/src/server/daemon-config-store.ts b/packages/server/src/server/daemon-config-store.ts
index 3e285bded0..15ed0308fe 100644
--- a/packages/server/src/server/daemon-config-store.ts
+++ b/packages/server/src/server/daemon-config-store.ts
@@ -29,6 +29,7 @@ interface SupportedMutableConfigPatch {
terminalProfiles?: MutableDaemonConfig["terminalProfiles"];
agentProfiles?: MutableDaemonConfig["agentProfiles"];
skills?: MutableDaemonConfig["skills"];
+ manualVoice?: MutableDaemonConfigPatch["manualVoice"];
pluginsEnabled?: boolean;
plugins?: MutableDaemonConfig["plugins"];
}
@@ -188,6 +189,7 @@ const RELOADABLE_PATHS = [
"agents.catalogRefreshTimeoutMs",
"agents.metadataGeneration",
"agents.skills.selection",
+ "manualVoice.orchestrator",
"pluginsEnabled",
] as const;
@@ -211,6 +213,7 @@ const PERSISTED_TO_MUTABLE_PATH = new Map([
["agents.catalogRefreshTimeoutMs", "catalogRefreshTimeoutMs"],
["agents.metadataGeneration", "metadataGeneration"],
["agents.skills.selection", "skills.selection"],
+ ["manualVoice.orchestrator", "manualVoice.orchestrator"],
["pluginsEnabled", "pluginsEnabled"],
]);
@@ -274,6 +277,9 @@ function pickSupportedPatchFields(patch: MutableDaemonConfigPatch): SupportedMut
: {}),
...(patch.terminalProfiles !== undefined ? { terminalProfiles: patch.terminalProfiles } : {}),
...(patch.agentProfiles !== undefined ? { agentProfiles: patch.agentProfiles } : {}),
+ ...(patch.manualVoice?.orchestrator !== undefined
+ ? { manualVoice: { orchestrator: patch.manualVoice.orchestrator } }
+ : {}),
...(patch.pluginsEnabled !== undefined ? { pluginsEnabled: patch.pluginsEnabled } : {}),
...(patch.plugins !== undefined ? { plugins: patch.plugins } : {}),
};
@@ -584,15 +590,34 @@ function mergeMutablePatchIntoPersistedConfig(params: {
const { persisted, patch, removeProviders, persistRelayEnabled } = params;
const daemon = mergeMutableDaemonPatch(persisted.daemon, patch, persistRelayEnabled);
const agents = mergeMutableAgentPatch(persisted.agents, patch, removeProviders);
+ const manualVoice = mergeMutableManualVoicePatch(persisted.manualVoice, patch);
return {
...persisted,
...(patch.pluginsEnabled !== undefined ? { pluginsEnabled: patch.pluginsEnabled } : {}),
...(patch.plugins !== undefined ? { plugins: patch.plugins } : {}),
...(daemon ? { daemon } : { daemon: undefined }),
...(agents ? { agents } : { agents: undefined }),
+ ...(manualVoice ? { manualVoice } : { manualVoice: undefined }),
} as PersistedConfig;
}
+function mergeMutableManualVoicePatch(
+ persistedManualVoice: PersistedConfig["manualVoice"],
+ patch: Omit,
+): PersistedConfig["manualVoice"] {
+ const orchestrator = patch.manualVoice?.orchestrator;
+ if (!orchestrator) return persistedManualVoice;
+ const nextOrchestrator = { ...persistedManualVoice?.orchestrator };
+ for (const [key, value] of Object.entries(orchestrator)) {
+ if (value === null) delete nextOrchestrator[key as keyof typeof nextOrchestrator];
+ else if (value !== undefined) Object.assign(nextOrchestrator, { [key]: value });
+ }
+ return {
+ ...persistedManualVoice,
+ orchestrator: nextOrchestrator,
+ };
+}
+
function mergeMutableAgentPatch(
persistedAgents: PersistedConfig["agents"],
patch: Omit,
diff --git a/packages/server/src/server/persisted-config.test.ts b/packages/server/src/server/persisted-config.test.ts
index 8ea1c5c25d..eeb85da59a 100644
--- a/packages/server/src/server/persisted-config.test.ts
+++ b/packages/server/src/server/persisted-config.test.ts
@@ -625,39 +625,55 @@ describe("PersistedConfigSchema logging config", () => {
});
});
-describe("PersistedConfigSchema voice mode config", () => {
- test("accepts a dedicated turn detection provider", () => {
+describe("PersistedConfigSchema manual voice config", () => {
+ test("accepts the provider-owned manual voice configuration", () => {
const parsed = PersistedConfigSchema.parse({
+ manualVoice: {
+ orchestrator: {
+ provider: "codex",
+ model: "gpt-5.4",
+ modeId: "full-access",
+ thinkingOptionId: "high",
+ },
+ stt: { provider: "openai", model: "gpt-4o-mini-transcribe", language: " de " },
+ turnDetection: { provider: "local" },
+ tts: { provider: "openai", model: "tts-1-hd", voice: "nova" },
+ },
+ });
+
+ expect(parsed.manualVoice).toEqual({
+ orchestrator: {
+ provider: "codex",
+ model: "gpt-5.4",
+ modeId: "full-access",
+ thinkingOptionId: "high",
+ },
+ stt: { provider: "openai", model: "gpt-4o-mini-transcribe", language: "de" },
+ turnDetection: { provider: "local" },
+ tts: { provider: "openai", model: "tts-1-hd", voice: "nova" },
+ });
+ });
+
+ test("rejects the removed feature voice-mode path", () => {
+ const parsed = PersistedConfigSchema.safeParse({
features: {
voiceMode: {
- turnDetection: {
- provider: "local",
- },
+ llm: { provider: "codex", modeId: "full-access" },
},
},
});
- expect(parsed.features?.voiceMode?.turnDetection?.provider).toBe("local");
+ expect(parsed.success).toBe(false);
});
+});
- test("accepts trimmed STT language fields", () => {
+describe("PersistedConfigSchema dictation language", () => {
+ test("accepts a trimmed STT language", () => {
const parsed = PersistedConfigSchema.parse({
- features: {
- dictation: {
- stt: {
- language: " fr ",
- },
- },
- voiceMode: {
- stt: {
- language: " de ",
- },
- },
- },
+ features: { dictation: { stt: { language: " fr " } } },
});
expect(parsed.features?.dictation?.stt?.language).toBe("fr");
- expect(parsed.features?.voiceMode?.stt?.language).toBe("de");
});
});
@@ -734,6 +750,37 @@ describe("loadPersistedConfig", () => {
rmSync(home, { recursive: true, force: true });
}
});
+
+ test("loads a config that still contains the removed feature voice-mode block", () => {
+ const home = createTempHome();
+ const configPath = path.join(home, "config.json");
+ try {
+ writeFileSync(
+ configPath,
+ `${JSON.stringify(
+ {
+ version: 1,
+ features: {
+ dictation: { enabled: true },
+ voiceMode: {
+ enabled: true,
+ llm: { provider: "codex", modeId: "full-access" },
+ },
+ },
+ },
+ null,
+ 2,
+ )}\n`,
+ );
+
+ const config = loadPersistedConfig(home);
+
+ expect(config.features?.dictation?.enabled).toBe(true);
+ expect((config.features as Record)?.voiceMode).toBeUndefined();
+ } finally {
+ rmSync(home, { recursive: true, force: true });
+ }
+ });
});
describe.skipIf(process.platform === "win32")("persisted config file permissions", () => {
diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts
index 1cdaf209d8..25bde6c617 100644
--- a/packages/server/src/server/persisted-config.ts
+++ b/packages/server/src/server/persisted-config.ts
@@ -1,6 +1,7 @@
import { existsSync, readFileSync } from "node:fs";
import path from "node:path";
import { z } from "zod";
+import { ManualVoicePersistedConfigSchema } from "./voice-chat/providers/manual/config.js";
import {
AgentProviderRuntimeSettingsMapSchema,
@@ -119,43 +120,6 @@ const FeatureDictationSchema = z
})
.strict();
-const FeatureVoiceModeSchema = z
- .object({
- enabled: z.boolean().optional(),
- llm: z
- .object({
- provider: z.string().optional(),
- model: z.string().min(1).optional(),
- })
- .strict()
- .optional(),
- stt: z
- .object({
- provider: SpeechProviderIdSchema.optional(),
- model: z.string().min(1).optional(),
- language: z.string().trim().min(1).optional(),
- })
- .strict()
- .optional(),
- turnDetection: z
- .object({
- provider: SpeechProviderIdSchema.optional(),
- })
- .strict()
- .optional(),
- tts: z
- .object({
- provider: SpeechProviderIdSchema.optional(),
- model: z.string().min(1).optional(),
- voice: z.enum(["alloy", "echo", "fable", "onyx", "nova", "shimmer"]).optional(),
- speakerId: z.number().int().optional(),
- speed: z.number().optional(),
- })
- .strict()
- .optional(),
- })
- .strict();
-
const FeatureWebUiSchema = z
.object({
enabled: z.boolean().optional(),
@@ -310,6 +274,7 @@ export const PersistedConfigSchema = z
.optional(),
providers: ProvidersSchema.optional(),
+ manualVoice: ManualVoicePersistedConfigSchema.optional(),
pluginsEnabled: z.boolean().optional(),
plugins: z.record(PluginIdSchema, PluginSourceSchema).optional(),
worktrees: WorktreesConfigSchema.optional(),
@@ -325,7 +290,6 @@ export const PersistedConfigSchema = z
features: z
.object({
dictation: FeatureDictationSchema.optional(),
- voiceMode: FeatureVoiceModeSchema.optional(),
webUi: FeatureWebUiSchema.optional(),
})
.strict()
@@ -374,15 +338,23 @@ function getLogger(logger: LoggerLike | undefined): LoggerLike | undefined {
}
// Removed config fields are stripped before parsing so the strict schema does not
-// reject a config written by an older release. The stripped values are discarded,
-// not migrated — there is no back-compat for the removed `providers.openai.voice`
-// block (use `providers.openai.stt` / `providers.openai.tts`).
+// reject a config written by an older release. The values are discarded, not
+// migrated; this only keeps removed settings from preventing daemon startup.
function stripRemovedConfigFields(parsed: unknown): unknown {
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
return parsed;
}
const root = { ...(parsed as Record) };
+
+ const features = root.features;
+ if (features && typeof features === "object" && !Array.isArray(features)) {
+ const featuresRecord = { ...(features as Record) };
+ // COMPAT(removedVoiceModeConfig): added in v0.7.0, remove after 2027-02-28.
+ delete featuresRecord.voiceMode;
+ root.features = featuresRecord;
+ }
+
const providers = root.providers;
if (!providers || typeof providers !== "object" || Array.isArray(providers)) {
return root;
diff --git a/packages/server/src/server/session.test.ts b/packages/server/src/server/session.test.ts
index af9fd2d539..4480ccc6d7 100644
--- a/packages/server/src/server/session.test.ts
+++ b/packages/server/src/server/session.test.ts
@@ -314,6 +314,7 @@ interface SessionForTestOptions {
hubExecutionAgents?: SessionOptions["hubExecutionAgents"];
stt?: SessionOptions["stt"];
voice?: SessionOptions["voice"];
+ voiceChat?: SessionOptions["voiceChat"];
paseoHome?: string;
serverId?: SessionOptions["serverId"];
daemonVersion?: SessionOptions["daemonVersion"];
@@ -425,6 +426,7 @@ function createSessionForTest(options: SessionForTestOptions = {}): Session {
getDaemonTcpPort: options.getDaemonTcpPort,
getDaemonTcpHost: options.getDaemonTcpHost,
voice: options.voice,
+ voiceChat: options.voiceChat,
serverId: options.serverId,
daemonVersion: options.daemonVersion,
daemonRuntimeConfig: options.daemonRuntimeConfig,
@@ -433,6 +435,20 @@ function createSessionForTest(options: SessionForTestOptions = {}): Session {
return new Session(sessionOptions);
}
+test("transport disconnect terminates voice without cleaning up the reconnectable session", async () => {
+ const close = vi.fn().mockResolvedValue(undefined);
+ const session = createSessionForTest({
+ voiceChat: {
+ openClientSession: vi.fn(() => ({ handle: vi.fn(), disconnect: close, close: vi.fn() })),
+ close: vi.fn(),
+ },
+ });
+
+ await session.handleTransportDisconnect();
+
+ expect(close).toHaveBeenCalledOnce();
+});
+
test("routes host-scoped agent skills requests through the daemon owner", async () => {
const messages: SessionOutboundMessage[] = [];
const status = {
diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts
index c5862381cd..76767f52be 100644
--- a/packages/server/src/server/session.ts
+++ b/packages/server/src/server/session.ts
@@ -8,7 +8,6 @@ import { formatPluginSourceReference } from "@getpaseo/protocol/plugin-source-re
import {
serializeAgentStreamEvent,
type AgentSnapshotPayload,
- type AgentAttachment,
type FirstAgentContext,
type SessionInboundMessage,
type SessionOutboundMessage,
@@ -33,8 +32,7 @@ import type { BinaryFrame } from "@getpaseo/protocol/binary-frames/index";
import { CursorError } from "./pagination/cursor.js";
import { SortablePager, type SortSpec } from "./pagination/sortable-pager.js";
import { describeAgentHistoryMatches, rankAgentHistoryCandidates } from "./agent-history-search.js";
-import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech/speech-provider.js";
-import type { TurnDetectionProvider } from "./speech/turn-detection-provider.js";
+import type { SpeechToTextProvider } from "./speech/speech-provider.js";
import {
buildConfigOverrides,
isStoredAgentProviderAvailable,
@@ -51,7 +49,6 @@ import {
resolveFirstAgentPromptTitle,
} from "./agent/create-agent-title.js";
import { respondToAgentPermission } from "./agent/permission-response.js";
-import type { VoiceCallerContext, VoiceSpeakHandler } from "./voice-types.js";
import type { ScriptHealthState } from "./script-health-monitor.js";
import { spawnWorkspaceScript } from "./worktree-bootstrap.js";
import type { WorkspaceScriptRuntimeStore } from "./workspace-script-runtime-store.js";
@@ -127,7 +124,6 @@ import {
getAgentStreamEventTurnId,
type AgentPersistenceHandle,
type AgentPermissionResponse,
- type AgentRunOptions,
type AgentSessionConfig,
} from "./agent/agent-sdk-types.js";
import type { StoredAgentRecord } from "./agent/agent-storage.js";
@@ -155,14 +151,20 @@ import {
type WorkspaceMutation,
type WorkspaceRegistry,
} from "./workspace-registry.js";
-import { wrapSpokenInput } from "./voice-config.js";
-import { isVoicePermissionAllowed } from "./voice-permission-policy.js";
+import {
+ rejectLegacyVoiceMode,
+ type VoiceChat,
+ type VoiceChatClientSession,
+} from "./voice-chat/index.js";
import {
ProjectIconReader,
removeProjectCustomIcon,
setProjectCustomIcon,
} from "../utils/project-custom-icon.js";
-import { VoiceSession } from "./session/voice/voice-session.js";
+import {
+ createDictationSession,
+ type DictationSession,
+} from "./session/dictation/dictation-session.js";
import { CheckoutSession } from "./session/checkout/checkout-session.js";
import {
createWorkspaceGitObserverService,
@@ -505,9 +507,6 @@ export interface SessionOptions {
};
orchestrationSkills?: import("./orchestration-skills/index.js").OrchestrationSkills;
mcpBaseUrl?: string | null;
- stt: Resolvable;
- sttLanguage?: string;
- tts: Resolvable;
terminalManager: TerminalManager | null;
providerSnapshotManager: ProviderSnapshotManager;
providerUsageService: ProviderUsageService;
@@ -526,15 +525,7 @@ export interface SessionOptions {
getDaemonTcpHost?: () => string | null;
serviceProxyPublicBaseUrl?: string | null;
resolveScriptHealth?: (hostname: string) => ScriptHealthState | null;
- voice?: {
- turnDetection?: Resolvable;
- };
- voiceBridge?: {
- registerVoiceSpeakHandler?: (agentId: string, handler: VoiceSpeakHandler) => void;
- unregisterVoiceSpeakHandler?: (agentId: string) => void;
- registerVoiceCallerContext?: (agentId: string, context: VoiceCallerContext) => void;
- unregisterVoiceCallerContext?: (agentId: string) => void;
- };
+ voiceChat?: VoiceChat | null;
dictation?: {
finalTimeoutMs?: number;
stt?: Resolvable;
@@ -561,6 +552,31 @@ export type SessionLifecycleIntent =
reason: string;
};
+function createClientDictationSession(input: {
+ options: SessionOptions["dictation"];
+ logger: pino.Logger;
+ sessionId: string;
+ emit(message: SessionOutboundMessage): void;
+}): DictationSession {
+ return createDictationSession({
+ logger: input.logger,
+ sessionId: input.sessionId,
+ emit: input.emit,
+ stt: input.options?.stt ?? null,
+ language: input.options?.sttLanguage,
+ finalTimeoutMs: input.options?.finalTimeoutMs,
+ getSpeechReadiness: input.options?.getSpeechReadiness,
+ });
+}
+
+function openVoiceChatSession(
+ voiceChat: VoiceChat | null | undefined,
+ input: { emit(message: SessionOutboundMessage): void },
+): VoiceChatClientSession | null {
+ if (!voiceChat) return null;
+ return voiceChat.openClientSession(input);
+}
+
function parseClientCapabilities(
capabilities: Record | null | undefined,
): ReadonlySet {
@@ -735,7 +751,8 @@ export class Session {
private readonly workspaceSetupRuntime: WorkspaceSetupRuntime;
private readonly workspaceGitObserver: WorkspaceGitObserverService;
private readonly workspaceDirectory: WorkspaceDirectory;
- private readonly voiceSession: VoiceSession;
+ private readonly dictationSession: DictationSession;
+ private readonly voiceChatSession: VoiceChatClientSession | null;
private readonly checkoutSession: CheckoutSession;
private readonly scheduleSession: ScheduleSession;
private readonly providerCatalogSession: ProviderCatalogSession;
@@ -781,9 +798,6 @@ export class Session {
daemonConfigStore,
pluginRuntime,
orchestrationSkills,
- stt,
- sttLanguage,
- tts,
terminalManager,
providerSnapshotManager,
providerUsageService,
@@ -796,8 +810,7 @@ export class Session {
getDaemonTcpHost,
serviceProxyPublicBaseUrl,
resolveScriptHealth,
- voice,
- voiceBridge,
+ voiceChat,
dictation,
serverId,
daemonVersion,
@@ -1075,39 +1088,14 @@ export class Session {
buildWorkspaceDescriptor: (input) => this.buildWorkspaceDescriptor(input),
});
- this.voiceSession = new VoiceSession({
- host: {
- emit: (msg) => this.emit(msg),
- loadAgent: (agentId) =>
- ensureAgentLoaded(agentId, {
- agentManager: this.agentManager,
- agentStorage: this.agentStorage,
- logger: this.sessionLogger,
- }),
- reloadAgentSession: (agentId, overrides) =>
- this.agentManager.reloadAgentSession(agentId, overrides),
- sendSpokenInput: async (agentId, text) => {
- await this.handleSendAgentMessage(
- agentId,
- text,
- undefined,
- undefined,
- undefined,
- undefined,
- { spokenInput: true },
- );
- },
- interruptAgentIfRunning: (agentId) => this.interruptAgentIfRunning(agentId),
- hasActiveAgentRun: (agentId) => this.hasActiveAgentRun(agentId),
- },
+ this.dictationSession = createClientDictationSession({
+ options: dictation,
logger: this.sessionLogger,
sessionId: this.sessionId,
- sttLanguage,
- tts,
- stt,
- voice,
- voiceBridge,
- dictation,
+ emit: (message) => this.emit(message),
+ });
+ this.voiceChatSession = openVoiceChatSession(voiceChat, {
+ emit: (message) => this.emit(message),
});
this.subscribeToAgentEvents();
@@ -1467,13 +1455,6 @@ export class Session {
}
}
- private hasActiveAgentRun(agentId: string | null): boolean {
- if (!agentId) {
- return false;
- }
- return this.agentManager.hasInFlightRun(agentId);
- }
-
private handleAgentRunError(agentId: string, error: unknown, context: string): void {
const message = errorToFriendlyMessage(error);
this.sessionLogger.error({ err: error, agentId, context }, `${context} for agent ${agentId}`);
@@ -1719,28 +1700,6 @@ export class Session {
return;
}
- if (
- this.voiceSession.isActiveForAgent(event.agentId) &&
- event.event.type === "permission_requested" &&
- isVoicePermissionAllowed(event.event.request)
- ) {
- const requestId = event.event.request.id;
- void this.agentManager
- .respondToPermission(event.agentId, requestId, {
- behavior: "allow",
- })
- .catch((error) => {
- this.sessionLogger.warn(
- {
- err: error,
- agentId: event.agentId,
- requestId,
- },
- "Failed to auto-allow speak tool permission in voice mode",
- );
- });
- }
-
const serializedEvent = serializeAgentStreamEvent(event.event);
if (!serializedEvent) {
return;
@@ -2222,28 +2181,37 @@ export class Session {
private dispatchVoiceAndControlMessage(msg: SessionInboundMessage): Promise | undefined {
switch (msg.type) {
+ case "voice.call.start.request":
+ case "voice.call.stop.request":
+ case "voice.call.context.update.request":
+ case "voice.call.transport.client":
+ if (!this.voiceChatSession) {
+ throw new Error("Voice chat is unavailable");
+ }
+ return this.voiceChatSession.handle(msg);
case "voice_audio_chunk":
- return this.voiceSession.handleAudioChunk(msg);
+ return undefined;
case "abort_request":
- return this.voiceSession.handleAbort();
+ return undefined;
case "audio_played":
- this.voiceSession.handleAudioPlayed(msg.id);
return undefined;
case "set_voice_mode":
- return this.voiceSession.handleSetVoiceMode(msg.enabled, msg.agentId, msg.requestId);
+ // COMPAT(legacyVoiceMode): added in v0.7.0, remove after 2027-08-29.
+ rejectLegacyVoiceMode({ requestId: msg.requestId, emit: (message) => this.emit(message) });
+ return undefined;
case "dictation_stream_start":
- return this.voiceSession.handleDictationStreamStart(msg);
+ return this.dictationSession.start(msg);
case "dictation_stream_chunk":
- return this.voiceSession.handleDictationChunk({
+ return this.dictationSession.append({
dictationId: msg.dictationId,
seq: msg.seq,
audioBase64: msg.audio,
format: msg.format,
});
case "dictation_stream_finish":
- return this.voiceSession.handleDictationFinish(msg.dictationId, msg.finalSeq);
+ return this.dictationSession.finish(msg.dictationId, msg.finalSeq);
case "dictation_stream_cancel":
- this.voiceSession.handleDictationCancel(msg.dictationId);
+ this.dictationSession.cancel(msg.dictationId);
return undefined;
case "restart_server_request":
return this.handleRestartServerRequest(msg.requestId, msg.reason);
@@ -3467,60 +3435,6 @@ export class Session {
}
}
- /**
- * Handle text message to agent (with optional image attachments)
- */
- private async handleSendAgentMessage(
- agentId: string,
- text: string,
- messageId?: string,
- images?: Array<{ data: string; mimeType: string }>,
- attachments?: AgentAttachment[],
- runOptions?: AgentRunOptions,
- options?: { spokenInput?: boolean },
- ): Promise<{ ok: true } | { ok: false; error: string }> {
- this.sessionLogger.info(
- {
- agentId,
- textPreview: text.substring(0, 50),
- imageCount: images?.length ?? 0,
- attachmentCount: attachments?.length ?? 0,
- },
- `Sending text to agent ${agentId}${
- images && images.length > 0 ? ` with ${images.length} image attachment(s)` : ""
- }${
- attachments && attachments.length > 0
- ? ` and ${attachments.length} structured attachment(s)`
- : ""
- }`,
- );
-
- const promptText = options?.spokenInput ? wrapSpokenInput(text) : text;
- const prompt = buildAgentPrompt(promptText, images, attachments);
-
- try {
- await sendPromptToAgent({
- agentManager: this.agentManager,
- agentStorage: this.agentStorage,
- agentId,
- prompt,
- messageId,
- runOptions,
- // A typed or spoken message from the human answers any permission the
- // agent is blocked on.
- clearPendingPermissions: true,
- logger: this.sessionLogger,
- });
- return { ok: true };
- } catch (error) {
- this.handleAgentRunError(agentId, error, "Failed to send agent message");
- return {
- ok: false,
- error: error instanceof Error ? error.message : String(error),
- };
- }
- }
-
/**
* Handle create agent request
*/
@@ -7740,7 +7654,8 @@ export class Session {
}
this.providerCatalogSession.dispose();
- await this.voiceSession.cleanup();
+ await this.dictationSession.close();
+ await this.voiceChatSession?.close();
this.terminalController.dispose();
@@ -7749,6 +7664,10 @@ export class Session {
this.workspaceGitObserver.dispose();
this.workspaceFilesSession.dispose();
}
+
+ public async handleTransportDisconnect(): Promise {
+ await this.voiceChatSession?.disconnect();
+ }
}
interface CloneRepositoryInput {
diff --git a/packages/server/src/server/session.voice-mcp-config.test.ts b/packages/server/src/server/session.voice-mcp-config.test.ts
deleted file mode 100644
index 98b1d41020..0000000000
--- a/packages/server/src/server/session.voice-mcp-config.test.ts
+++ /dev/null
@@ -1,78 +0,0 @@
-import { describe, expect, test } from "vitest";
-
-import {
- buildVoiceAgentMcpServerConfig,
- buildVoiceModeSystemPrompt,
- stripVoiceModeSystemPrompt,
-} from "./voice-config.js";
-
-describe("voice MCP stdio config", () => {
- test("builds stdio MCP config for voice agent", () => {
- const config = buildVoiceAgentMcpServerConfig({
- command: "/usr/local/bin/node",
- baseArgs: ["/tmp/mcp-stdio-socket-bridge-cli.mjs"],
- socketPath: "/tmp/paseo-voice.sock",
- env: {
- ELECTRON_RUN_AS_NODE: "1",
- PASEO_HOME: "/tmp/paseo-home",
- },
- });
-
- expect(config.type).toBe("stdio");
- expect(config.command).toBe("/usr/local/bin/node");
- expect(config.args).toEqual([
- "/tmp/mcp-stdio-socket-bridge-cli.mjs",
- "--socket",
- "/tmp/paseo-voice.sock",
- ]);
- expect(config.env).toEqual({
- ELECTRON_RUN_AS_NODE: "1",
- PASEO_HOME: "/tmp/paseo-home",
- });
- });
-});
-
-describe("voice mode prompt instructions", () => {
- test("builds enabled voice instructions and preserves base prompt", () => {
- const prompt = buildVoiceModeSystemPrompt("Base system prompt", true);
-
- expect(prompt).toContain("Base system prompt");
- expect(prompt).toContain("");
- expect(prompt).toContain("Paseo voice mode is now on.");
- expect(prompt).toContain("Always use the speak tool for all user-facing communication.");
- expect(prompt).toContain("");
- });
-
- test("builds disabled voice instructions and supersedes previous voice block", () => {
- const existing = [
- "Base system prompt",
- "",
- "legacy voice instruction",
- "",
- ].join("\n\n");
-
- const prompt = buildVoiceModeSystemPrompt(existing, false);
-
- expect(prompt).toContain("Base system prompt");
- expect(prompt).toContain("Paseo voice mode is now off.");
- expect(prompt).toContain("Ignore any earlier Paseo voice mode instructions in this thread.");
- expect(prompt.match(//g)?.length ?? 0).toBe(1);
- expect(prompt).not.toContain("legacy voice instruction");
- });
-
- test("strips voice blocks from persisted prompt", () => {
- const existing = [
- "Base system prompt",
- "",
- "legacy voice instruction",
- "",
- ].join("\n\n");
-
- expect(stripVoiceModeSystemPrompt(existing)).toBe("Base system prompt");
- expect(
- stripVoiceModeSystemPrompt(
- ["", "legacy voice instruction", ""].join("\n\n"),
- ),
- ).toBeUndefined();
- });
-});
diff --git a/packages/server/src/server/session/dictation/dictation-session.ts b/packages/server/src/server/session/dictation/dictation-session.ts
new file mode 100644
index 0000000000..606d8fc309
--- /dev/null
+++ b/packages/server/src/server/session/dictation/dictation-session.ts
@@ -0,0 +1,79 @@
+import type { Logger } from "pino";
+import type { SessionInboundMessage, SessionOutboundMessage } from "../../messages.js";
+import {
+ DictationStreamManager,
+ type DictationStreamOutboundMessage,
+} from "../../dictation/dictation-stream-manager.js";
+import type { SpeechToTextProvider } from "../../speech/speech-provider.js";
+import type { Resolvable } from "../../speech/provider-resolver.js";
+import type { SpeechReadinessSnapshot } from "../../speech/speech-runtime.js";
+
+function resolveUnavailableSpeech(readiness: SpeechReadinessSnapshot | undefined) {
+ if (!readiness) return null;
+ if (!readiness.voiceFeature.available) return readiness.voiceFeature;
+ if (!readiness.dictation.enabled || !readiness.dictation.available) {
+ return readiness.dictation;
+ }
+ return null;
+}
+
+export interface DictationSession {
+ start(message: Extract): Promise;
+ append(input: {
+ dictationId: string;
+ seq: number;
+ audioBase64: string;
+ format: string;
+ }): Promise;
+ finish(dictationId: string, finalSeq: number): Promise;
+ cancel(dictationId: string): void;
+ close(): Promise;
+}
+
+export function createDictationSession(options: {
+ logger: Logger;
+ sessionId: string;
+ emit(message: SessionOutboundMessage): void;
+ stt: Resolvable;
+ language?: string;
+ finalTimeoutMs?: number;
+ getSpeechReadiness?: () => SpeechReadinessSnapshot;
+}): DictationSession {
+ const manager = new DictationStreamManager({
+ logger: options.logger,
+ sessionId: options.sessionId,
+ emit(message: DictationStreamOutboundMessage) {
+ options.emit(message as SessionOutboundMessage);
+ },
+ stt: options.stt,
+ language: options.language,
+ finalTimeoutMs: options.finalTimeoutMs,
+ });
+
+ return {
+ async start(message) {
+ const readiness = options.getSpeechReadiness?.();
+ const unavailable = resolveUnavailableSpeech(readiness);
+ if (unavailable) {
+ options.emit({
+ type: "dictation_stream_error",
+ payload: {
+ dictationId: message.dictationId,
+ error: unavailable.message,
+ retryable: unavailable.retryable,
+ reasonCode: unavailable.reasonCode,
+ missingModelIds: [...unavailable.missingModelIds],
+ },
+ });
+ return;
+ }
+ await manager.handleStart(message.dictationId, message.format);
+ },
+ append: (input) => manager.handleChunk(input),
+ finish: (dictationId, finalSeq) => manager.handleFinish(dictationId, finalSeq),
+ cancel: (dictationId) => manager.handleCancel(dictationId),
+ async close() {
+ manager.cleanupAll();
+ },
+ };
+}
diff --git a/packages/server/src/server/speech/providers/local/config.ts b/packages/server/src/server/speech/providers/local/config.ts
index 8e7d500dc2..b001136bb9 100644
--- a/packages/server/src/server/speech/providers/local/config.ts
+++ b/packages/server/src/server/speech/providers/local/config.ts
@@ -126,9 +126,7 @@ function buildLocalSpeechLanguageResolutionInput(params: {
DEFAULT_STT_LANGUAGE,
]),
voiceLanguage: firstNonEmptyString([
- env.PASEO_VOICE_LANGUAGE,
env.PASEO_DICTATION_LANGUAGE,
- persisted.features?.voiceMode?.stt?.language,
persisted.features?.dictation?.stt?.language,
DEFAULT_STT_LANGUAGE,
]),
@@ -159,33 +157,11 @@ function buildLocalSpeechResolutionInput(params: {
),
DEFAULT_LOCAL_STT_MODEL,
]),
- voiceLocalSttModel: firstDefinedValue([
- env.PASEO_VOICE_LOCAL_STT_MODEL,
- persistedLocalFeatureModel(
- providers.voiceStt.provider,
- providers.voiceStt.enabled,
- persisted.features?.voiceMode?.stt?.model,
- ),
- DEFAULT_LOCAL_STT_MODEL,
- ]),
- voiceLocalTtsModel: firstDefinedValue([
- env.PASEO_VOICE_LOCAL_TTS_MODEL,
- persistedLocalFeatureModel(
- providers.voiceTts.provider,
- providers.voiceTts.enabled,
- persisted.features?.voiceMode?.tts?.model,
- ),
- DEFAULT_LOCAL_TTS_MODEL,
- ]),
+ voiceLocalSttModel: firstDefinedValue([DEFAULT_LOCAL_STT_MODEL]),
+ voiceLocalTtsModel: firstDefinedValue([DEFAULT_LOCAL_TTS_MODEL]),
...buildLocalSpeechLanguageResolutionInput({ env, persisted }),
- voiceLocalTtsSpeakerId: firstDefinedValue([
- env.PASEO_VOICE_LOCAL_TTS_SPEAKER_ID,
- persisted.features?.voiceMode?.tts?.speakerId,
- ]),
- voiceLocalTtsSpeed: firstDefinedValue([
- env.PASEO_VOICE_LOCAL_TTS_SPEED,
- persisted.features?.voiceMode?.tts?.speed,
- ]),
+ voiceLocalTtsSpeakerId: firstDefinedValue([]),
+ voiceLocalTtsSpeed: firstDefinedValue([]),
};
}
diff --git a/packages/server/src/server/speech/providers/openai/config.ts b/packages/server/src/server/speech/providers/openai/config.ts
index ac8d2ffa3a..f4953cee32 100644
--- a/packages/server/src/server/speech/providers/openai/config.ts
+++ b/packages/server/src/server/speech/providers/openai/config.ts
@@ -93,29 +93,15 @@ function buildOpenAiSttInput(params: {
]),
sttModel: firstDefined([
env.STT_MODEL,
- pickIfOpenAi(providers.voiceStt, persisted.features?.voiceMode?.stt?.model),
pickIfOpenAi(providers.dictationStt, persisted.features?.dictation?.stt?.model),
]),
};
}
-function buildOpenAiTtsInput(params: {
- env: NodeJS.ProcessEnv;
- persisted: PersistedConfig;
- providers: RequestedSpeechProviders;
-}): Record {
- const { env, persisted, providers } = params;
+function buildOpenAiTtsInput(env: NodeJS.ProcessEnv): Record {
return {
- ttsVoice: firstDefined([
- env.TTS_VOICE,
- pickIfOpenAi(providers.voiceTts, persisted.features?.voiceMode?.tts?.voice),
- "alloy",
- ]),
- ttsModel: firstDefined([
- env.TTS_MODEL,
- pickIfOpenAi(providers.voiceTts, persisted.features?.voiceMode?.tts?.model),
- DEFAULT_OPENAI_TTS_MODEL,
- ]),
+ ttsVoice: firstDefined([env.TTS_VOICE, "alloy"]),
+ ttsModel: firstDefined([env.TTS_MODEL, DEFAULT_OPENAI_TTS_MODEL]),
};
}
@@ -152,7 +138,7 @@ function buildOpenAiResolutionInput(params: {
env.OPENAI_BASE_URL,
]),
...buildOpenAiSttInput(params),
- ...buildOpenAiTtsInput(params),
+ ...buildOpenAiTtsInput(params.env),
};
}
diff --git a/packages/server/src/server/speech/speech-config-resolver.test.ts b/packages/server/src/server/speech/speech-config-resolver.test.ts
index 0ee9df888a..87b9003d67 100644
--- a/packages/server/src/server/speech/speech-config-resolver.test.ts
+++ b/packages/server/src/server/speech/speech-config-resolver.test.ts
@@ -23,21 +23,9 @@ describe("resolveSpeechConfig", () => {
explicit: false,
enabled: true,
});
- expect(result.speech.providers.voiceTurnDetection).toEqual({
- provider: "local",
- explicit: false,
- enabled: true,
- });
- expect(result.speech.providers.voiceStt).toEqual({
- provider: "local",
- explicit: false,
- enabled: true,
- });
- expect(result.speech.providers.voiceTts).toEqual({
- provider: "local",
- explicit: false,
- enabled: true,
- });
+ expect(result.speech.providers.voiceTurnDetection.enabled).toBe(false);
+ expect(result.speech.providers.voiceStt.enabled).toBe(false);
+ expect(result.speech.providers.voiceTts.enabled).toBe(false);
expect(result.speech.local).toEqual({
modelsDir: path.join(paseoHome, "models", "local-speech"),
models: {
@@ -48,40 +36,23 @@ describe("resolveSpeechConfig", () => {
},
});
expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8");
- expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8");
- expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19");
- expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(0);
expect(result.speech.sttLanguages).toEqual({
dictation: "en",
voice: "en",
});
});
- test("resolves feature-scoped local speech settings", () => {
+ test("resolves dictation settings without reading manual voice configuration", () => {
const persisted = PersistedConfigSchema.parse({
features: {
- voiceMode: {
- turnDetection: { provider: "local" },
- stt: { provider: "openai", model: "gpt-4o-transcribe" },
- },
- },
- providers: {
- openai: { apiKey: "persisted-key" },
+ dictation: { stt: { provider: "local", language: "fr" } },
},
});
const env = {
PASEO_DICTATION_LOCAL_STT_MODEL: "parakeet-tdt-0.6b-v2-int8",
- PASEO_VOICE_LOCAL_STT_MODEL: "parakeet-tdt-0.6b-v2-int8",
- PASEO_VOICE_LOCAL_TTS_MODEL: "kokoro-en-v0_19",
- PASEO_VOICE_LOCAL_TTS_SPEAKER_ID: "5",
- PASEO_VOICE_LOCAL_TTS_SPEED: "1.35",
PASEO_DICTATION_LANGUAGE: "es",
- PASEO_VOICE_LANGUAGE: "pt",
PASEO_LOCAL_MODELS_DIR: "/tmp/models",
- OPENAI_API_KEY: "env-key",
- PASEO_VOICE_STT_PROVIDER: "openai",
PASEO_DICTATION_STT_PROVIDER: "local",
- PASEO_VOICE_TTS_PROVIDER: "local",
} as NodeJS.ProcessEnv;
const result = resolveSpeechConfig({
@@ -96,8 +67,7 @@ describe("resolveSpeechConfig", () => {
dictationStt: "parakeet-tdt-0.6b-v2-int8",
voiceStt: "parakeet-tdt-0.6b-v2-int8",
voiceTts: "kokoro-en-v0_19",
- voiceTtsSpeakerId: 5,
- voiceTtsSpeed: 1.35,
+ voiceTtsSpeakerId: 0,
},
});
expect(result.speech.providers.dictationStt).toEqual({
@@ -105,71 +75,19 @@ describe("resolveSpeechConfig", () => {
explicit: true,
enabled: true,
});
- expect(result.speech.providers.voiceStt).toEqual({
- provider: "openai",
- explicit: true,
- enabled: true,
- });
- expect(result.speech.providers.voiceTurnDetection).toEqual({
- provider: "local",
- explicit: true,
- enabled: true,
- });
- expect(result.speech.providers.voiceTts).toEqual({
- provider: "local",
- explicit: true,
- enabled: true,
- });
- expect(result.speech.local?.models.dictationStt).toBe("parakeet-tdt-0.6b-v2-int8");
- expect(result.speech.local?.models.voiceStt).toBe("parakeet-tdt-0.6b-v2-int8");
- expect(result.speech.local?.models.voiceTts).toBe("kokoro-en-v0_19");
- expect(result.speech.local?.models.voiceTtsSpeakerId).toBe(5);
- expect(result.speech.local?.models.voiceTtsSpeed).toBe(1.35);
- expect(result.speech.sttLanguages).toEqual({
- dictation: "es",
- voice: "pt",
- });
- expect(result.openai?.stt?.apiKey).toBe("persisted-key");
- expect(result.openai?.tts?.apiKey).toBe("persisted-key");
- expect(result.openai?.stt?.model).toBe("gpt-4o-transcribe");
- });
-
- test("resolves STT language from env, settings, and voice-to-dictation fallback", () => {
- const persisted = PersistedConfigSchema.parse({
- features: {
- dictation: {
- stt: {
- language: "fr",
- },
- },
- voiceMode: {
- stt: {
- language: "de",
- },
- },
- },
- });
-
- const result = resolveSpeechConfig({
- paseoHome: "/tmp/paseo-home",
- env: {
- PASEO_DICTATION_LANGUAGE: "es",
- PASEO_VOICE_LANGUAGE: " ",
- } as NodeJS.ProcessEnv,
- persisted,
- });
-
+ expect(result.speech.providers.voiceStt.enabled).toBe(false);
+ expect(result.speech.providers.voiceTurnDetection.enabled).toBe(false);
+ expect(result.speech.providers.voiceTts.enabled).toBe(false);
expect(result.speech.sttLanguages).toEqual({
dictation: "es",
voice: "es",
});
});
- test("respects disabled dictation and voice mode feature flags", () => {
+ test("respects disabled dictation", () => {
const persisted = PersistedConfigSchema.parse({
features: {
dictation: { enabled: false },
- voiceMode: { enabled: false },
},
});
@@ -184,20 +102,8 @@ describe("resolveSpeechConfig", () => {
explicit: false,
enabled: false,
});
- expect(result.speech.providers.voiceTurnDetection).toEqual({
- provider: "local",
- explicit: false,
- enabled: false,
- });
- expect(result.speech.providers.voiceStt).toEqual({
- provider: "local",
- explicit: false,
- enabled: false,
- });
- expect(result.speech.providers.voiceTts).toEqual({
- provider: "local",
- explicit: false,
- enabled: false,
- });
+ expect(result.speech.providers.voiceTurnDetection.enabled).toBe(false);
+ expect(result.speech.providers.voiceStt.enabled).toBe(false);
+ expect(result.speech.providers.voiceTts.enabled).toBe(false);
});
});
diff --git a/packages/server/src/server/speech/speech-config-resolver.ts b/packages/server/src/server/speech/speech-config-resolver.ts
index 95c78707a5..3f34589afe 100644
--- a/packages/server/src/server/speech/speech-config-resolver.ts
+++ b/packages/server/src/server/speech/speech-config-resolver.ts
@@ -65,12 +65,6 @@ function buildFeatureProviderInputs(params: {
env: NodeJS.ProcessEnv;
persisted: PersistedConfig;
}): Record {
- const voiceModeEnabled = resolveOptionalBooleanFlag(
- firstSpeechDefinedValue([
- params.env.PASEO_VOICE_MODE_ENABLED,
- params.persisted.features?.voiceMode?.enabled,
- ]),
- );
return {
dictationStt: {
configuredValue: firstSpeechDefinedValue([
@@ -85,25 +79,16 @@ function buildFeatureProviderInputs(params: {
),
},
voiceTurnDetection: {
- configuredValue: firstSpeechDefinedValue([
- params.env.PASEO_VOICE_TURN_DETECTION_PROVIDER,
- params.persisted.features?.voiceMode?.turnDetection?.provider,
- ]),
- enabled: voiceModeEnabled,
+ configuredValue: undefined,
+ enabled: false,
},
voiceStt: {
- configuredValue: firstSpeechDefinedValue([
- params.env.PASEO_VOICE_STT_PROVIDER,
- params.persisted.features?.voiceMode?.stt?.provider,
- ]),
- enabled: voiceModeEnabled,
+ configuredValue: undefined,
+ enabled: false,
},
voiceTts: {
- configuredValue: firstSpeechDefinedValue([
- params.env.PASEO_VOICE_TTS_PROVIDER,
- params.persisted.features?.voiceMode?.tts?.provider,
- ]),
- enabled: voiceModeEnabled,
+ configuredValue: undefined,
+ enabled: false,
},
};
}
diff --git a/packages/server/src/server/test-utils/paseo-daemon.ts b/packages/server/src/server/test-utils/paseo-daemon.ts
index 9cdce46865..2ec1b198fe 100644
--- a/packages/server/src/server/test-utils/paseo-daemon.ts
+++ b/packages/server/src/server/test-utils/paseo-daemon.ts
@@ -37,9 +37,7 @@ interface TestPaseoDaemonOptions {
cleanup?: boolean;
openai?: PaseoOpenAIConfig;
speech?: PaseoSpeechConfig;
- voiceLlmProvider?: PaseoDaemonConfig["voiceLlmProvider"];
- voiceLlmProviderExplicit?: boolean;
- voiceLlmModel?: string | null;
+ manualVoice?: PaseoDaemonConfig["manualVoice"];
dictationFinalTimeoutMs?: number;
auth?: PaseoDaemonConfig["auth"];
pushNotificationSender?: PushNotificationSender;
@@ -194,9 +192,7 @@ async function prepareTestDaemonConfig(
trustedProxies: options.trustedProxies,
openai: options.openai,
speech: options.speech,
- voiceLlmProvider: options.voiceLlmProvider ?? null,
- voiceLlmProviderExplicit: options.voiceLlmProviderExplicit ?? false,
- voiceLlmModel: options.voiceLlmModel ?? null,
+ manualVoice: options.manualVoice,
dictationFinalTimeoutMs: options.dictationFinalTimeoutMs,
downloadTokenTtlMs: options.downloadTokenTtlMs,
agentProfiles: options.agentProfiles,
diff --git a/packages/server/src/server/voice-chat/index.test.ts b/packages/server/src/server/voice-chat/index.test.ts
new file mode 100644
index 0000000000..9a0e6b0503
--- /dev/null
+++ b/packages/server/src/server/voice-chat/index.test.ts
@@ -0,0 +1,249 @@
+import { describe, expect, test, vi } from "vitest";
+import type {
+ SessionOutboundMessage,
+ VoiceAppContext,
+ VoiceCallOutboundMessage,
+} from "@getpaseo/protocol/messages";
+import { createVoiceChat, rejectLegacyVoiceMode } from "./index.js";
+import type {
+ VoiceCallProvider,
+ VoiceProviderCall,
+ VoiceProviderTerminal,
+} from "./internal/provider.js";
+
+function deferred() {
+ let resolve: (value: T) => void = () => undefined;
+ const promise = new Promise((resolvePromise) => {
+ resolve = resolvePromise;
+ });
+ return { promise, resolve };
+}
+
+function createProviderCall(input?: {
+ terminal?: ReturnType>;
+ onStop?: () => void;
+}): VoiceProviderCall {
+ const terminal = input?.terminal ?? deferred();
+ let stopped = false;
+ return {
+ transport: { kind: "webrtc", answerSdp: "answer-sdp" },
+ closed: terminal.promise,
+ async updateContext() {},
+ async handleTransportMessage() {},
+ async stop() {
+ if (stopped) return;
+ stopped = true;
+ input?.onStop?.();
+ terminal.resolve({ type: "closed" });
+ },
+ };
+}
+
+function startRequest(requestId = "start-1") {
+ return {
+ type: "voice.call.start.request" as const,
+ requestId,
+ context: { workspaceId: "workspace-1", agentId: null },
+ transports: [{ kind: "webrtc" as const, offerSdp: "offer-sdp" }],
+ };
+}
+
+function createClient(provider: VoiceCallProvider) {
+ const messages: VoiceCallOutboundMessage[] = [];
+ const voice = createVoiceChat({ provider });
+ const client = voice.openClientSession({
+ emit: (message) => messages.push(message),
+ });
+ return { voice, client, messages };
+}
+
+function acceptedCallId(messages: VoiceCallOutboundMessage[], requestId = "start-1"): string {
+ const response = messages.find(
+ (message) =>
+ message.type === "voice.call.start.response" && message.payload.requestId === requestId,
+ );
+ if (response?.type !== "voice.call.start.response" || !response.payload.callId) {
+ throw new Error(`Expected accepted start ${requestId}`);
+ }
+ return response.payload.callId;
+}
+
+describe("voice call provider boundary", () => {
+ test("runs a WebRTC-shaped provider through the provider-neutral core", async () => {
+ const starts: VoiceAppContext[] = [];
+ const contexts: VoiceAppContext[] = [];
+ const transportMessages: unknown[] = [];
+ let stops = 0;
+ const provider: VoiceCallProvider = {
+ getReadiness: () => ({ ready: true }),
+ async start(input) {
+ starts.push(input.context);
+ input.emit({
+ type: "event",
+ event: { type: "notice", id: "provider-ready", text: "Provider ready" },
+ });
+ input.emit({ type: "transport", data: { type: "candidate", value: "server-data" } });
+ const call = createProviderCall({ onStop: () => (stops += 1) });
+ return {
+ ...call,
+ async handleTransportMessage(data) {
+ transportMessages.push(data);
+ },
+ async updateContext(context) {
+ contexts.push(context);
+ },
+ };
+ },
+ };
+ const { client, messages } = createClient(provider);
+ await client.handle(startRequest());
+ const callId = acceptedCallId(messages);
+
+ await client.handle({
+ type: "voice.call.transport.client",
+ callId,
+ data: { type: "candidate", value: "client-data" },
+ });
+ await client.handle({
+ type: "voice.call.context.update.request",
+ requestId: "context-1",
+ callId,
+ context: { workspaceId: "workspace-2", agentId: "agent-2" },
+ });
+ await client.handle({ type: "voice.call.stop.request", requestId: "stop-1", callId });
+
+ expect(starts).toEqual([{ workspaceId: "workspace-1", agentId: null }]);
+ expect(contexts).toEqual([{ workspaceId: "workspace-2", agentId: "agent-2" }]);
+ expect(transportMessages).toEqual([{ type: "candidate", value: "client-data" }]);
+ expect(stops).toBe(1);
+ expect(messages).toContainEqual(
+ expect.objectContaining({
+ type: "voice.call.event",
+ payload: expect.objectContaining({
+ event: expect.objectContaining({ id: "provider-ready" }),
+ }),
+ }),
+ );
+ expect(messages).toContainEqual({
+ type: "voice.call.transport.server",
+ payload: { callId, data: { type: "candidate", value: "server-data" } },
+ });
+ });
+
+ test("reserves startup before awaiting the provider", async () => {
+ const gate = deferred();
+ const start = vi.fn(async () => {
+ await gate.promise;
+ return createProviderCall();
+ });
+ const { client, messages } = createClient({ getReadiness: () => ({ ready: true }), start });
+
+ const first = client.handle(startRequest("start-1"));
+ await client.handle(startRequest("start-2"));
+ gate.resolve();
+ await first;
+
+ expect(start).toHaveBeenCalledTimes(1);
+ expect(messages).toContainEqual({
+ type: "voice.call.start.response",
+ payload: {
+ requestId: "start-2",
+ accepted: false,
+ callId: null,
+ error: "A voice call is already active.",
+ },
+ });
+ });
+
+ test("disposes a provider call that finishes starting after client close", async () => {
+ const gate = deferred();
+ let sawAbort = false;
+ let stops = 0;
+ const provider: VoiceCallProvider = {
+ getReadiness: () => ({ ready: true }),
+ async start(input) {
+ await gate.promise;
+ sawAbort = input.signal.aborted;
+ return createProviderCall({ onStop: () => (stops += 1) });
+ },
+ };
+ const { client } = createClient(provider);
+
+ const starting = client.handle(startRequest());
+ const closing = client.disconnect();
+ gate.resolve();
+ await Promise.all([starting, closing]);
+
+ expect(sawAbort).toBe(true);
+ expect(stops).toBe(1);
+ });
+
+ test.each([
+ [{ type: "closed" } as const, "ended", null],
+ [{ type: "failed", error: "upstream expired" } as const, "failed", "upstream expired"],
+ ])("ends the call when the provider signals %s", async (terminal, lifecycle, error) => {
+ const providerTerminal = deferred();
+ const provider: VoiceCallProvider = {
+ getReadiness: () => ({ ready: true }),
+ async start() {
+ return createProviderCall({ terminal: providerTerminal });
+ },
+ };
+ const { client, messages } = createClient(provider);
+ await client.handle(startRequest());
+ const callId = acceptedCallId(messages);
+
+ providerTerminal.resolve(terminal);
+ await vi.waitFor(() => {
+ expect(messages).toContainEqual({
+ type: "voice.call.state",
+ payload: { callId, lifecycle, input: "idle", error },
+ });
+ });
+
+ await client.handle(startRequest("start-2"));
+ expect(acceptedCallId(messages, "start-2")).toBeTruthy();
+ });
+
+ test("cleanup is idempotent across concurrent stop and close", async () => {
+ let stops = 0;
+ const provider: VoiceCallProvider = {
+ getReadiness: () => ({ ready: true }),
+ async start() {
+ return createProviderCall({ onStop: () => (stops += 1) });
+ },
+ };
+ const { client, messages } = createClient(provider);
+ await client.handle(startRequest());
+ const callId = acceptedCallId(messages);
+
+ await Promise.all([
+ client.handle({ type: "voice.call.stop.request", requestId: "stop-1", callId }),
+ client.disconnect(),
+ client.disconnect(),
+ ]);
+
+ expect(stops).toBe(1);
+ await client.handle(startRequest("start-2"));
+ expect(acceptedCallId(messages, "start-2")).toBeTruthy();
+ });
+});
+
+describe("legacy voice mode tombstone", () => {
+ test("rejects without starting a compatibility path", () => {
+ const messages: SessionOutboundMessage[] = [];
+ rejectLegacyVoiceMode({ requestId: "request-1", emit: (message) => messages.push(message) });
+ expect(messages).toEqual([
+ {
+ type: "set_voice_mode_response",
+ payload: {
+ requestId: "request-1",
+ enabled: false,
+ agentId: null,
+ accepted: false,
+ error: "Update Paseo to use voice chat.",
+ },
+ },
+ ]);
+ });
+});
diff --git a/packages/server/src/server/voice-chat/index.ts b/packages/server/src/server/voice-chat/index.ts
new file mode 100644
index 0000000000..8f53bb8152
--- /dev/null
+++ b/packages/server/src/server/voice-chat/index.ts
@@ -0,0 +1,421 @@
+import { v4 as uuidv4 } from "uuid";
+import type {
+ SessionOutboundMessage,
+ VoiceAppContext,
+ VoiceCallInboundMessage,
+ VoiceCallOutboundMessage,
+ VoiceCallState,
+ VoiceCallTransportOffer,
+} from "@getpaseo/protocol/messages";
+import { getErrorMessage } from "@getpaseo/protocol/error-utils";
+import type {
+ VoiceCallProvider,
+ VoiceProviderCall,
+ VoiceProviderOutput,
+ VoiceProviderTerminal,
+} from "./internal/provider.js";
+
+const UPDATE_APP_MESSAGE = "Update Paseo to use voice chat.";
+const CALL_ENDED_MESSAGE = "Voice call is no longer active.";
+
+export interface VoiceChatClientSession {
+ handle(message: VoiceCallInboundMessage): Promise;
+ disconnect(): Promise;
+ close(): Promise;
+}
+
+export interface VoiceChat {
+ openClientSession(input: {
+ emit(message: VoiceCallOutboundMessage): void;
+ }): VoiceChatClientSession;
+ close(): Promise;
+}
+
+export interface CreateVoiceChatOptions {
+ provider: VoiceCallProvider | null;
+}
+
+interface ActiveCall {
+ callId: string;
+ context: VoiceAppContext;
+ providerCall: VoiceProviderCall;
+ state: VoiceCallState;
+}
+
+interface StartingCall {
+ kind: "starting";
+ callId: string;
+ controller: AbortController;
+ completion: Promise;
+}
+
+interface ActiveCallLifecycle {
+ kind: "active";
+ call: ActiveCall;
+}
+
+interface StoppingCall {
+ kind: "stopping";
+ call: ActiveCall;
+ completion: Promise;
+}
+
+type CallLifecycle = { kind: "idle" } | StartingCall | ActiveCallLifecycle | StoppingCall;
+
+export function createVoiceChat(options: CreateVoiceChatOptions): VoiceChat {
+ const clients = new Set();
+
+ function openClientSession(input: {
+ emit(message: VoiceCallOutboundMessage): void;
+ }): VoiceChatClientSession {
+ const client = createClientSession({ options, input });
+ clients.add(client);
+ return {
+ handle: client.handle,
+ disconnect: client.disconnect,
+ async close() {
+ clients.delete(client);
+ await client.close();
+ },
+ };
+ }
+
+ return {
+ openClientSession,
+ async close() {
+ const activeClients = [...clients];
+ clients.clear();
+ await Promise.all(activeClients.map((client) => client.close()));
+ },
+ };
+}
+
+function createClientSession(params: {
+ options: CreateVoiceChatOptions;
+ input: {
+ emit(message: VoiceCallOutboundMessage): void;
+ };
+}): VoiceChatClientSession {
+ const { options, input } = params;
+ let lifecycle: CallLifecycle = { kind: "idle" };
+
+ function isCurrentCall(call: ActiveCall): boolean {
+ return lifecycle.kind === "active" && lifecycle.call === call;
+ }
+
+ function emitState(call: ActiveCall, patch: Partial): void {
+ call.state = { ...call.state, ...patch };
+ input.emit({ type: "voice.call.state", payload: call.state });
+ }
+
+ function emitProviderOutput(call: ActiveCall, output: VoiceProviderOutput): void {
+ if (!isCurrentCall(call)) return;
+ if (output.type === "state") {
+ emitState(call, {
+ ...(output.input ? { input: output.input } : {}),
+ ...(output.error !== undefined ? { error: output.error } : {}),
+ });
+ return;
+ }
+ if (output.type === "event") {
+ input.emit({
+ type: "voice.call.event",
+ payload: { callId: call.callId, event: output.event },
+ });
+ return;
+ }
+ input.emit({
+ type: "voice.call.transport.server",
+ payload: { callId: call.callId, data: output.data },
+ });
+ }
+
+ function emitStartRejected(requestId: string, error: string): void {
+ input.emit({
+ type: "voice.call.start.response",
+ payload: { requestId, accepted: false, callId: null, error },
+ });
+ }
+
+ function validateStart(
+ message: Extract,
+ ): { provider: VoiceCallProvider; transportOffers: VoiceCallTransportOffer[] } | null {
+ if (lifecycle.kind !== "idle") {
+ emitStartRejected(message.requestId, "A voice call is already active.");
+ return null;
+ }
+ const provider = options.provider;
+ if (!provider) {
+ emitStartRejected(message.requestId, "Configure a voice call provider to use voice chat.");
+ return null;
+ }
+ const readiness = provider.getReadiness();
+ if (!readiness.ready) {
+ emitStartRejected(message.requestId, readiness.reason);
+ return null;
+ }
+ if (!message.transports || message.transports.length === 0) {
+ emitStartRejected(message.requestId, UPDATE_APP_MESSAGE);
+ return null;
+ }
+ return { provider, transportOffers: message.transports };
+ }
+
+ async function finishStart(inputParams: {
+ provider: VoiceCallProvider;
+ transportOffers: VoiceCallTransportOffer[];
+ message: Extract;
+ starting: StartingCall;
+ }): Promise {
+ const { provider, transportOffers, message, starting } = inputParams;
+ const pendingOutputs: VoiceProviderOutput[] = [];
+ let call: ActiveCall | null = null;
+ try {
+ const providerCall = await provider.start({
+ callId: starting.callId,
+ context: message.context,
+ transportOffers,
+ signal: starting.controller.signal,
+ emit(output) {
+ if (call) emitProviderOutput(call, output);
+ else pendingOutputs.push(output);
+ },
+ });
+ const selectedWasOffered = transportOffers.some(
+ (offer) => offer.kind === providerCall.transport.kind,
+ );
+ const startWasCanceled = starting.controller.signal.aborted || lifecycle !== starting;
+ if (!selectedWasOffered || startWasCanceled) {
+ await providerCall.stop();
+ if (!selectedWasOffered) {
+ throw new Error(
+ `Voice provider selected an unoffered '${providerCall.transport.kind}' transport`,
+ );
+ }
+ throw new Error("Voice call start was canceled.");
+ }
+
+ call = {
+ callId: starting.callId,
+ context: message.context,
+ providerCall,
+ state: {
+ callId: starting.callId,
+ lifecycle: "active",
+ input: "idle",
+ error: null,
+ },
+ };
+ lifecycle = { kind: "active", call };
+ input.emit({
+ type: "voice.call.start.response",
+ payload: {
+ requestId: message.requestId,
+ accepted: true,
+ callId: call.callId,
+ transport: providerCall.transport,
+ error: null,
+ },
+ });
+ input.emit({ type: "voice.call.state", payload: call.state });
+ for (const output of pendingOutputs) emitProviderOutput(call, output);
+ watchProviderTerminal(call);
+ } catch (error) {
+ if (lifecycle === starting) lifecycle = { kind: "idle" };
+ emitStartRejected(message.requestId, getErrorMessage(error));
+ }
+ }
+
+ async function start(
+ message: Extract,
+ ): Promise {
+ const validated = validateStart(message);
+ if (!validated) return;
+ const starting: StartingCall = {
+ kind: "starting",
+ callId: uuidv4(),
+ controller: new AbortController(),
+ completion: Promise.resolve(),
+ };
+ lifecycle = starting;
+ starting.completion = finishStart({ ...validated, message, starting });
+ await starting.completion;
+ }
+
+ function watchProviderTerminal(call: ActiveCall): void {
+ void call.providerCall.closed.then(
+ (terminal) => finishProviderTerminal(call, terminal),
+ (error) => finishProviderTerminal(call, { type: "failed", error: getErrorMessage(error) }),
+ );
+ }
+
+ async function finishProviderTerminal(
+ call: ActiveCall,
+ terminal: VoiceProviderTerminal,
+ ): Promise {
+ if (lifecycle.kind !== "active" || lifecycle.call !== call) return;
+ const completion = Promise.resolve().then(async () => {
+ try {
+ await call.providerCall.stop();
+ } catch (error) {
+ terminal = { type: "failed", error: getErrorMessage(error) };
+ }
+ finishStoppedCall(call, terminal);
+ return undefined;
+ });
+ lifecycle = { kind: "stopping", call, completion };
+ await completion;
+ }
+
+ async function stopCurrentCall(): Promise {
+ if (lifecycle.kind === "idle") return;
+ if (lifecycle.kind === "starting") {
+ const starting = lifecycle;
+ starting.controller.abort();
+ await starting.completion;
+ return;
+ }
+ if (lifecycle.kind === "stopping") {
+ await lifecycle.completion;
+ return;
+ }
+ const { call } = lifecycle;
+ const completion = Promise.resolve().then(async () => {
+ let terminal: VoiceProviderTerminal = { type: "closed" };
+ try {
+ await call.providerCall.stop();
+ } catch (error) {
+ terminal = { type: "failed", error: getErrorMessage(error) };
+ }
+ finishStoppedCall(call, terminal);
+ return undefined;
+ });
+ lifecycle = { kind: "stopping", call, completion };
+ await completion;
+ }
+
+ function finishStoppedCall(call: ActiveCall, terminal: VoiceProviderTerminal): void {
+ if (lifecycle.kind !== "stopping" || lifecycle.call !== call) return;
+ lifecycle = { kind: "idle" };
+ emitState(
+ call,
+ terminal.type === "failed"
+ ? { lifecycle: "failed", input: "idle", error: terminal.error }
+ : { lifecycle: "ended", input: "idle", error: null },
+ );
+ }
+
+ function activeCallFor(message: VoiceCallInboundMessage): ActiveCall | null {
+ if (!("callId" in message)) return null;
+ if (lifecycle.kind !== "active" || lifecycle.call.callId !== message.callId) return null;
+ return lifecycle.call;
+ }
+
+ async function handle(message: VoiceCallInboundMessage): Promise {
+ if (message.type === "voice.call.start.request") {
+ await start(message);
+ return;
+ }
+ const call = activeCallFor(message);
+ if (!call) {
+ emitInactiveCallResponse(message);
+ return;
+ }
+ if (message.type === "voice.call.transport.client") {
+ await call.providerCall.handleTransportMessage(message.data);
+ return;
+ }
+ if (message.type === "voice.call.context.update.request") {
+ try {
+ await call.providerCall.updateContext(message.context);
+ call.context = message.context;
+ input.emit({
+ type: "voice.call.context.update.response",
+ payload: {
+ requestId: message.requestId,
+ callId: call.callId,
+ accepted: true,
+ error: null,
+ },
+ });
+ } catch (error) {
+ input.emit({
+ type: "voice.call.context.update.response",
+ payload: {
+ requestId: message.requestId,
+ callId: call.callId,
+ accepted: false,
+ error: getErrorMessage(error),
+ },
+ });
+ }
+ return;
+ }
+ try {
+ await stopCurrentCall();
+ input.emit({
+ type: "voice.call.stop.response",
+ payload: {
+ requestId: message.requestId,
+ callId: message.callId,
+ accepted: true,
+ error: null,
+ },
+ });
+ } catch (error) {
+ input.emit({
+ type: "voice.call.stop.response",
+ payload: {
+ requestId: message.requestId,
+ callId: message.callId,
+ accepted: false,
+ error: getErrorMessage(error),
+ },
+ });
+ }
+ }
+
+ function emitInactiveCallResponse(message: VoiceCallInboundMessage): void {
+ if (message.type === "voice.call.stop.request") {
+ input.emit({
+ type: "voice.call.stop.response",
+ payload: {
+ requestId: message.requestId,
+ callId: message.callId,
+ accepted: false,
+ error: CALL_ENDED_MESSAGE,
+ },
+ });
+ }
+ if (message.type === "voice.call.context.update.request") {
+ input.emit({
+ type: "voice.call.context.update.response",
+ payload: {
+ requestId: message.requestId,
+ callId: message.callId,
+ accepted: false,
+ error: CALL_ENDED_MESSAGE,
+ },
+ });
+ }
+ }
+
+ return { handle, disconnect: stopCurrentCall, close: stopCurrentCall };
+}
+
+export function rejectLegacyVoiceMode(input: {
+ requestId: string | undefined;
+ emit(message: SessionOutboundMessage): void;
+}): void {
+ if (!input.requestId) return;
+ input.emit({
+ type: "set_voice_mode_response",
+ payload: {
+ requestId: input.requestId,
+ enabled: false,
+ agentId: null,
+ accepted: false,
+ error: UPDATE_APP_MESSAGE,
+ },
+ });
+}
diff --git a/packages/server/src/server/voice-permission-policy.test.ts b/packages/server/src/server/voice-chat/internal/permission.test.ts
similarity index 52%
rename from packages/server/src/server/voice-permission-policy.test.ts
rename to packages/server/src/server/voice-chat/internal/permission.test.ts
index 73476c5829..176add1cf2 100644
--- a/packages/server/src/server/voice-permission-policy.test.ts
+++ b/packages/server/src/server/voice-chat/internal/permission.test.ts
@@ -1,7 +1,7 @@
import { describe, expect, test } from "vitest";
-import type { AgentPermissionRequest } from "./agent/agent-sdk-types.js";
-import { isVoicePermissionAllowed } from "./voice-permission-policy.js";
+import type { AgentPermissionRequest } from "../../agent/agent-sdk-types.js";
+import { isManualVoicePermissionAllowed } from "../providers/manual/permission.js";
function buildRequest(partial: Partial): AgentPermissionRequest {
return {
@@ -13,30 +13,34 @@ function buildRequest(partial: Partial): AgentPermission
};
}
-describe("isVoicePermissionAllowed", () => {
+describe("isManualVoicePermissionAllowed", () => {
test("allows direct speak tool names across provider conventions", () => {
- const result = isVoicePermissionAllowed(buildRequest({ name: "speak" }));
+ const result = isManualVoicePermissionAllowed(buildRequest({ name: "speak" }));
expect(result).toBe(true);
- expect(isVoicePermissionAllowed(buildRequest({ name: "paseo_voice.speak" }))).toBe(true);
- expect(isVoicePermissionAllowed(buildRequest({ name: "mcp__paseo_voice__speak" }))).toBe(true);
+ expect(isManualVoicePermissionAllowed(buildRequest({ name: "paseo_voice.speak" }))).toBe(true);
+ expect(isManualVoicePermissionAllowed(buildRequest({ name: "mcp__paseo_voice__speak" }))).toBe(
+ true,
+ );
});
test("denies non-speak tool names", () => {
- expect(isVoicePermissionAllowed(buildRequest({ name: "mcp__paseo__create_agent" }))).toBe(
+ expect(isManualVoicePermissionAllowed(buildRequest({ name: "mcp__paseo__create_agent" }))).toBe(
+ false,
+ );
+ expect(isManualVoicePermissionAllowed(buildRequest({ name: "paseo_create_agent" }))).toBe(
false,
);
- expect(isVoicePermissionAllowed(buildRequest({ name: "paseo_create_agent" }))).toBe(false);
});
test("denies non-tool permission kinds", () => {
- const result = isVoicePermissionAllowed(
+ const result = isManualVoicePermissionAllowed(
buildRequest({ kind: "mode", name: "mcp__paseo__create_agent" }),
);
expect(result).toBe(false);
});
test("denies wrapper tools even when metadata references speak", () => {
- const denied = isVoicePermissionAllowed(
+ const denied = isManualVoicePermissionAllowed(
buildRequest({
name: "codextool",
metadata: {
diff --git a/packages/server/src/server/voice-chat/internal/prompt.test.ts b/packages/server/src/server/voice-chat/internal/prompt.test.ts
new file mode 100644
index 0000000000..073e14456b
--- /dev/null
+++ b/packages/server/src/server/voice-chat/internal/prompt.test.ts
@@ -0,0 +1,20 @@
+import { describe, expect, test } from "vitest";
+
+import { buildManualVoiceSystemPrompt, wrapSpokenInput } from "../providers/manual/prompt.js";
+
+describe("manual voice prompt", () => {
+ test("requires user-facing output to use the provider-owned speak tool", () => {
+ const prompt = buildManualVoiceSystemPrompt();
+
+ expect(prompt).toContain("Paseo voice orchestrator");
+ expect(prompt).toContain("Use the Paseo tools as your primary way");
+ expect(prompt).toContain("delegate to an appropriate agent by default");
+ expect(prompt).toContain("Always use the speak tool for all user-facing communication.");
+ });
+
+ test("marks transcribed input without changing the conversation owner", () => {
+ expect(wrapSpokenInput("Please check the build")).toContain(
+ "\nPlease check the build\n",
+ );
+ });
+});
diff --git a/packages/server/src/server/voice-chat/internal/provider.ts b/packages/server/src/server/voice-chat/internal/provider.ts
new file mode 100644
index 0000000000..069854024d
--- /dev/null
+++ b/packages/server/src/server/voice-chat/internal/provider.ts
@@ -0,0 +1,40 @@
+import type {
+ VoiceAppContext,
+ VoiceCallEvent,
+ VoiceCallTransport,
+ VoiceCallTransportOffer,
+} from "@getpaseo/protocol/messages";
+
+export type VoiceProviderReadiness = { ready: true } | { ready: false; reason: string };
+
+export type VoiceProviderOutput =
+ | {
+ type: "state";
+ input?: "idle" | "listening" | "speech_detected";
+ error?: string | null;
+ }
+ | { type: "event"; event: VoiceCallEvent }
+ | { type: "transport"; data: unknown };
+
+export type VoiceProviderTerminal = { type: "closed" } | { type: "failed"; error: string };
+
+export interface VoiceProviderCall {
+ readonly transport: VoiceCallTransport;
+ readonly closed: Promise;
+ updateContext(context: VoiceAppContext): Promise;
+ handleTransportMessage(data: unknown): Promise;
+ stop(): Promise;
+}
+
+export interface VoiceProviderStartInput {
+ callId: string;
+ context: VoiceAppContext;
+ transportOffers: readonly VoiceCallTransportOffer[];
+ signal: AbortSignal;
+ emit(output: VoiceProviderOutput): void;
+}
+
+export interface VoiceCallProvider {
+ getReadiness(): VoiceProviderReadiness;
+ start(input: VoiceProviderStartInput): Promise;
+}
diff --git a/packages/server/src/server/voice-chat/providers/manual/config.test.ts b/packages/server/src/server/voice-chat/providers/manual/config.test.ts
new file mode 100644
index 0000000000..77ff0e2830
--- /dev/null
+++ b/packages/server/src/server/voice-chat/providers/manual/config.test.ts
@@ -0,0 +1,28 @@
+import { describe, expect, it } from "vitest";
+import { resolveManualVoiceOrchestratorConfig } from "./config.js";
+
+describe("manual voice orchestrator configuration", () => {
+ it("uses every configured value exactly", () => {
+ expect(
+ resolveManualVoiceOrchestratorConfig({
+ provider: "codex",
+ model: "gpt-exact",
+ modeId: "full-access",
+ thinkingOptionId: "high",
+ }),
+ ).toEqual({
+ provider: "codex",
+ model: "gpt-exact",
+ modeId: "full-access",
+ thinkingOptionId: "high",
+ });
+ });
+
+ it.each([
+ { provider: null, model: null, modeId: null, thinkingOptionId: null },
+ { provider: "codex", model: "gpt-exact", modeId: null, thinkingOptionId: "high" },
+ { provider: null, model: null, modeId: "full-access", thinkingOptionId: null },
+ ])("rejects missing or partial required selection", (settings) => {
+ expect(resolveManualVoiceOrchestratorConfig(settings)).toBeNull();
+ });
+});
diff --git a/packages/server/src/server/voice-chat/providers/manual/config.ts b/packages/server/src/server/voice-chat/providers/manual/config.ts
new file mode 100644
index 0000000000..63e9689de7
--- /dev/null
+++ b/packages/server/src/server/voice-chat/providers/manual/config.ts
@@ -0,0 +1,254 @@
+import path from "node:path";
+import { z } from "zod";
+import type { AgentSessionConfig } from "../../../agent/agent-sdk-types.js";
+import type { OpenAiSpeechProviderConfig } from "../../../speech/providers/openai/config.js";
+import type { LocalSpeechProviderConfig } from "../../../speech/providers/local/config.js";
+import {
+ DEFAULT_LOCAL_STT_MODEL,
+ DEFAULT_LOCAL_TTS_MODEL,
+ LocalSttModelIdSchema,
+ LocalTtsModelIdSchema,
+} from "../../../speech/providers/local/models.js";
+import type { RequestedSpeechProviders } from "../../../speech/speech-types.js";
+
+const SpeechProviderSchema = z.enum(["local", "openai"]);
+const OpenAiVoiceSchema = z.enum(["alloy", "echo", "fable", "onyx", "nova", "shimmer"]);
+const OpenAiTtsModelSchema = z.enum(["tts-1", "tts-1-hd"]);
+
+export const ManualVoicePersistedConfigSchema = z
+ .object({
+ orchestrator: z
+ .object({
+ provider: z.string().optional(),
+ model: z.string().min(1).optional(),
+ modeId: z.string().trim().min(1).optional(),
+ thinkingOptionId: z.string().trim().min(1).optional(),
+ })
+ .strict()
+ .optional(),
+ stt: z
+ .object({
+ provider: SpeechProviderSchema.optional(),
+ model: z.string().min(1).optional(),
+ language: z.string().trim().min(1).optional(),
+ })
+ .strict()
+ .optional(),
+ turnDetection: z.object({ provider: SpeechProviderSchema.optional() }).strict().optional(),
+ tts: z
+ .object({
+ provider: SpeechProviderSchema.optional(),
+ model: OpenAiTtsModelSchema.or(LocalTtsModelIdSchema).optional(),
+ voice: OpenAiVoiceSchema.optional(),
+ speakerId: z.number().int().optional(),
+ speed: z.number().finite().optional(),
+ })
+ .strict()
+ .optional(),
+ })
+ .strict();
+
+export type ManualVoicePersistedConfig = z.infer;
+
+export interface ManualVoiceOrchestratorConfig {
+ provider: AgentSessionConfig["provider"];
+ model?: string;
+ modeId: string;
+ thinkingOptionId?: string;
+}
+
+export interface ManualVoiceOrchestratorSettings {
+ provider: AgentSessionConfig["provider"] | null;
+ model: string | null;
+ modeId: string | null;
+ thinkingOptionId: string | null;
+}
+
+export interface ManualVoiceConfig {
+ orchestrator: ManualVoiceOrchestratorSettings;
+ speech: {
+ providers: RequestedSpeechProviders;
+ sttLanguages: { dictation: string; voice: string };
+ local?: LocalSpeechProviderConfig;
+ };
+ openai?: OpenAiSpeechProviderConfig;
+}
+
+interface ManualVoiceProviderCredentials {
+ openai?: {
+ apiKey?: string;
+ baseUrl?: string;
+ stt?: { apiKey?: string; baseUrl?: string };
+ tts?: { apiKey?: string; baseUrl?: string };
+ };
+ local?: { modelsDir?: string };
+}
+
+interface ManualVoiceResolutionInput {
+ paseoHome: string;
+ persisted: ManualVoicePersistedConfig | undefined;
+ providers: ManualVoiceProviderCredentials | undefined;
+}
+
+interface SelectedManualSpeechProviders {
+ stt: z.infer;
+ turnDetection: z.infer;
+ tts: z.infer;
+}
+
+function selectManualSpeechProviders(
+ persisted: ManualVoicePersistedConfig | undefined,
+): SelectedManualSpeechProviders {
+ return {
+ stt: persisted?.stt?.provider ?? "local",
+ turnDetection: persisted?.turnDetection?.provider ?? "local",
+ tts: persisted?.tts?.provider ?? "local",
+ };
+}
+
+function resolveManualLocalConfig(
+ input: ManualVoiceResolutionInput,
+ selected: SelectedManualSpeechProviders,
+): LocalSpeechProviderConfig | undefined {
+ if (![selected.stt, selected.turnDetection, selected.tts].includes("local")) return undefined;
+
+ const voiceStt = LocalSttModelIdSchema.parse(
+ selected.stt === "local"
+ ? (input.persisted?.stt?.model ?? DEFAULT_LOCAL_STT_MODEL)
+ : DEFAULT_LOCAL_STT_MODEL,
+ );
+ const voiceTts = LocalTtsModelIdSchema.parse(
+ selected.tts === "local"
+ ? (input.persisted?.tts?.model ?? DEFAULT_LOCAL_TTS_MODEL)
+ : DEFAULT_LOCAL_TTS_MODEL,
+ );
+ const models: LocalSpeechProviderConfig["models"] = {
+ dictationStt: DEFAULT_LOCAL_STT_MODEL,
+ voiceStt,
+ voiceTts,
+ };
+ if (input.persisted?.tts?.speakerId !== undefined) {
+ models.voiceTtsSpeakerId = input.persisted.tts.speakerId;
+ } else if (voiceTts === "kokoro-en-v0_19") {
+ models.voiceTtsSpeakerId = 0;
+ }
+ if (input.persisted?.tts?.speed !== undefined) {
+ models.voiceTtsSpeed = input.persisted.tts.speed;
+ }
+ return {
+ modelsDir:
+ input.providers?.local?.modelsDir ?? path.join(input.paseoHome, "models", "local-speech"),
+ models,
+ };
+}
+
+function resolveManualOpenAiStt(
+ input: ManualVoiceResolutionInput,
+ provider: string,
+): NonNullable | undefined {
+ if (provider !== "openai") return undefined;
+ const credentials = input.providers?.openai;
+ const sttApiKey = credentials?.stt?.apiKey ?? credentials?.apiKey;
+ if (!sttApiKey) return undefined;
+ const config: NonNullable = { apiKey: sttApiKey };
+ const baseUrl = credentials?.stt?.baseUrl ?? credentials?.baseUrl;
+ if (baseUrl) config.baseUrl = baseUrl;
+ if (input.persisted?.stt?.model) config.model = input.persisted.stt.model;
+ return config;
+}
+
+function resolveManualOpenAiTts(
+ input: ManualVoiceResolutionInput,
+ provider: string,
+): NonNullable | undefined {
+ if (provider !== "openai") return undefined;
+ const credentials = input.providers?.openai;
+ const apiKey = credentials?.tts?.apiKey ?? credentials?.apiKey;
+ if (!apiKey) return undefined;
+ const config: NonNullable = {
+ apiKey,
+ model: OpenAiTtsModelSchema.parse(input.persisted?.tts?.model ?? "tts-1"),
+ voice: input.persisted?.tts?.voice ?? "alloy",
+ responseFormat: "pcm",
+ };
+ const baseUrl = credentials?.tts?.baseUrl ?? credentials?.baseUrl;
+ if (baseUrl) config.baseUrl = baseUrl;
+ return config;
+}
+
+function resolveManualOpenAiConfig(
+ input: ManualVoiceResolutionInput,
+ selected: SelectedManualSpeechProviders,
+): OpenAiSpeechProviderConfig | undefined {
+ const stt = resolveManualOpenAiStt(input, selected.stt);
+ const tts = resolveManualOpenAiTts(input, selected.tts);
+ if (!stt && !tts) return undefined;
+ return { stt, tts };
+}
+
+function resolveManualSpeechConfig(
+ input: ManualVoiceResolutionInput,
+ selected: SelectedManualSpeechProviders,
+ local: LocalSpeechProviderConfig | undefined,
+): ManualVoiceConfig["speech"] {
+ const persisted = input.persisted;
+ const speech: ManualVoiceConfig["speech"] = {
+ providers: {
+ dictationStt: { provider: "local", explicit: false, enabled: false },
+ voiceTurnDetection: {
+ provider: selected.turnDetection,
+ explicit: persisted?.turnDetection?.provider !== undefined,
+ },
+ voiceStt: {
+ provider: selected.stt,
+ explicit: persisted?.stt?.provider !== undefined,
+ },
+ voiceTts: {
+ provider: selected.tts,
+ explicit: persisted?.tts?.provider !== undefined,
+ },
+ },
+ sttLanguages: { dictation: "en", voice: persisted?.stt?.language ?? "en" },
+ };
+ if (local) speech.local = local;
+ return speech;
+}
+
+function resolveManualOrchestrator(
+ persisted: ManualVoicePersistedConfig | undefined,
+): ManualVoiceOrchestratorSettings {
+ return {
+ provider: persisted?.orchestrator?.provider ?? null,
+ model: persisted?.orchestrator?.model ?? null,
+ modeId: persisted?.orchestrator?.modeId ?? null,
+ thinkingOptionId: persisted?.orchestrator?.thinkingOptionId ?? null,
+ };
+}
+
+export function resolveManualVoiceConfig(input: {
+ paseoHome: string;
+ persisted: ManualVoicePersistedConfig | undefined;
+ providers: ManualVoiceProviderCredentials | undefined;
+}): ManualVoiceConfig {
+ const selected = selectManualSpeechProviders(input.persisted);
+ const local = resolveManualLocalConfig(input, selected);
+ const openai = resolveManualOpenAiConfig(input, selected);
+ const config: ManualVoiceConfig = {
+ orchestrator: resolveManualOrchestrator(input.persisted),
+ speech: resolveManualSpeechConfig(input, selected, local),
+ };
+ if (openai) config.openai = openai;
+ return config;
+}
+
+export function resolveManualVoiceOrchestratorConfig(
+ settings: ManualVoiceOrchestratorSettings,
+): ManualVoiceOrchestratorConfig | null {
+ if (!settings.provider || !settings.modeId) return null;
+ return {
+ provider: settings.provider,
+ modeId: settings.modeId,
+ ...(settings.model ? { model: settings.model } : {}),
+ ...(settings.thinkingOptionId ? { thinkingOptionId: settings.thinkingOptionId } : {}),
+ };
+}
diff --git a/packages/server/src/server/session/voice/voice-session.test.ts b/packages/server/src/server/voice-chat/providers/manual/index.test.ts
similarity index 88%
rename from packages/server/src/server/session/voice/voice-session.test.ts
rename to packages/server/src/server/voice-chat/providers/manual/index.test.ts
index d6232ff4d8..6bb6186db7 100644
--- a/packages/server/src/server/session/voice/voice-session.test.ts
+++ b/packages/server/src/server/voice-chat/providers/manual/index.test.ts
@@ -2,19 +2,19 @@ import { EventEmitter } from "node:events";
import pino from "pino";
import { describe, expect, test, vi } from "vitest";
-import { VoiceSession, type VoiceSessionHost } from "./voice-session.js";
-import type { ManagedAgent } from "../../agent/agent-manager.js";
-import type { SessionOutboundMessage } from "../../messages.js";
+import { ManualVoiceCall, type ManualVoiceCallHost } from "./index.js";
+import type { ManagedAgent } from "../../../agent/agent-manager.js";
+import type { SessionOutboundMessage } from "../../../messages.js";
import type {
SpeechToTextProvider,
StreamingTranscriptionCommittedEvent,
StreamingTranscriptionEvent,
StreamingTranscriptionSession,
-} from "../../speech/speech-provider.js";
+} from "../../../speech/speech-provider.js";
import type {
TurnDetectionProvider,
TurnDetectionSession,
-} from "../../speech/turn-detection-provider.js";
+} from "../../../speech/turn-detection-provider.js";
const VOICE_AGENT_ID = "11111111-1111-4111-8111-111111111111";
@@ -54,7 +54,7 @@ class FakeVoiceSttSession extends EventEmitter implements StreamingTranscription
}
}
-interface FakeVoiceHost extends VoiceSessionHost {
+interface FakeVoiceHost extends ManualVoiceCallHost {
readonly emitted: SessionOutboundMessage[];
readonly spokenInput: Array<{ agentId: string; text: string }>;
}
@@ -70,7 +70,6 @@ function createFakeHost(): FakeVoiceHost {
},
loadAgent: async (agentId) =>
({ id: agentId, config: { systemPrompt: undefined } }) as unknown as ManagedAgent,
- reloadAgentSession: async (agentId) => ({ id: agentId }) as unknown as ManagedAgent,
sendSpokenInput: async (agentId, text) => {
spokenInput.push({ agentId, text });
},
@@ -91,7 +90,7 @@ function createVoiceSession() {
createSession: vi.fn(() => detector),
};
const host = createFakeHost();
- const voiceSession = new VoiceSession({
+ const voiceSession = new ManualVoiceCall({
host,
logger: pino({ level: "silent" }),
sessionId: "voice-session-test",
@@ -109,14 +108,14 @@ async function settle(): Promise {
await Promise.resolve();
}
-describe("VoiceSession streaming transcription", () => {
+describe("manual voice call streaming transcription", () => {
test("surfaces a refused voice-mode agent interruption", async () => {
const { voiceSession, host } = createVoiceSession();
host.interruptAgentIfRunning = vi.fn(async () => {
throw new Error("active run cancellation was not acknowledged");
});
- await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
+ await voiceSession.start(VOICE_AGENT_ID);
await expect(voiceSession.handleAbort()).rejects.toThrow(
"active run cancellation was not acknowledged",
@@ -139,7 +138,7 @@ describe("VoiceSession streaming transcription", () => {
test("delivers the streaming final transcript to the agent exactly once", async () => {
const { voiceSession, detector, sttSession, host } = createVoiceSession();
- await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
+ await voiceSession.start(VOICE_AGENT_ID);
detector.emit("speech_started");
await settle();
detector.emit("speech_stopped");
@@ -178,7 +177,7 @@ describe("VoiceSession streaming transcription", () => {
try {
const { voiceSession, detector, sttSession, host } = createVoiceSession();
- await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
+ await voiceSession.start(VOICE_AGENT_ID);
detector.emit("speech_started");
await settle();
detector.emit("speech_stopped");
@@ -205,7 +204,7 @@ describe("VoiceSession streaming transcription", () => {
test("filters a low-confidence streaming final without submitting to the agent", async () => {
const { voiceSession, detector, sttSession, host } = createVoiceSession();
- await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
+ await voiceSession.start(VOICE_AGENT_ID);
detector.emit("speech_started");
await settle();
detector.emit("speech_stopped");
diff --git a/packages/server/src/server/session/voice/voice-session.ts b/packages/server/src/server/voice-chat/providers/manual/index.ts
similarity index 71%
rename from packages/server/src/server/session/voice/voice-session.ts
rename to packages/server/src/server/voice-chat/providers/manual/index.ts
index f6331dba3f..937ed608da 100644
--- a/packages/server/src/server/session/voice/voice-session.ts
+++ b/packages/server/src/server/voice-chat/providers/manual/index.ts
@@ -2,25 +2,25 @@ import { v4 as uuidv4 } from "uuid";
import { z } from "zod";
import type pino from "pino";
import { getErrorMessage } from "@getpaseo/protocol/error-utils";
-import type { SessionInboundMessage, SessionOutboundMessage } from "../../messages.js";
-import { TTSManager } from "../../agent/tts-manager.js";
-import { STTManager } from "../../agent/stt-manager.js";
-import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech/speech-provider.js";
-import type { TurnDetectionProvider } from "../../speech/turn-detection-provider.js";
-import { maybePersistTtsDebugAudio } from "../../agent/tts-debug.js";
-import { isPaseoDictationDebugEnabled } from "../../agent/recordings-debug.js";
-import {
- DictationStreamManager,
- type DictationStreamOutboundMessage,
-} from "../../dictation/dictation-stream-manager.js";
-import { createVoiceTurnController, type VoiceTurnController } from "./voice-turn-controller.js";
-import { buildVoiceModeSystemPrompt, stripVoiceModeSystemPrompt } from "../../voice-config.js";
-import type { VoiceCallerContext, VoiceSpeakHandler } from "../../voice-types.js";
-import type { ManagedAgent } from "../../agent/agent-manager.js";
-import type { AgentSessionConfig } from "../../agent/agent-sdk-types.js";
-import type { LocalSpeechModelId } from "../../speech/providers/local/models.js";
-import { toResolver, type Resolvable } from "../../speech/provider-resolver.js";
-import type { SpeechReadinessSnapshot, SpeechReadinessState } from "../../speech/speech-runtime.js";
+import type { SessionInboundMessage, SessionOutboundMessage } from "../../../messages.js";
+import { TTSManager } from "./synthesis.js";
+import { STTManager } from "./transcription.js";
+import type {
+ SpeechToTextProvider,
+ TextToSpeechProvider,
+} from "../../../speech/speech-provider.js";
+import type { TurnDetectionProvider } from "../../../speech/turn-detection-provider.js";
+import { maybePersistTtsDebugAudio } from "../../../agent/tts-debug.js";
+import { isPaseoDictationDebugEnabled } from "../../../agent/recordings-debug.js";
+import { createVoiceTurnController, type VoiceTurnController } from "./turn.js";
+import type { VoiceSpeakHandler } from "./speak.js";
+import type { ManagedAgent } from "../../../agent/agent-manager.js";
+import type { LocalSpeechModelId } from "../../../speech/providers/local/models.js";
+import { toResolver, type Resolvable } from "../../../speech/provider-resolver.js";
+import type {
+ SpeechReadinessSnapshot,
+ SpeechReadinessState,
+} from "../../../speech/speech-runtime.js";
const PCM_SAMPLE_RATE = 16000;
const PCM_CHANNELS = 1;
@@ -34,10 +34,6 @@ const AgentIdSchema = z.guid();
type ProcessingPhase = "idle" | "transcribing";
-interface VoiceModeBaseConfig {
- systemPrompt?: string;
-}
-
interface AudioBufferState {
chunks: Buffer[];
format: string;
@@ -64,26 +60,6 @@ interface VoiceFeatureUnavailableContext {
missingModelIds: LocalSpeechModelId[];
}
-interface VoiceFeatureUnavailableResponseMetadata {
- reasonCode?: SpeechReadinessSnapshot["voiceFeature"]["reasonCode"];
- retryable?: boolean;
- missingModelIds?: LocalSpeechModelId[];
-}
-
-class VoiceFeatureUnavailableError extends Error {
- readonly reasonCode: SpeechReadinessSnapshot["voiceFeature"]["reasonCode"];
- readonly retryable: boolean;
- readonly missingModelIds: LocalSpeechModelId[];
-
- constructor(context: VoiceFeatureUnavailableContext) {
- super(context.message);
- this.name = "VoiceFeatureUnavailableError";
- this.reasonCode = context.reasonCode;
- this.retryable = context.retryable;
- this.missingModelIds = [...context.missingModelIds];
- }
-}
-
function convertPCMToWavBuffer(
pcmBuffer: Buffer,
sampleRate: number,
@@ -114,25 +90,21 @@ function convertPCMToWavBuffer(
}
/**
- * The agent-facing operations VoiceSession needs from the session that owns it.
- * VoiceSession owns all voice/audio state; it reaches back through this narrow
+ * The agent-facing operations the manual call needs from its owner.
+ * ManualVoiceCall owns all voice/audio state; it reaches back through this narrow
* seam only to deliver transcripts to the agent, drive TTS playback, and
* abort/inspect the active agent run.
*/
-export interface VoiceSessionHost {
+export interface ManualVoiceCallHost {
emit(msg: SessionOutboundMessage): void;
loadAgent(agentId: string): Promise;
- reloadAgentSession(
- agentId: string,
- overrides: Partial,
- ): Promise;
sendSpokenInput(agentId: string, text: string): Promise;
interruptAgentIfRunning(agentId: string): Promise;
hasActiveAgentRun(agentId: string | null): boolean;
}
-export interface VoiceSessionOptions {
- host: VoiceSessionHost;
+export interface ManualVoiceCallOptions {
+ host: ManualVoiceCallHost;
logger: pino.Logger;
sessionId: string;
sttLanguage?: string;
@@ -144,26 +116,20 @@ export interface VoiceSessionOptions {
voiceBridge?: {
registerVoiceSpeakHandler?: (agentId: string, handler: VoiceSpeakHandler) => void;
unregisterVoiceSpeakHandler?: (agentId: string) => void;
- registerVoiceCallerContext?: (agentId: string, context: VoiceCallerContext) => void;
- unregisterVoiceCallerContext?: (agentId: string) => void;
- };
- dictation?: {
- finalTimeoutMs?: number;
- stt?: Resolvable;
- sttLanguage?: string;
- getSpeechReadiness?: () => SpeechReadinessSnapshot;
};
+ getSpeechReadiness?: () => SpeechReadinessSnapshot;
+ onFatalError?: (error: Error) => void;
}
/**
* Owns the voice half of a client session: speech-to-text/text-to-speech
- * managers, dictation streaming, the barge-in audio-buffering state machine,
+ * managers, the barge-in audio-buffering state machine,
* voice-turn detection, and the MCP voice bridge. The session delegates the
- * voice/dictation/abort message types here and otherwise knows nothing about
+ * voice/abort message types here and otherwise knows nothing about
* audio buffering or processing phases.
*/
-export class VoiceSession {
- private readonly host: VoiceSessionHost;
+export class ManualVoiceCall {
+ private readonly host: ManualVoiceCallHost;
private readonly sessionLogger: pino.Logger;
private readonly sessionId: string;
private readonly sttLanguage: string;
@@ -174,7 +140,6 @@ export class VoiceSession {
private isVoiceMode = false;
private speechInProgress = false;
- private readonly dictationStreamManager: DictationStreamManager;
private readonly resolveVoiceTurnDetection: () => TurnDetectionProvider | null;
private voiceTurnController: VoiceTurnController | null = null;
private voiceInputChunkCount = 0;
@@ -197,19 +162,13 @@ export class VoiceSession {
handler: VoiceSpeakHandler,
) => void;
private readonly unregisterVoiceSpeakHandler?: (agentId: string) => void;
- private readonly registerVoiceCallerContext?: (
- agentId: string,
- context: VoiceCallerContext,
- ) => void;
- private readonly unregisterVoiceCallerContext?: (agentId: string) => void;
private readonly getSpeechReadiness?: () => SpeechReadinessSnapshot;
+ private readonly onFatalError?: (error: Error) => void;
private voiceModeAgentId: string | null = null;
- private voiceModeBaseConfig: VoiceModeBaseConfig | null = null;
- constructor(options: VoiceSessionOptions) {
- const { host, logger, sessionId, sttLanguage, tts, stt, voice, voiceBridge, dictation } =
- options;
+ constructor(options: ManualVoiceCallOptions) {
+ const { host, logger, sessionId, sttLanguage, tts, stt, voice, voiceBridge } = options;
this.host = host;
this.sessionLogger = logger;
this.sessionId = sessionId;
@@ -219,63 +178,13 @@ export class VoiceSession {
this.resolveVoiceTurnDetection = toResolver(voice?.turnDetection ?? null);
this.registerVoiceSpeakHandler = voiceBridge?.registerVoiceSpeakHandler;
this.unregisterVoiceSpeakHandler = voiceBridge?.unregisterVoiceSpeakHandler;
- this.registerVoiceCallerContext = voiceBridge?.registerVoiceCallerContext;
- this.unregisterVoiceCallerContext = voiceBridge?.unregisterVoiceCallerContext;
- this.getSpeechReadiness = dictation?.getSpeechReadiness;
+ this.getSpeechReadiness = options.getSpeechReadiness;
+ this.onFatalError = options.onFatalError;
this.ttsManager = new TTSManager(this.sessionId, this.sessionLogger, tts);
this.sttManager = new STTManager(this.sessionId, this.sessionLogger, stt, {
language: sttLanguage,
});
- this.dictationStreamManager = new DictationStreamManager({
- logger: this.sessionLogger,
- sessionId: this.sessionId,
- emit: (msg) => this.handleDictationManagerMessage(msg),
- stt: dictation?.stt ?? null,
- language: dictation?.sttLanguage,
- finalTimeoutMs: dictation?.finalTimeoutMs,
- });
- }
-
- isActiveForAgent(agentId: string): boolean {
- return this.isVoiceMode && this.voiceModeAgentId === agentId;
- }
-
- handleDictationChunk(params: {
- dictationId: string;
- seq: number;
- audioBase64: string;
- format: string;
- }): Promise {
- return this.dictationStreamManager.handleChunk(params);
- }
-
- handleDictationFinish(dictationId: string, finalSeq: number): Promise {
- return this.dictationStreamManager.handleFinish(dictationId, finalSeq);
- }
-
- handleDictationCancel(dictationId: string): void {
- this.dictationStreamManager.handleCancel(dictationId);
- }
-
- async handleDictationStreamStart(
- msg: Extract,
- ): Promise {
- const unavailable = this.resolveVoiceFeatureUnavailableContext("dictation");
- if (unavailable) {
- this.emit({
- type: "dictation_stream_error",
- payload: {
- dictationId: msg.dictationId,
- error: unavailable.message,
- retryable: unavailable.retryable,
- reasonCode: unavailable.reasonCode,
- missingModelIds: unavailable.missingModelIds,
- },
- });
- return;
- }
- await this.dictationStreamManager.handleStart(msg.dictationId, msg.format);
}
private toVoiceFeatureUnavailableContext(
@@ -289,38 +198,13 @@ export class VoiceSession {
};
}
- private resolveModeReadinessState(
- readiness: SpeechReadinessSnapshot,
- mode: "voice_mode" | "dictation",
- ): SpeechReadinessState {
- if (mode === "voice_mode") {
- return readiness.realtimeVoice;
- }
- return readiness.dictation;
- }
-
- private getVoiceFeatureUnavailableResponseMetadata(
- error: unknown,
- ): VoiceFeatureUnavailableResponseMetadata {
- if (!(error instanceof VoiceFeatureUnavailableError)) {
- return {};
- }
- return {
- reasonCode: error.reasonCode,
- retryable: error.retryable,
- missingModelIds: error.missingModelIds,
- };
- }
-
- private resolveVoiceFeatureUnavailableContext(
- mode: "voice_mode" | "dictation",
- ): VoiceFeatureUnavailableContext | null {
+ private resolveVoiceFeatureUnavailableContext(): VoiceFeatureUnavailableContext | null {
const readiness = this.getSpeechReadiness?.();
if (!readiness) {
return null;
}
- const modeReadiness = this.resolveModeReadinessState(readiness, mode);
+ const modeReadiness = readiness.realtimeVoice;
if (!modeReadiness.enabled) {
return this.toVoiceFeatureUnavailableContext(modeReadiness);
}
@@ -333,134 +217,27 @@ export class VoiceSession {
return null;
}
- /**
- * Handle voice mode toggle
- */
- async handleSetVoiceMode(enabled: boolean, agentId?: string, requestId?: string): Promise {
- const startedAt = Date.now();
+ async start(agentId: string): Promise {
+ if (this.isVoiceMode) return;
+ const unavailable = this.resolveVoiceFeatureUnavailableContext();
+ if (unavailable) throw new Error(unavailable.message);
+ const normalizedAgentId = this.parseVoiceTargetAgentId(agentId, "manual voice call");
+ this.voiceModeAgentId = await this.enableVoiceModeForAgent(normalizedAgentId);
try {
- this.sessionLogger.info(
- { enabled, requestedAgentId: agentId ?? null, requestId: requestId ?? null },
- "set_voice_mode started",
- );
- if (enabled) {
- const unavailable = this.resolveVoiceFeatureUnavailableContext("voice_mode");
- if (unavailable) {
- throw new VoiceFeatureUnavailableError(unavailable);
- }
-
- const normalizedAgentId = this.parseVoiceTargetAgentId(agentId ?? "", "set_voice_mode");
-
- if (
- this.isVoiceMode &&
- this.voiceModeAgentId &&
- this.voiceModeAgentId !== normalizedAgentId
- ) {
- this.sessionLogger.info(
- {
- previousAgentId: this.voiceModeAgentId,
- nextAgentId: normalizedAgentId,
- elapsedMs: Date.now() - startedAt,
- },
- "set_voice_mode disabling previous active voice agent",
- );
- await this.disableVoiceModeForActiveAgent(true);
- }
-
- if (!this.isVoiceMode || this.voiceModeAgentId !== normalizedAgentId) {
- this.sessionLogger.info(
- { agentId: normalizedAgentId, elapsedMs: Date.now() - startedAt },
- "set_voice_mode enabling voice for agent",
- );
- const refreshedAgentId = await this.enableVoiceModeForAgent(normalizedAgentId);
- this.voiceModeAgentId = refreshedAgentId;
- this.sessionLogger.info(
- { agentId: refreshedAgentId, elapsedMs: Date.now() - startedAt },
- "set_voice_mode agent enable complete",
- );
- }
-
- this.sessionLogger.info(
- { agentId: this.voiceModeAgentId, elapsedMs: Date.now() - startedAt },
- "set_voice_mode starting voice turn controller",
- );
- await this.startVoiceTurnController();
- this.sessionLogger.info(
- { agentId: this.voiceModeAgentId, elapsedMs: Date.now() - startedAt },
- "set_voice_mode voice turn controller started",
- );
- this.isVoiceMode = true;
- this.sessionLogger.info(
- {
- agentId: this.voiceModeAgentId,
- elapsedMs: Date.now() - startedAt,
- },
- "Voice mode enabled for existing agent",
- );
- if (requestId) {
- this.emit({
- type: "set_voice_mode_response",
- payload: {
- requestId,
- enabled: true,
- agentId: this.voiceModeAgentId,
- accepted: true,
- error: null,
- },
- });
- }
- return;
- }
-
- this.sessionLogger.info(
- { agentId: this.voiceModeAgentId, elapsedMs: Date.now() - startedAt },
- "set_voice_mode disabling active voice mode",
- );
- await this.disableVoiceModeForActiveAgent(true);
- this.isVoiceMode = false;
- this.sessionLogger.info({ elapsedMs: Date.now() - startedAt }, "Voice mode disabled");
- if (requestId) {
- this.emit({
- type: "set_voice_mode_response",
- payload: {
- requestId,
- enabled: false,
- agentId: null,
- accepted: true,
- error: null,
- },
- });
- }
+ await this.startVoiceTurnController();
+ this.isVoiceMode = true;
} catch (error) {
- const errorMessage = error instanceof Error ? error.message : "Failed to set voice mode";
- const unavailable = this.getVoiceFeatureUnavailableResponseMetadata(error);
- this.sessionLogger.error(
- {
- err: error,
- enabled,
- requestedAgentId: agentId ?? null,
- elapsedMs: Date.now() - startedAt,
- },
- "set_voice_mode failed",
- );
- if (requestId) {
- this.emit({
- type: "set_voice_mode_response",
- payload: {
- requestId,
- enabled: this.isVoiceMode,
- agentId: this.voiceModeAgentId,
- accepted: false,
- error: errorMessage,
- ...unavailable,
- },
- });
- return;
- }
+ await this.disableVoiceModeForActiveAgent();
throw error;
}
}
+ async stop(): Promise {
+ if (!this.isVoiceMode && !this.voiceModeAgentId) return;
+ await this.disableVoiceModeForActiveAgent();
+ this.isVoiceMode = false;
+ }
+
private parseVoiceTargetAgentId(rawId: string, source: string): string {
const parsed = AgentIdSchema.safeParse(rawId.trim());
if (!parsed.success) {
@@ -480,67 +257,21 @@ export class VoiceSession {
this.registerVoiceBridgeForAgent(agentId);
- const baseConfig: VoiceModeBaseConfig = {
- systemPrompt: stripVoiceModeSystemPrompt(existing.config.systemPrompt),
- };
- this.voiceModeBaseConfig = baseConfig;
- const refreshOverrides: Partial = {
- systemPrompt: buildVoiceModeSystemPrompt(baseConfig.systemPrompt, true),
- };
-
- try {
- this.sessionLogger.info(
- { agentId, elapsedMs: Date.now() - startedAt },
- "enableVoiceModeForAgent.reloadAgentSession.start",
- );
- const refreshed = await this.host.reloadAgentSession(agentId, refreshOverrides);
- this.sessionLogger.info(
- { agentId, refreshedAgentId: refreshed.id, elapsedMs: Date.now() - startedAt },
- "enableVoiceModeForAgent.reloadAgentSession.done",
- );
- return refreshed.id;
- } catch (error) {
- this.unregisterVoiceSpeakHandler?.(agentId);
- this.unregisterVoiceCallerContext?.(agentId);
- this.voiceModeBaseConfig = null;
- throw error;
- }
+ return existing.id;
}
- private async disableVoiceModeForActiveAgent(restoreAgentConfig: boolean): Promise {
+ private async disableVoiceModeForActiveAgent(): Promise {
await this.stopVoiceTurnController();
const agentId = this.voiceModeAgentId;
if (!agentId) {
- this.voiceModeBaseConfig = null;
return;
}
this.unregisterVoiceSpeakHandler?.(agentId);
- this.unregisterVoiceCallerContext?.(agentId);
-
- if (restoreAgentConfig && this.voiceModeBaseConfig) {
- const baseConfig = this.voiceModeBaseConfig;
- try {
- await this.host.reloadAgentSession(agentId, {
- systemPrompt: buildVoiceModeSystemPrompt(baseConfig.systemPrompt, false),
- });
- } catch (error) {
- this.sessionLogger.warn(
- { err: error, agentId },
- "Failed to restore agent config while disabling voice mode",
- );
- }
- }
-
- this.voiceModeBaseConfig = null;
this.voiceModeAgentId = null;
}
- private handleDictationManagerMessage(msg: DictationStreamOutboundMessage): void {
- this.emit(msg as unknown as SessionOutboundMessage);
- }
-
private async startVoiceTurnController(): Promise {
if (this.voiceTurnController) {
this.sessionLogger.info("startVoiceTurnController skipped: already running");
@@ -631,6 +362,7 @@ export class VoiceSession {
},
onError: (error) => {
this.sessionLogger.error({ err: error }, "Voice turn controller failed");
+ this.onFatalError?.(error);
},
},
});
@@ -1059,12 +791,6 @@ export class VoiceSession {
},
});
});
-
- this.registerVoiceCallerContext?.(agentId, {
- childAgentDefaultLabels: {},
- allowCustomCwd: false,
- enableVoiceTools: true,
- });
}
/**
@@ -1301,9 +1027,7 @@ export class VoiceSession {
this.ttsManager.cleanup();
this.sttManager.cleanup();
- this.dictationStreamManager.cleanupAll();
-
- await this.disableVoiceModeForActiveAgent(true);
+ await this.disableVoiceModeForActiveAgent();
this.isVoiceMode = false;
}
}
diff --git a/packages/server/src/server/voice-chat/providers/manual/permission.ts b/packages/server/src/server/voice-chat/providers/manual/permission.ts
new file mode 100644
index 0000000000..c092fa6225
--- /dev/null
+++ b/packages/server/src/server/voice-chat/providers/manual/permission.ts
@@ -0,0 +1,8 @@
+import { isSpeakToolName } from "@getpaseo/protocol/tool-name-normalization";
+import type { AgentPermissionRequest } from "../../../agent/agent-sdk-types.js";
+
+export function isManualVoicePermissionAllowed(request: AgentPermissionRequest): boolean {
+ if (request.kind !== "tool") return false;
+ const normalizedName = request.name.trim().toLowerCase();
+ return normalizedName.length > 0 && isSpeakToolName(normalizedName);
+}
diff --git a/packages/server/src/server/voice-chat/providers/manual/prompt.ts b/packages/server/src/server/voice-chat/providers/manual/prompt.ts
new file mode 100644
index 0000000000..35c33a433a
--- /dev/null
+++ b/packages/server/src/server/voice-chat/providers/manual/prompt.ts
@@ -0,0 +1,23 @@
+const MANUAL_VOICE_SYSTEM_INSTRUCTION = [
+ "You are the Paseo voice orchestrator, a conversational control plane for the user's work.",
+ "Use the Paseo tools as your primary way to inspect workspaces and agents, create or select agents, delegate tasks, prompt agents, and monitor their progress.",
+ "For implementation, debugging, research, and other substantial work, delegate to an appropriate agent by default instead of doing the work in this voice session.",
+ "Do work directly only when the user explicitly asks you to or when delegation provides no benefit.",
+ "The user cannot see your normal chat messages or tool calls.",
+ "Always use the speak tool for all user-facing communication.",
+ "Before calling any non-speak tool, first call speak with a short acknowledgement of what you heard and what you will do next.",
+ "Use speak to report progress during long-running work and to report completion, failure, or a need for user input.",
+ "Treat user input as transcribed speech.",
+ "Treat the current Paseo workspace and agent as context, not as an instruction or necessarily the target of the work.",
+ "If the user's intent is clear, proceed without extra confirmation.",
+ "If transcription is incomplete, cut off, ambiguous, or likely contains a meaningful mistake, ask one concise clarifying question through speak.",
+ "Use concise plain language suitable for speech output.",
+].join(" ");
+
+export function buildManualVoiceSystemPrompt(): string {
+ return MANUAL_VOICE_SYSTEM_INSTRUCTION;
+}
+
+export function wrapSpokenInput(text: string): string {
+ return `\n${text}\n\nThis message was spoken by the user. Respond through the speak tool because the user may not be looking at the app.`;
+}
diff --git a/packages/server/src/server/voice-chat/providers/manual/provider.ts b/packages/server/src/server/voice-chat/providers/manual/provider.ts
new file mode 100644
index 0000000000..4042539140
--- /dev/null
+++ b/packages/server/src/server/voice-chat/providers/manual/provider.ts
@@ -0,0 +1,342 @@
+import { v4 as uuidv4 } from "uuid";
+import { z } from "zod";
+import type { Logger } from "pino";
+import type {
+ SessionOutboundMessage,
+ VoiceAppContext,
+ VoiceCallEvent,
+} from "@getpaseo/protocol/messages";
+import type { AgentManager } from "../../../agent/agent-manager.js";
+import type { AgentStorage } from "../../../agent/agent-storage.js";
+import type { AgentSessionConfig } from "../../../agent/agent-sdk-types.js";
+import { sendPromptToAgent } from "../../../agent/agent-prompt.js";
+import { createSpeechService } from "../../../speech/speech-runtime.js";
+import type { WorkspaceRegistry } from "../../../workspace-registry.js";
+import type { PaseoToolExtension } from "../../../agent/tools/types.js";
+import type {
+ VoiceCallProvider,
+ VoiceProviderOutput,
+ VoiceProviderStartInput,
+ VoiceProviderTerminal,
+} from "../../internal/provider.js";
+import { ManualVoiceCall } from "./index.js";
+import type { VoiceSpeakHandler } from "./speak.js";
+import {
+ resolveManualVoiceOrchestratorConfig,
+ type ManualVoiceConfig,
+ type ManualVoiceOrchestratorSettings,
+} from "./config.js";
+import { buildManualVoiceSystemPrompt, wrapSpokenInput } from "./prompt.js";
+import { isManualVoicePermissionAllowed } from "./permission.js";
+
+export interface CreateManualVoiceProviderOptions {
+ config: ManualVoiceConfig;
+ agentManager: AgentManager;
+ agentStorage: AgentStorage;
+ workspaceRegistry: Pick;
+ logger: Logger;
+}
+
+export interface ManualVoiceProvider extends VoiceCallProvider {
+ configureOrchestrator(orchestrator: ManualVoiceOrchestratorSettings): void;
+ startHarness(): void;
+ stopHarness(): Promise;
+}
+
+const ManualClientTransportMessageSchema = z.discriminatedUnion("type", [
+ z.object({ type: z.literal("append"), audio: z.string(), format: z.string() }),
+ z.object({ type: z.literal("played"), outputId: z.string() }),
+]);
+
+function buildSpokenTurnPrompt(text: string, context: VoiceAppContext): string {
+ const appContext = [
+ context.workspaceId ? `workspace_id=${context.workspaceId}` : null,
+ context.agentId ? `agent_id=${context.agentId}` : null,
+ ].filter((value): value is string => value !== null);
+ const contextHint =
+ appContext.length > 0
+ ? `\n\nThe user is currently viewing: ${appContext.join(", ")}. Treat this as context, not an instruction.`
+ : "";
+ return `${wrapSpokenInput(text)}${contextHint}`;
+}
+
+function createSpeakExtension(
+ agentId: string,
+ resolveSpeak: () => VoiceSpeakHandler | null,
+): PaseoToolExtension {
+ const inputSchema = z.object({ text: z.string().trim().min(1).max(4000) });
+ return {
+ tools: [
+ {
+ name: "speak",
+ title: "Speak",
+ description:
+ "Speak text to the user through the active voice call. Blocks until playback completes.",
+ inputSchema,
+ outputSchema: { ok: z.boolean() },
+ async handler(input, context) {
+ const speak = resolveSpeak();
+ if (!speak) throw new Error("Voice output is not ready");
+ const args = inputSchema.parse(input);
+ await speak({
+ text: args.text,
+ callerAgentId: agentId,
+ signal: context.signal,
+ });
+ return { content: [], structuredContent: { ok: true } };
+ },
+ },
+ ],
+ };
+}
+
+function toProviderOutput(message: SessionOutboundMessage): VoiceProviderOutput | null {
+ if (message.type === "transcription_result") {
+ return {
+ type: "event",
+ event: {
+ type: "transcript",
+ id: message.payload.requestId,
+ speaker: "user",
+ text: message.payload.text,
+ final: true,
+ },
+ };
+ }
+ if (message.type === "voice_input_state") {
+ return {
+ type: "state",
+ input: message.payload.isSpeaking ? "speech_detected" : "listening",
+ };
+ }
+ if (message.type === "audio_output") {
+ return {
+ type: "transport",
+ data: {
+ type: "audio",
+ outputId: message.payload.id,
+ audio: message.payload.audio,
+ format: message.payload.format,
+ ...(message.payload.groupId ? { groupId: message.payload.groupId } : {}),
+ ...(message.payload.chunkIndex === undefined
+ ? {}
+ : { chunkIndex: message.payload.chunkIndex }),
+ ...(message.payload.isLastChunk === undefined
+ ? {}
+ : { isLastChunk: message.payload.isLastChunk }),
+ },
+ };
+ }
+ if (message.type !== "activity_log") return null;
+ let event: VoiceCallEvent | null = null;
+ if (message.payload.type === "assistant") {
+ event = {
+ type: "transcript",
+ id: message.payload.id,
+ speaker: "assistant",
+ text: message.payload.content,
+ final: true,
+ };
+ } else if (message.payload.type === "error") {
+ event = { type: "error", id: message.payload.id, message: message.payload.content };
+ } else if (message.payload.type === "system") {
+ event = { type: "notice", id: message.payload.id, text: message.payload.content };
+ }
+ return event ? { type: "event", event } : null;
+}
+
+export function createManualVoiceProvider(
+ options: CreateManualVoiceProviderOptions,
+): ManualVoiceProvider {
+ let orchestratorSettings = options.config.orchestrator;
+ const speech = createSpeechService({
+ logger: options.logger,
+ openaiConfig: options.config.openai,
+ speechConfig: options.config.speech,
+ });
+ return {
+ configureOrchestrator(orchestrator) {
+ orchestratorSettings = orchestrator;
+ },
+ startHarness() {
+ speech.start();
+ },
+ stopHarness() {
+ return speech.stop();
+ },
+ getReadiness() {
+ if (!resolveManualVoiceOrchestratorConfig(orchestratorSettings)) {
+ return { ready: false, reason: "Configure the manual voice orchestrator and mode." };
+ }
+ return { ready: true };
+ },
+ async start(input) {
+ return startManualVoiceCall(options, speech, orchestratorSettings, input);
+ },
+ };
+}
+
+async function startManualVoiceCall(
+ options: CreateManualVoiceProviderOptions,
+ speech: ReturnType,
+ orchestratorSettings: ManualVoiceOrchestratorSettings,
+ input: VoiceProviderStartInput,
+) {
+ if (!input.transportOffers.some((transport) => transport.kind === "daemon-audio")) {
+ throw new Error("The client does not support daemon audio for manual voice chat");
+ }
+ const contextAgent = input.context.agentId
+ ? options.agentManager.getAgent(input.context.agentId)
+ : null;
+ const workspaceId = input.context.workspaceId ?? contextAgent?.workspaceId ?? null;
+ if (!workspaceId) throw new Error("Open a workspace before starting voice chat.");
+ const workspace = await options.workspaceRegistry.get(workspaceId);
+ if (!workspace || workspace.archivedAt) throw new Error("The selected workspace is unavailable.");
+ const orchestrator = resolveManualVoiceOrchestratorConfig(orchestratorSettings);
+ if (!orchestrator) throw new Error("Configure the manual voice orchestrator and mode.");
+ if (input.signal.aborted) throw new Error("Voice call start was canceled.");
+
+ const agentId = uuidv4();
+ let context = input.context;
+ let speak: VoiceSpeakHandler | null = null;
+ const config: AgentSessionConfig = {
+ provider: orchestrator.provider,
+ cwd: workspace.cwd,
+ internal: true,
+ systemPrompt: buildManualVoiceSystemPrompt(),
+ modeId: orchestrator.modeId,
+ ...(orchestrator.model ? { model: orchestrator.model } : {}),
+ ...(orchestrator.thinkingOptionId ? { thinkingOptionId: orchestrator.thinkingOptionId } : {}),
+ };
+ let manual: ManualVoiceCall | null = null;
+ let unsubscribeAgent: (() => void) | null = null;
+ let internalAgent: Awaited<
+ ReturnType
+ > | null = null;
+ let terminal: VoiceProviderTerminal | null = null;
+ let resolveClosed: (terminal: VoiceProviderTerminal) => void = () => undefined;
+ const closed = new Promise((resolve) => {
+ resolveClosed = resolve;
+ });
+ function settle(next: VoiceProviderTerminal): void {
+ if (terminal) return;
+ terminal = next;
+ resolveClosed(next);
+ }
+ try {
+ internalAgent = await options.agentManager.launchInternalAgentWithRequiredPaseoTools({
+ agentId,
+ config,
+ workspaceId: workspace.workspaceId,
+ labels: { surface: "voice" },
+ initialTitle: "Voice chat",
+ tools: {
+ caller: { childAgentDefaultLabels: {}, allowCustomCwd: false },
+ extension: createSpeakExtension(agentId, () => speak),
+ },
+ systemPromptBehavior: "authoritative",
+ });
+ const agent = internalAgent.agent;
+ if (input.signal.aborted) throw new Error("Voice call start was canceled.");
+ manual = new ManualVoiceCall({
+ host: {
+ emit(message) {
+ const output = toProviderOutput(message);
+ if (output) input.emit(output);
+ },
+ async loadAgent() {
+ const loaded = options.agentManager.getAgent(agent.id);
+ if (!loaded) throw new Error(`Voice agent ${agent.id} is unavailable`);
+ return loaded;
+ },
+ async sendSpokenInput(id, text) {
+ await sendPromptToAgent({
+ agentManager: options.agentManager,
+ agentStorage: options.agentStorage,
+ agentId: id,
+ prompt: buildSpokenTurnPrompt(text, context),
+ clearPendingPermissions: true,
+ logger: options.logger,
+ });
+ },
+ async interruptAgentIfRunning(id) {
+ await options.agentManager.cancelAgentRun(id);
+ },
+ hasActiveAgentRun: (id) => (id ? options.agentManager.hasInFlightRun(id) : false),
+ },
+ logger: options.logger.child({ module: "voice-chat", provider: "manual" }),
+ sessionId: input.callId,
+ sttLanguage: speech.resolveSttLanguage(),
+ tts: speech.resolveTts,
+ stt: speech.resolveStt,
+ voice: { turnDetection: speech.resolveTurnDetection },
+ onFatalError(error) {
+ settle({ type: "failed", error: error.message });
+ },
+ voiceBridge: {
+ registerVoiceSpeakHandler(_id, handler) {
+ speak = handler;
+ },
+ unregisterVoiceSpeakHandler() {
+ speak = null;
+ },
+ },
+ getSpeechReadiness: speech.getReadiness,
+ });
+ unsubscribeAgent = options.agentManager.subscribe(
+ (event) => {
+ if (event.type !== "agent_stream" || event.event.type !== "permission_requested") return;
+ if (!isManualVoicePermissionAllowed(event.event.request)) return;
+ void options.agentManager
+ .respondToPermission(agent.id, event.event.request.id, { behavior: "allow" })
+ .catch((error) =>
+ options.logger.warn({ err: error }, "Failed to allow voice speak tool"),
+ );
+ },
+ { agentId: agent.id, replayState: false },
+ );
+ await manual.start(agent.id);
+ if (input.signal.aborted) throw new Error("Voice call start was canceled.");
+ input.emit({ type: "state", input: "listening" });
+
+ let stopped = false;
+
+ async function stop(): Promise {
+ if (stopped) return;
+ unsubscribeAgent?.();
+ unsubscribeAgent = null;
+ await manual?.stop();
+ await manual?.cleanup();
+ await internalAgent?.dispose();
+ stopped = true;
+ settle({ type: "closed" });
+ }
+
+ return {
+ transport: { kind: "daemon-audio" as const },
+ closed,
+ async handleTransportMessage(data: unknown) {
+ const message = ManualClientTransportMessageSchema.parse(data);
+ if (message.type === "append") {
+ await manual?.handleAudioChunk({
+ type: "voice_audio_chunk",
+ audio: message.audio,
+ format: message.format,
+ isLast: false,
+ });
+ return;
+ }
+ manual?.handleAudioPlayed(message.outputId);
+ },
+ async updateContext(nextContext: VoiceAppContext) {
+ context = nextContext;
+ },
+ stop,
+ };
+ } catch (error) {
+ unsubscribeAgent?.();
+ await manual?.cleanup().catch(() => undefined);
+ await internalAgent?.dispose().catch(() => undefined);
+ throw error;
+ }
+}
diff --git a/packages/server/src/server/voice-chat/providers/manual/speak.ts b/packages/server/src/server/voice-chat/providers/manual/speak.ts
new file mode 100644
index 0000000000..cdd8081be9
--- /dev/null
+++ b/packages/server/src/server/voice-chat/providers/manual/speak.ts
@@ -0,0 +1,5 @@
+export type VoiceSpeakHandler = (params: {
+ text: string;
+ callerAgentId: string;
+ signal?: AbortSignal;
+}) => Promise;
diff --git a/packages/server/src/server/agent/tts-manager.test.ts b/packages/server/src/server/voice-chat/providers/manual/synthesis.test.ts
similarity index 97%
rename from packages/server/src/server/agent/tts-manager.test.ts
rename to packages/server/src/server/voice-chat/providers/manual/synthesis.test.ts
index 6dfc198306..480ed4efb8 100644
--- a/packages/server/src/server/agent/tts-manager.test.ts
+++ b/packages/server/src/server/voice-chat/providers/manual/synthesis.test.ts
@@ -2,9 +2,9 @@ import { describe, expect, it, vi } from "vitest";
import pino from "pino";
import { Readable } from "node:stream";
-import { TTSManager } from "./tts-manager.js";
-import type { TextToSpeechProvider } from "../speech/speech-provider.js";
-import type { SessionOutboundMessage } from "../messages.js";
+import { TTSManager } from "./synthesis.js";
+import type { TextToSpeechProvider } from "../../../speech/speech-provider.js";
+import type { SessionOutboundMessage } from "../../../messages.js";
type AudioOutputMessage = Extract;
diff --git a/packages/server/src/server/agent/tts-manager.ts b/packages/server/src/server/voice-chat/providers/manual/synthesis.ts
similarity index 98%
rename from packages/server/src/server/agent/tts-manager.ts
rename to packages/server/src/server/voice-chat/providers/manual/synthesis.ts
index ee6c747019..bd59c7b56a 100644
--- a/packages/server/src/server/agent/tts-manager.ts
+++ b/packages/server/src/server/voice-chat/providers/manual/synthesis.ts
@@ -1,9 +1,9 @@
import type pino from "pino";
import type { Readable } from "node:stream";
import { v4 as uuidv4 } from "uuid";
-import type { TextToSpeechProvider } from "../speech/speech-provider.js";
-import { toResolver, type Resolvable } from "../speech/provider-resolver.js";
-import type { SessionOutboundMessage } from "../messages.js";
+import type { TextToSpeechProvider } from "../../../speech/speech-provider.js";
+import { toResolver, type Resolvable } from "../../../speech/provider-resolver.js";
+import type { SessionOutboundMessage } from "../../../messages.js";
interface PendingPlayback {
resolve: () => void;
diff --git a/packages/server/src/server/agent/stt-manager.test.ts b/packages/server/src/server/voice-chat/providers/manual/transcription.test.ts
similarity index 62%
rename from packages/server/src/server/agent/stt-manager.test.ts
rename to packages/server/src/server/voice-chat/providers/manual/transcription.test.ts
index e3f6fb7d9c..09f287418c 100644
--- a/packages/server/src/server/agent/stt-manager.test.ts
+++ b/packages/server/src/server/voice-chat/providers/manual/transcription.test.ts
@@ -2,14 +2,12 @@ import { describe, expect, it } from "vitest";
import pino from "pino";
import { EventEmitter } from "node:events";
-import { STTManager } from "./stt-manager.js";
-import { PersistedConfigSchema } from "../persisted-config.js";
-import { resolveSpeechConfig } from "../speech/speech-config-resolver.js";
+import { STTManager } from "./transcription.js";
import type {
SpeechToTextProvider,
StreamingTranscriptionSession,
TranscriptionResult,
-} from "../speech/speech-provider.js";
+} from "../../../speech/speech-provider.js";
type SessionParams = Parameters[0];
type StreamingOn = StreamingTranscriptionSession["on"];
@@ -96,116 +94,16 @@ class SequencedFakeStt implements SpeechToTextProvider {
}
describe("STTManager", () => {
- function resolveVoiceLanguage(params: { env?: NodeJS.ProcessEnv; persisted?: unknown }): string {
- const result = resolveSpeechConfig({
- paseoHome: "/tmp/paseo-home",
- env: params.env ?? ({} as NodeJS.ProcessEnv),
- persisted: PersistedConfigSchema.parse(params.persisted ?? {}),
- });
- return result.speech.sttLanguages.voice;
- }
-
- async function transcribeWithResolvedVoiceLanguage(params: {
- env?: NodeJS.ProcessEnv;
- persisted?: unknown;
- }): Promise {
+ it("passes the configured provider language to STT", async () => {
const fakeStt = new FakeStt({ text: "hi", isLowConfidence: false });
const manager = new STTManager("s1", pino({ level: "silent" }), fakeStt, {
- language: resolveVoiceLanguage(params),
+ language: "pt",
});
await manager.transcribe(Buffer.alloc(2), "audio/pcm;rate=24000");
- return fakeStt;
- }
-
- it("defaults to English when no voice language config is set", async () => {
- const fakeStt = await transcribeWithResolvedVoiceLanguage({});
-
- expect(fakeStt.lastLanguage).toBe("en");
- });
-
- it("uses PASEO_VOICE_LANGUAGE over PASEO_DICTATION_LANGUAGE", async () => {
- const fakeStt = await transcribeWithResolvedVoiceLanguage({
- env: {
- PASEO_VOICE_LANGUAGE: "pt",
- PASEO_DICTATION_LANGUAGE: "es",
- } as NodeJS.ProcessEnv,
- });
expect(fakeStt.lastLanguage).toBe("pt");
});
- it("uses PASEO_DICTATION_LANGUAGE when PASEO_VOICE_LANGUAGE is unset", async () => {
- const fakeStt = await transcribeWithResolvedVoiceLanguage({
- env: {
- PASEO_DICTATION_LANGUAGE: "pt",
- } as NodeJS.ProcessEnv,
- });
-
- expect(fakeStt.lastLanguage).toBe("pt");
- });
-
- it("treats empty voice language env vars as unset", async () => {
- const fakeStt = await transcribeWithResolvedVoiceLanguage({
- env: {
- PASEO_VOICE_LANGUAGE: "",
- PASEO_DICTATION_LANGUAGE: " ",
- } as NodeJS.ProcessEnv,
- });
-
- expect(fakeStt.lastLanguage).toBe("en");
- });
-
- it("uses settings voice STT language when no env var is set", async () => {
- const fakeStt = await transcribeWithResolvedVoiceLanguage({
- persisted: {
- features: {
- voiceMode: {
- stt: {
- language: "fr",
- },
- },
- },
- },
- });
-
- expect(fakeStt.lastLanguage).toBe("fr");
- });
-
- it("uses env voice language over settings voice STT language", async () => {
- const fakeStt = await transcribeWithResolvedVoiceLanguage({
- env: {
- PASEO_VOICE_LANGUAGE: "pt",
- } as NodeJS.ProcessEnv,
- persisted: {
- features: {
- voiceMode: {
- stt: {
- language: "fr",
- },
- },
- },
- },
- });
-
- expect(fakeStt.lastLanguage).toBe("pt");
- });
-
- it("falls back to settings dictation STT language when voice language is unset", async () => {
- const fakeStt = await transcribeWithResolvedVoiceLanguage({
- persisted: {
- features: {
- dictation: {
- stt: {
- language: "es",
- },
- },
- },
- },
- });
-
- expect(fakeStt.lastLanguage).toBe("es");
- });
-
it("returns empty text for low-confidence transcriptions", async () => {
const manager = new STTManager(
"s1",
diff --git a/packages/server/src/server/agent/stt-manager.ts b/packages/server/src/server/voice-chat/providers/manual/transcription.ts
similarity index 96%
rename from packages/server/src/server/agent/stt-manager.ts
rename to packages/server/src/server/voice-chat/providers/manual/transcription.ts
index 391287f0f3..6e70ccb704 100644
--- a/packages/server/src/server/agent/stt-manager.ts
+++ b/packages/server/src/server/voice-chat/providers/manual/transcription.ts
@@ -1,9 +1,9 @@
import type pino from "pino";
-import type { SpeechToTextProvider, TranscriptionResult } from "../speech/speech-provider.js";
-import { toResolver, type Resolvable } from "../speech/provider-resolver.js";
-import { maybePersistDebugAudio } from "./stt-debug.js";
-import { parsePcm16MonoWav, parsePcmRateFromFormat } from "../speech/audio.js";
-import { Pcm16MonoResampler } from "./pcm16-resampler.js";
+import type { SpeechToTextProvider, TranscriptionResult } from "../../../speech/speech-provider.js";
+import { toResolver, type Resolvable } from "../../../speech/provider-resolver.js";
+import { maybePersistDebugAudio } from "../../../agent/stt-debug.js";
+import { parsePcm16MonoWav, parsePcmRateFromFormat } from "../../../speech/audio.js";
+import { Pcm16MonoResampler } from "../../../agent/pcm16-resampler.js";
interface TranscriptionMetadata {
agentId?: string;
diff --git a/packages/server/src/server/session/voice/voice-turn-controller.test.ts b/packages/server/src/server/voice-chat/providers/manual/turn.test.ts
similarity index 96%
rename from packages/server/src/server/session/voice/voice-turn-controller.test.ts
rename to packages/server/src/server/voice-chat/providers/manual/turn.test.ts
index 4f2a7e96cf..9ca91e4272 100644
--- a/packages/server/src/server/session/voice/voice-turn-controller.test.ts
+++ b/packages/server/src/server/voice-chat/providers/manual/turn.test.ts
@@ -7,12 +7,12 @@ import type {
StreamingTranscriptionCommittedEvent,
StreamingTranscriptionEvent,
StreamingTranscriptionSession,
-} from "../../speech/speech-provider.js";
+} from "../../../speech/speech-provider.js";
import type {
TurnDetectionProvider,
TurnDetectionSession,
-} from "../../speech/turn-detection-provider.js";
-import { createVoiceTurnController } from "./voice-turn-controller.js";
+} from "../../../speech/turn-detection-provider.js";
+import { createVoiceTurnController } from "./turn.js";
class FakeTurnDetectionSession extends EventEmitter implements TurnDetectionSession {
public readonly requiredSampleRate = 16000;
@@ -515,7 +515,7 @@ describe("voice turn controller", () => {
}
});
- it("reports STT errors and attempts one reconnect", async () => {
+ it("reports an STT failure only after its one reconnect is exhausted", async () => {
const harness = createControllerHarness();
await harness.controller.start();
@@ -523,12 +523,17 @@ describe("voice turn controller", () => {
firstSession?.emitError(new Error("stream failed"));
await settleSerialQueue();
- expect(harness.onError).toHaveBeenCalledTimes(1);
- expect(harness.onError).toHaveBeenCalledWith(
- expect.objectContaining({ message: "stream failed" }),
- );
+ expect(harness.onError).not.toHaveBeenCalled();
expect(firstSession?.closeCount).toBe(1);
expect(harness.sttSessions).toHaveLength(2);
expect(harness.sttSessions[1]?.connectCount).toBe(1);
+
+ harness.sttSessions[1]?.emitError(new Error("replacement stream failed"));
+ await settleSerialQueue();
+
+ expect(harness.onError).toHaveBeenCalledOnce();
+ expect(harness.onError).toHaveBeenCalledWith(
+ expect.objectContaining({ message: "replacement stream failed" }),
+ );
});
});
diff --git a/packages/server/src/server/session/voice/voice-turn-controller.ts b/packages/server/src/server/voice-chat/providers/manual/turn.ts
similarity index 98%
rename from packages/server/src/server/session/voice/voice-turn-controller.ts
rename to packages/server/src/server/voice-chat/providers/manual/turn.ts
index 20e2b4ba1f..d1e8a8e1f6 100644
--- a/packages/server/src/server/session/voice/voice-turn-controller.ts
+++ b/packages/server/src/server/voice-chat/providers/manual/turn.ts
@@ -2,14 +2,14 @@ import { Buffer } from "node:buffer";
import type { Logger } from "pino";
import { v4 as uuidv4 } from "uuid";
-import { Pcm16MonoResampler } from "../../agent/pcm16-resampler.js";
-import { parsePcmRateFromFormat } from "../../speech/audio.js";
+import { Pcm16MonoResampler } from "../../../agent/pcm16-resampler.js";
+import { parsePcmRateFromFormat } from "../../../speech/audio.js";
import type {
SpeechToTextProvider,
StreamingTranscriptionEvent,
StreamingTranscriptionSession,
-} from "../../speech/speech-provider.js";
-import type { TurnDetectionProvider } from "../../speech/turn-detection-provider.js";
+} from "../../../speech/speech-provider.js";
+import type { TurnDetectionProvider } from "../../../speech/turn-detection-provider.js";
const VOICE_FINAL_TRANSCRIPT_TIMEOUT_MS = 10_000;
@@ -253,9 +253,9 @@ export function createVoiceTurnController(params: {
}
function handleSttError(error: unknown): void {
- fail(error);
params.logger.warn({ err: error }, "voice_turn.stt_error");
if (reconnectAttemptedForTurn) {
+ fail(error);
sttSession?.close();
sttSession = null;
return;
diff --git a/packages/server/src/server/voice-config.ts b/packages/server/src/server/voice-config.ts
deleted file mode 100644
index d8cb1992f4..0000000000
--- a/packages/server/src/server/voice-config.ts
+++ /dev/null
@@ -1,77 +0,0 @@
-const VOICE_PROMPT_BLOCK_START = "";
-const VOICE_PROMPT_BLOCK_END = "";
-
-const VOICE_AGENT_SYSTEM_INSTRUCTION = [
- "Paseo voice mode is now on.",
- "You are the Paseo voice assistant.",
- "The user cannot see your chat messages or tool calls.",
- "Always use the speak tool for all user-facing communication.",
- "Before calling any non-speak tool, first call speak with a short acknowledgement of what you heard and what you will do next.",
- "For long-running work, use speak to provide progress updates before and during execution.",
- "Treat the user input as transcribed speech.",
- "If the user intent is clear, proceed without extra confirmation.",
- "If the transcription seems incomplete, cut off, ambiguous, or may contain a non-obvious mistake or misspelling, ask a clarifying question via speak before taking action.",
- "Use concise plain language suitable for speech output.",
-].join(" ");
-
-const VOICE_AGENT_DISABLED_INSTRUCTION = [
- "Paseo voice mode is now off.",
- "Ignore any earlier Paseo voice mode instructions in this thread.",
-].join(" ");
-
-function escapeRegExp(value: string): string {
- return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-
-function makeVoicePromptBlockRegex(): RegExp {
- return new RegExp(
- `${escapeRegExp(VOICE_PROMPT_BLOCK_START)}[\\s\\S]*?${escapeRegExp(VOICE_PROMPT_BLOCK_END)}`,
- "g",
- );
-}
-
-export function stripVoiceModeSystemPrompt(existing?: string): string | undefined {
- const trimmed = existing?.trim();
- if (!trimmed) {
- return undefined;
- }
- const stripped = trimmed.replace(makeVoicePromptBlockRegex(), "").trim();
- return stripped.length > 0 ? stripped : undefined;
-}
-
-export function buildVoiceModeSystemPrompt(existing: string | undefined, enabled: boolean): string {
- const basePrompt = stripVoiceModeSystemPrompt(existing);
- const voiceInstruction = enabled
- ? VOICE_AGENT_SYSTEM_INSTRUCTION
- : VOICE_AGENT_DISABLED_INSTRUCTION;
- const voiceBlock = [VOICE_PROMPT_BLOCK_START, voiceInstruction, VOICE_PROMPT_BLOCK_END].join(
- "\n",
- );
-
- return [basePrompt, voiceBlock]
- .filter((entry): entry is string => typeof entry === "string" && entry.length > 0)
- .join("\n\n");
-}
-
-export function wrapSpokenInput(text: string): string {
- return `\n${text}\n\nThis message was spoken by the user. Respond using the speak tool only, not normal messages, because the user may not be looking at the chat.`;
-}
-
-export function buildVoiceAgentMcpServerConfig(params: {
- command: string;
- baseArgs: string[];
- socketPath: string;
- env?: Record;
-}): {
- type: "stdio";
- command: string;
- args: string[];
- env?: Record;
-} {
- return {
- type: "stdio",
- command: params.command,
- args: [...params.baseArgs, "--socket", params.socketPath],
- ...(params.env ? { env: params.env } : {}),
- };
-}
diff --git a/packages/server/src/server/voice-local-agent.e2e.test.ts b/packages/server/src/server/voice-local-agent.e2e.test.ts
index 9dbe49d74a..59f6f7b44f 100644
--- a/packages/server/src/server/voice-local-agent.e2e.test.ts
+++ b/packages/server/src/server/voice-local-agent.e2e.test.ts
@@ -14,32 +14,28 @@ type SessionMessage = Extract<
{ type: T }
>;
-function makeTranscriptionHandler(resolve: (value: string) => void) {
- return (msg: SessionMessage<"transcription_result">) => {
- if (msg.type !== "transcription_result") return;
- const text = (msg.payload.text ?? "").trim();
+function makeTranscriptionHandler(callId: string, resolve: (value: string) => void) {
+ return (msg: SessionMessage<"voice.call.event">) => {
+ if (msg.payload.callId !== callId || msg.payload.event.type !== "transcript") return;
+ if (msg.payload.event.speaker !== "user") return;
+ const text = msg.payload.event.text.trim();
if (!text) return;
resolve(text);
};
}
-function makeErrorHandler(reject: (error: Error) => void) {
- return (msg: SessionMessage<"activity_log">) => {
- if (msg.type !== "activity_log") return;
- if (msg.payload.type !== "error") return;
- reject(new Error(msg.payload.content));
+function makeErrorHandler(callId: string, reject: (error: Error) => void) {
+ return (msg: SessionMessage<"voice.call.event">) => {
+ if (msg.payload.callId !== callId || msg.payload.event.type !== "error") return;
+ reject(new Error(msg.payload.event.message));
};
}
-function makeSpeakToolHandler(resolve: (value: string) => void) {
- return (msg: SessionMessage<"agent_stream">) => {
- if (msg.type !== "agent_stream") return;
- if (msg.payload.event.type !== "timeline") return;
- const item = msg.payload.event.item;
- if (item.type !== "tool_call") return;
- const name = item.name ?? "";
- if (!name.toLowerCase().includes("speak")) return;
- resolve(name);
+function makeSpokenReplyHandler(callId: string, resolve: (value: string) => void) {
+ return (msg: SessionMessage<"voice.call.event">) => {
+ if (msg.payload.callId !== callId || msg.payload.event.type !== "transcript") return;
+ if (msg.payload.event.speaker !== "assistant") return;
+ resolve(msg.payload.event.text);
};
}
@@ -87,9 +83,24 @@ function waitForSignal(
voiceTts: { provider: "openai", explicit: true },
},
},
- voiceLlmProvider: "codex",
- voiceLlmProviderExplicit: true,
- voiceLlmModel: "gpt-5.4-mini",
+ manualVoice: {
+ orchestrator: {
+ provider: "codex",
+ model: "gpt-5.4-mini",
+ modeId: "full-access",
+ thinkingOptionId: null,
+ },
+ openai: { stt: { apiKey: openaiApiKey! }, tts: { apiKey: openaiApiKey! } },
+ speech: {
+ providers: {
+ dictationStt: { provider: "openai", explicit: false, enabled: false },
+ voiceTurnDetection: { provider: "local", explicit: true },
+ voiceStt: { provider: "openai", explicit: true },
+ voiceTts: { provider: "openai", explicit: true },
+ },
+ sttLanguages: { dictation: "en", voice: "en" },
+ },
+ },
});
}, 120000);
@@ -109,30 +120,49 @@ function waitForSignal(
);
const voiceCwd = mkdtempSync(path.join(tmpdir(), "voice-local-agent-"));
+ const workspaceResult = await ctx.client.createWorkspace({
+ source: { kind: "directory", path: voiceCwd },
+ title: "Voice local agent",
+ });
+ if (!workspaceResult.workspace) {
+ throw new Error(workspaceResult.error ?? "Failed to create voice test workspace");
+ }
const targetAgent = await ctx.client.createAgent({
config: {
...getFullAccessConfig("codex"),
cwd: voiceCwd,
},
+ workspaceId: workspaceResult.workspace.workspaceId,
});
- const voiceMode = await ctx.client.setVoiceMode(true, targetAgent.id);
- expect(voiceMode.accepted).toBe(true);
+ const voiceCall = await ctx.client.startVoiceCall(
+ { workspaceId: workspaceResult.workspace.workspaceId, agentId: targetAgent.id },
+ [{ kind: "daemon-audio" }],
+ );
const transcriptionPromise = waitForSignal(120000, (resolve, reject) => {
const offTranscript = ctx.client.on(
- "transcription_result",
- makeTranscriptionHandler(resolve),
+ "voice.call.event",
+ makeTranscriptionHandler(voiceCall.callId, resolve),
+ );
+ const offError = ctx.client.on(
+ "voice.call.event",
+ makeErrorHandler(voiceCall.callId, reject),
);
- const offError = ctx.client.on("activity_log", makeErrorHandler(reject));
return () => {
offTranscript();
offError();
};
});
- const speakToolPromise = waitForSignal(120000, (resolve, reject) => {
- const offStream = ctx.client.on("agent_stream", makeSpeakToolHandler(resolve));
- const offError = ctx.client.on("activity_log", makeErrorHandler(reject));
+ const spokenReplyPromise = waitForSignal(120000, (resolve, reject) => {
+ const offStream = ctx.client.on(
+ "voice.call.event",
+ makeSpokenReplyHandler(voiceCall.callId, resolve),
+ );
+ const offError = ctx.client.on(
+ "voice.call.event",
+ makeErrorHandler(voiceCall.callId, reject),
+ );
return () => {
offStream();
offError();
@@ -150,21 +180,20 @@ function waitForSignal