diff --git a/packages/app/src/components/sliding-sidebar.tsx b/packages/app/src/components/sliding-sidebar.tsx index b672c95ba..05b12aeb9 100644 --- a/packages/app/src/components/sliding-sidebar.tsx +++ b/packages/app/src/components/sliding-sidebar.tsx @@ -1,5 +1,5 @@ import { useCallback, useMemo, useState, useEffect } from "react"; -import { View, Pressable, Text, Platform, Alert } from "react-native"; +import { View, Pressable, Text, Platform, Alert, ActivityIndicator } from "react-native"; import { useSafeAreaInsets } from "react-native-safe-area-context"; import Animated, { useAnimatedStyle, @@ -57,7 +57,7 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) { const trafficLightPadding = useTrafficLightPadding(); const dragHandlers = useTauriDragHandlers(); const { connectionStates } = useDaemonConnections(); - const { isVoiceMode, startVoice, stopVoice } = useVoice(); + const { isVoiceMode, isVoiceSwitching, startVoice, stopVoice } = useVoice(); const [showVoiceHostPicker, setShowVoiceHostPicker] = useState(false); // Track user-initiated refresh to avoid showing spinner on background revalidation @@ -148,6 +148,9 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) { const hasAnyConfiguredHosts = connectionStates.size > 0; const handleToggleVoice = useCallback(() => { + if (isVoiceSwitching) { + return; + } if (isVoiceMode) { void stopVoice().catch((error) => { console.error("[SlidingSidebar] Failed to stop voice", error); @@ -181,7 +184,7 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) { } setShowVoiceHostPicker(true); - }, [hasAnyConfiguredHosts, isVoiceMode, startVoice, stopVoice, voiceEligibleHosts]); + }, [hasAnyConfiguredHosts, isVoiceMode, isVoiceSwitching, startVoice, stopVoice, voiceEligibleHosts]); const handleSelectVoiceHost = useCallback( (serverId: string) => { @@ -324,11 +327,13 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) { accessible accessibilityLabel="Voice" accessibilityRole="button" + disabled={isVoiceSwitching} onPress={handleToggleVoice} > {({ hovered }) => ( - - )} + ) : ( + + ) + )} {({ hovered }) => ( - + : hovered + ? theme.colors.foreground + : theme.colors.foregroundMuted + } + /> + ) : ( + + ) )} ({ event: event as AgentStreamEventPayload, @@ -1059,15 +1051,8 @@ export function SessionProvider({ }); if (!isFinalChunk) { - console.log( - `[Session] Buffered chunk ${chunkIndex} for group ${playbackGroupId}` - ); return; } - - console.log( - `[Session] Received final chunk for group ${playbackGroupId}, total chunks: ${buffer.length}` - ); buffer.sort((a, b) => a.chunkIndex - b.chunkIndex); let playbackFailed = false; @@ -1109,10 +1094,6 @@ export function SessionProvider({ offset += chunk.length; } - console.log( - `[Session] Playing concatenated audio: ${buffer.length} chunks, ${totalSize} bytes` - ); - const audioBlob = { type: mimeType, size: totalSize, @@ -1264,11 +1245,7 @@ export function SessionProvider({ const transcriptText = message.payload.text.trim(); if (!transcriptText) { - console.log( - "[Session] Empty transcription (false positive) - ignoring" - ); } else { - console.log("[Session] Transcription received - stopping playback"); audioPlayer.stop(); setIsPlayingAudio(serverId, false); setCurrentAssistantMessage(serverId, ""); diff --git a/packages/app/src/contexts/voice-context.tsx b/packages/app/src/contexts/voice-context.tsx index 9fcf233e7..0b66c243f 100644 --- a/packages/app/src/contexts/voice-context.tsx +++ b/packages/app/src/contexts/voice-context.tsx @@ -12,6 +12,7 @@ const VOICE_VAD_DETECTION_GRACE_PERIOD_MS = 700; interface VoiceContextValue { isVoiceMode: boolean; + isVoiceSwitching: boolean; volume: number; isMuted: boolean; isDetecting: boolean; @@ -57,7 +58,12 @@ export function VoiceProvider({ children }: VoiceProviderProps) { ); const realtimeSessionRef = useRef(null); const [isVoiceMode, setIsVoiceMode] = useState(false); + const [isVoiceSwitching, setIsVoiceSwitching] = useState(false); const bargeInPlaybackStopRef = useRef(null); + const wasVoiceSocketConnectedRef = useRef(false); + const lastVoiceModeSyncedClientRef = useRef(null); + const voiceTransportReadyRef = useRef(false); + const voiceResyncInFlightRef = useRef(false); const realtimeAudio = useSpeechmaticsAudio({ onSpeechStart: () => { @@ -91,6 +97,10 @@ export function VoiceProvider({ children }: VoiceProviderProps) { console.log("[Voice] Speech ended"); }, onAudioSegment: ({ audioData, isLast }) => { + if (!voiceTransportReadyRef.current) { + console.log("[Voice] Skipping audio segment: voice transport not ready"); + return; + } console.log( "[Voice] Sending audio segment, length:", audioData.length, @@ -134,6 +144,48 @@ export function VoiceProvider({ children }: VoiceProviderProps) { realtimeSessionRef.current = activeSession; }, [activeSession]); + useEffect(() => { + const connected = activeSession?.connection.isConnected ?? false; + const client = activeSession?.client ?? null; + if (!connected) { + voiceTransportReadyRef.current = false; + } + + if (!isVoiceMode || !activeServerId || !client) { + wasVoiceSocketConnectedRef.current = connected; + if (!isVoiceMode) { + lastVoiceModeSyncedClientRef.current = null; + } + voiceTransportReadyRef.current = false; + return; + } + + const connectionRecovered = connected && !wasVoiceSocketConnectedRef.current; + const clientChanged = lastVoiceModeSyncedClientRef.current !== client; + if (connected && (connectionRecovered || clientChanged)) { + if (!voiceResyncInFlightRef.current) { + voiceResyncInFlightRef.current = true; + voiceTransportReadyRef.current = false; + setIsVoiceSwitching(true); + void client.setVoiceMode(true).then( + () => { + console.log("[Voice] Re-synced voice mode after reconnect"); + lastVoiceModeSyncedClientRef.current = client; + voiceTransportReadyRef.current = true; + }, + (error) => { + console.error("[Voice] Failed to re-sync voice mode:", error); + } + ).finally(() => { + voiceResyncInFlightRef.current = false; + setIsVoiceSwitching(false); + }); + } + } + + wasVoiceSocketConnectedRef.current = connected; + }, [activeServerId, activeSession?.client, activeSession?.connection.isConnected, isVoiceMode]); + const isPlayingAudio = activeSession?.isPlayingAudio ?? false; useEffect(() => { @@ -155,56 +207,67 @@ export function VoiceProvider({ children }: VoiceProviderProps) { throw new Error(`Host ${serverId} is not connected`); } + setIsVoiceSwitching(true); + voiceTransportReadyRef.current = false; try { realtimeSessionRef.current = session; setActiveServerId(serverId); await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => { console.warn("[Voice] Failed to activate keep-awake:", error); }); - await session.audioPlayer?.warmup?.(); - await realtimeAudio.start(); - setIsVoiceMode(true); - console.log("[Voice] Mode enabled"); - if (session?.client) { await session.client.setVoiceMode(true); } else { console.warn("[Voice] setVoiceMode skipped: daemon unavailable"); } + await session.audioPlayer?.warmup?.(); + await realtimeAudio.start(); + voiceTransportReadyRef.current = true; + setIsVoiceMode(true); + lastVoiceModeSyncedClientRef.current = session.client; + console.log("[Voice] Mode enabled"); } catch (error: any) { console.error("[Voice] Failed to start:", error); + await realtimeAudio.stop().catch(() => undefined); setActiveServerId((current) => (current === serverId ? null : current)); await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined); throw error; + } finally { + setIsVoiceSwitching(false); } }, [getSession, realtimeAudio] ); const stopVoice = useCallback(async () => { + setIsVoiceSwitching(true); + voiceTransportReadyRef.current = false; try { const session = realtimeSessionRef.current; session?.audioPlayer?.stop(); + if (session?.client) { + await session.client.setVoiceMode(false); + lastVoiceModeSyncedClientRef.current = session.client; + } else { + console.warn("[Voice] setVoiceMode skipped: daemon unavailable"); + } await realtimeAudio.stop(); setIsVoiceMode(false); setActiveServerId(null); await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined); console.log("[Voice] Mode disabled"); - - if (session?.client) { - await session.client.setVoiceMode(false); - } else { - console.warn("[Voice] setVoiceMode skipped: daemon unavailable"); - } } catch (error: any) { console.error("[Voice] Failed to stop:", error); await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined); throw error; + } finally { + setIsVoiceSwitching(false); } }, [realtimeAudio]); const value: VoiceContextValue = { isVoiceMode, + isVoiceSwitching, volume: realtimeAudio.volume, isMuted: realtimeAudio.isMuted, isDetecting: realtimeAudio.isDetecting, diff --git a/packages/app/src/hooks/use-aggregated-agents.ts b/packages/app/src/hooks/use-aggregated-agents.ts index 90a94065a..e6b0a0593 100644 --- a/packages/app/src/hooks/use-aggregated-agents.ts +++ b/packages/app/src/hooks/use-aggregated-agents.ts @@ -4,7 +4,6 @@ import { useDaemonConnections } from "@/contexts/daemon-connections-context"; import { useSessionStore } from "@/stores/session-store"; import type { AgentDirectoryEntry } from "@/types/agent-directory"; import type { Agent } from "@/stores/session-store"; -import { isPerfLoggingEnabled } from "@/utils/perf"; import { derivePendingPermissionKey, normalizeAgentSnapshot } from "@/utils/agent-snapshots"; export interface AggregatedAgent extends AgentDirectoryEntry { @@ -44,14 +43,12 @@ export function useAggregatedAgents(): AggregatedAgentsResult { ); const refreshAll = useCallback(() => { - console.log('[useAggregatedAgents] Manual refresh triggered for all sessions'); for (const [serverId, client] of Object.entries(sessionClients)) { if (!client) { continue; } void (async () => { try { - console.log(`[useAggregatedAgents] Refreshing session ${serverId}`); const agentsList = await client.fetchAgents({ filter: { labels: { ui: "true" } }, }); @@ -131,27 +128,20 @@ export function useAggregatedAgents(): AggregatedAgentsResult { const hasAnyData = allAgents.length > 0; // Check if any connection is currently loading - const connectingReasons: string[] = []; - const isConnecting = Array.from(connectionStates.entries()).some(([id, c]) => { - const shortId = id.substring(0, 20); - + const isConnecting = Array.from(connectionStates.entries()).some(([, c]) => { // First-time connection (never received agent list) if (c.status === 'connecting' && !c.hasEverReceivedAgentList) { - connectingReasons.push(`${shortId}: first-time connecting`); return true; } if (c.status === 'online' && !c.hasEverReceivedAgentList) { - connectingReasons.push(`${shortId}: online but no fetch_agents yet`); return true; } // Reconnecting (have received agent list before) if (c.status === 'connecting' && c.hasEverReceivedAgentList) { - connectingReasons.push(`${shortId}: reconnecting`); return true; } if (c.status === 'online' && !c.agentListReady && c.hasEverReceivedAgentList) { - connectingReasons.push(`${shortId}: online but agentListReady=false (waiting for fetch_agents)`); return true; } @@ -167,28 +157,6 @@ export function useAggregatedAgents(): AggregatedAgentsResult { // isLoading: Generic loading flag (either initial or revalidating) const isLoading = isConnecting; - if (isPerfLoggingEnabled()) { - const connectionStatesArray = Array.from(connectionStates.entries()).map(([id, state]) => ({ - id: id.substring(0, 20) + (id.length > 20 ? '...' : ''), - status: state.status, - agentListReady: state.agentListReady, - hasEverReceivedAgentList: state.hasEverReceivedAgentList, - })); - - console.log('[useAggregatedAgents] States:', { - hasAnyData, - isConnecting, - isInitialLoad, - isRevalidating, - totalConnectionStates: connectionStates.size, - connectingReasons: connectingReasons.length > 0 ? connectingReasons : 'none', - }); - - console.log('[useAggregatedAgents] Connection States Detail:', - JSON.stringify(connectionStatesArray, null, 2) - ); - } - return { agents: allAgents, isLoading, diff --git a/packages/app/src/utils/agent-grouping.ts b/packages/app/src/utils/agent-grouping.ts index 455c0204a..c57e061c1 100644 --- a/packages/app/src/utils/agent-grouping.ts +++ b/packages/app/src/utils/agent-grouping.ts @@ -210,9 +210,6 @@ export interface GroupedAgents { const ACTIVE_GRACE_PERIOD_MS = 2 * 24 * 60 * 60 * 1000; // 2 days (temporary for screenshots) -// eslint-disable-next-line no-console -console.log('[agent-grouping] ACTIVE_GRACE_PERIOD_MS:', ACTIVE_GRACE_PERIOD_MS, 'ms =', ACTIVE_GRACE_PERIOD_MS / (1000 * 60 * 60), 'hours'); - interface GroupAgentsOptions { /** * Optional function to read a remote URL for an agent. @@ -252,17 +249,6 @@ export function groupAgents( const isRecentlyActive = ageDiff < ACTIVE_GRACE_PERIOD_MS; const isActive = isRunningOrAttention || isRecentlyActive; - // eslint-disable-next-line no-console - console.log('[agent-grouping] agent:', agent.title || agent.id, { - status: agent.status, - lastActivityAt: agent.lastActivityAt, - ageDiffHours: ageDiff / (1000 * 60 * 60), - gracePeriodHours: ACTIVE_GRACE_PERIOD_MS / (1000 * 60 * 60), - isRunningOrAttention, - isRecentlyActive, - isActive, - }); - if (isActive) { activeAgents.push(agent); } else { diff --git a/packages/app/src/utils/perf.ts b/packages/app/src/utils/perf.ts index 326e58811..45dc7f196 100644 --- a/packages/app/src/utils/perf.ts +++ b/packages/app/src/utils/perf.ts @@ -1,21 +1,4 @@ -const getIsDevEnvironment = (): boolean => { - const globalDev = (globalThis as { __DEV__?: boolean } | undefined)?.__DEV__; - if (typeof globalDev === "boolean") { - return globalDev; - } - if (typeof process !== "undefined" && process.env?.NODE_ENV) { - return process.env.NODE_ENV !== "production"; - } - return false; -}; - -const shouldDisablePerfLogging = - typeof process !== "undefined" && process.env?.EXPO_PUBLIC_DISABLE_PERF_LOGGING === "1"; -const shouldForcePerfLogging = - typeof process !== "undefined" && process.env?.EXPO_PUBLIC_ENABLE_PERF_LOGGING === "1"; - -const PERF_LOGGING_ENABLED = - (shouldForcePerfLogging || getIsDevEnvironment()) && !shouldDisablePerfLogging; +const PERF_LOGGING_ENABLED = false; export const isPerfLoggingEnabled = (): boolean => PERF_LOGGING_ENABLED; diff --git a/packages/server/src/client/daemon-client.ts b/packages/server/src/client/daemon-client.ts index 43387c76f..a3ebb15d3 100644 --- a/packages/server/src/client/daemon-client.ts +++ b/packages/server/src/client/daemon-client.ts @@ -201,6 +201,10 @@ type SpeechModelsListPayload = SpeechModelsListResponse["payload"]; type SpeechModelsDownloadPayload = SpeechModelsDownloadResponse["payload"]; type ListCommandsPayload = ListCommandsResponse["payload"]; type ExecuteCommandPayload = ExecuteCommandResponse["payload"]; +type SetVoiceModePayload = Extract< + SessionOutboundMessage, + { type: "set_voice_mode_response" } +>["payload"]; type AgentPermissionResolvedPayload = AgentPermissionResolvedMessage["payload"]; type ListTerminalsPayload = ListTerminalsResponse["payload"]; type CreateTerminalPayload = CreateTerminalResponse["payload"]; @@ -1285,8 +1289,31 @@ export class DaemonClient { // Audio / Voice // ============================================================================ - async setVoiceMode(enabled: boolean, voiceAgentId?: string): Promise { - this.sendSessionMessage({ type: "set_voice_mode", enabled, voiceAgentId }); + async setVoiceMode( + enabled: boolean, + voiceAgentId?: string + ): Promise { + const requestId = this.createRequestId(); + const message = SessionInboundMessageSchema.parse({ + type: "set_voice_mode", + enabled, + ...(voiceAgentId ? { voiceAgentId } : {}), + requestId, + }); + return this.sendRequest({ + requestId, + message, + timeout: 10000, + select: (msg) => { + if (msg.type !== "set_voice_mode_response") { + return null; + } + if (msg.payload.requestId !== requestId) { + return null; + } + return msg.payload; + }, + }); } async sendVoiceAudioChunk( diff --git a/packages/server/src/server/agent/tts-manager.test.ts b/packages/server/src/server/agent/tts-manager.test.ts index 5c0dea3be..6a98064b3 100644 --- a/packages/server/src/server/agent/tts-manager.test.ts +++ b/packages/server/src/server/agent/tts-manager.test.ts @@ -43,5 +43,83 @@ describe("TTSManager", () => { expect((audioMsgs[1] as any).payload.chunkIndex).toBe(1); expect((audioMsgs[1] as any).payload.isLastChunk).toBe(true); }); -}); + it("splits long text into safe synthesis segments", async () => { + const calls: string[] = []; + const tts: TextToSpeechProvider = { + async synthesizeSpeech(text: string): Promise<{ stream: Readable; format: string }> { + calls.push(text); + return { + stream: Readable.from([Buffer.from("x")]), + format: "pcm;rate=24000", + }; + }, + }; + + const manager = new TTSManager("s1", pino({ level: "silent" }), tts); + const abort = new AbortController(); + const longText = Array.from({ length: 180 }) + .map((_, i) => `Sentence ${i + 1}.`) + .join(" "); + + await manager.generateAndWaitForPlayback( + longText, + (msg) => { + if (msg.type === "audio_output") { + manager.confirmAudioPlayed(msg.payload.id); + } + }, + abort.signal, + true + ); + + expect(calls.length).toBeGreaterThan(1); + expect(calls.every((text) => text.length <= 400)).toBe(true); + }); + + it("does not emit unhandled rejections when stream iteration fails", async () => { + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown) => { + unhandled.push(reason); + }; + process.on("unhandledRejection", onUnhandled); + + try { + const tts: TextToSpeechProvider = { + async synthesizeSpeech(): Promise<{ stream: Readable; format: string }> { + const stream = Readable.from( + (async function* () { + yield Buffer.from("a"); + throw new Error("stream exploded"); + })() + ); + return { + stream, + format: "pcm;rate=24000", + }; + }, + }; + + const manager = new TTSManager("s1", pino({ level: "silent" }), tts); + const abort = new AbortController(); + + await expect( + manager.generateAndWaitForPlayback( + "hello", + (msg) => { + if (msg.type === "audio_output") { + manager.confirmAudioPlayed(msg.payload.id); + } + }, + abort.signal, + true + ) + ).rejects.toThrow("stream exploded"); + + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(unhandled).toHaveLength(0); + } finally { + process.off("unhandledRejection", onUnhandled); + } + }); +}); diff --git a/packages/server/src/server/agent/tts-manager.ts b/packages/server/src/server/agent/tts-manager.ts index 083c94096..223b03707 100644 --- a/packages/server/src/server/agent/tts-manager.ts +++ b/packages/server/src/server/agent/tts-manager.ts @@ -10,6 +10,84 @@ interface PendingPlayback { streamEnded: boolean; } +const MAX_TTS_SEGMENT_CHARS = 400; + +function splitTextForTts(text: string, maxChars: number): string[] { + const normalized = text.trim().replace(/\s+/g, " "); + if (!normalized) { + throw new Error("Cannot synthesize empty text"); + } + + if (normalized.length <= maxChars) { + return [normalized]; + } + + const parts: string[] = []; + const sentenceChunks = normalized.split(/(?<=[.!?])\s+/); + + let current = ""; + const pushCurrent = () => { + const trimmed = current.trim(); + if (trimmed.length > 0) { + parts.push(trimmed); + } + current = ""; + }; + + const appendFragment = (fragment: string) => { + const trimmed = fragment.trim(); + if (!trimmed) { + return; + } + + if (!current) { + current = trimmed; + return; + } + + const candidate = `${current} ${trimmed}`; + if (candidate.length <= maxChars) { + current = candidate; + return; + } + + pushCurrent(); + current = trimmed; + }; + + const splitLargeFragment = (fragment: string): string[] => { + const trimmed = fragment.trim(); + if (trimmed.length <= maxChars) { + return [trimmed]; + } + + const out: string[] = []; + let remaining = trimmed; + while (remaining.length > maxChars) { + let idx = remaining.lastIndexOf(" ", maxChars); + if (idx < Math.floor(maxChars * 0.5)) { + idx = maxChars; + } + out.push(remaining.slice(0, idx).trim()); + remaining = remaining.slice(idx).trim(); + } + if (remaining.length > 0) { + out.push(remaining); + } + return out; + }; + + for (const sentence of sentenceChunks) { + const fragments = splitLargeFragment(sentence); + for (const fragment of fragments) { + appendFragment(fragment); + } + } + pushCurrent(); + + return parts; +} + /** * Per-session TTS manager * Handles TTS audio generation and playback confirmation tracking @@ -33,6 +111,37 @@ export class TTSManager { emitMessage: (msg: SessionOutboundMessage) => void, abortSignal: AbortSignal, isVoiceMode: boolean + ): Promise { + this.logger.info( + { + isVoiceMode, + textLength: text.length, + text, + }, + "TTS input text" + ); + + const segments = splitTextForTts(text, MAX_TTS_SEGMENT_CHARS); + for (const segment of segments) { + if (abortSignal.aborted) { + this.logger.debug("Aborted before generating segmented audio"); + return; + } + + await this.generateSegmentAndWaitForPlayback( + segment, + emitMessage, + abortSignal, + isVoiceMode + ); + } + } + + private async generateSegmentAndWaitForPlayback( + text: string, + emitMessage: (msg: SessionOutboundMessage) => void, + abortSignal: AbortSignal, + isVoiceMode: boolean ): Promise { if (!this.tts) { throw new Error("TTS not configured"); @@ -121,11 +230,6 @@ export class TTSManager { }, }); - this.logger.debug( - { chunkId, isLastChunk: next.done }, - "Sent audio chunk" - ); - chunkIndex += 1; if (next.done) { @@ -151,7 +255,6 @@ export class TTSManager { } else { this.logger.error({ err: error }, "Error streaming audio"); this.pendingPlaybacks.delete(audioId); - pendingPlayback.reject(error as Error); throw error; } } finally { diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index 7cd6a0ef0..a2c371db6 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -147,8 +147,16 @@ describe("daemon client E2E", () => { }); expect(Array.isArray(voiceAgents)).toBe(true); - await expect(ctx.client.setVoiceMode(true)).resolves.toBeUndefined(); - await expect(ctx.client.setVoiceMode(false)).resolves.toBeUndefined(); + await expect(ctx.client.setVoiceMode(true)).resolves.toMatchObject({ + enabled: true, + accepted: true, + error: null, + }); + await expect(ctx.client.setVoiceMode(false)).resolves.toMatchObject({ + enabled: false, + accepted: true, + error: null, + }); await ctx.client.deleteAgent(randomUUID()); }, 30000); diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 2465a5807..8e0a8bc6e 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -879,7 +879,7 @@ export class Session { break; case "set_voice_mode": - await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId); + await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId, msg.requestId); break; case "send_agent_message_request": @@ -1259,7 +1259,8 @@ export class Session { */ private async handleSetVoiceMode( enabled: boolean, - voiceAgentId?: string + voiceAgentId?: string, + requestId?: string ): Promise { if (enabled) { let normalizedVoiceAgentId: string | null = null; @@ -1282,6 +1283,18 @@ export class Session { }, "Voice mode enabled (agent-backed)" ); + if (requestId) { + this.emit({ + type: "set_voice_mode_response", + payload: { + requestId, + enabled: true, + voiceAgentId: this.voiceAssistantAgentId, + accepted: true, + error: null, + }, + }); + } return; } @@ -1290,6 +1303,18 @@ export class Session { { voiceAssistantAgentId: this.voiceAssistantAgentId }, "Voice mode disabled (agent-backed)" ); + if (requestId) { + this.emit({ + type: "set_voice_mode_response", + payload: { + requestId, + enabled: false, + voiceAgentId: this.voiceAssistantAgentId, + accepted: true, + error: null, + }, + }); + } } private parseVoiceAgentId(rawId: string, source: string): string { @@ -2471,7 +2496,6 @@ export class Session { lastActivityAt: string; appVisible: boolean; }): void { - this.sessionLogger.debug({ heartbeat: msg }, "Client heartbeat"); this.clientActivity = { deviceType: msg.deviceType, focusedAgentId: msg.focusedAgentId, @@ -4152,6 +4176,12 @@ export class Session { private async handleAudioChunk( msg: Extract ): Promise { + if (!this.isVoiceMode) { + this.sessionLogger.warn( + "Received voice_audio_chunk while voice mode is disabled; transcript will be emitted but voice assistant turn is skipped" + ); + } + await this.handleVoiceSpeechStart(); const chunkBuffer = Buffer.from(msg.audio, "base64"); @@ -4193,11 +4223,6 @@ export class Session { this.audioBuffer.totalPCMBytes += chunkBuffer.length; } - this.sessionLogger.debug( - { bytes: chunkBuffer.length, chunks: this.audioBuffer.chunks.length, pcmBytes: this.audioBuffer.totalPCMBytes }, - `Buffered audio chunk (${chunkBuffer.length} bytes, chunks: ${this.audioBuffer.chunks.length}${this.audioBuffer.isPCM ? `, PCM bytes: ${this.audioBuffer.totalPCMBytes}` : ""})` - ); - // In voice mode, only process audio when the user has finished speaking (isLast = true) // This prevents partial transcriptions from being sent to the LLM if (this.isVoiceMode) { @@ -4343,6 +4368,15 @@ export class Session { }); const transcriptText = result.text.trim(); + this.sessionLogger.info( + { + requestId, + isVoiceMode: this.isVoiceMode, + transcriptLength: transcriptText.length, + transcript: transcriptText, + }, + "Transcription result" + ); // Emit transcription result this.emit({ @@ -4408,8 +4442,17 @@ export class Session { }, }); - // Set phase to LLM and process (TTS enabled in voice mode for voice agents) this.clearSpeechInProgress("transcription complete"); + if (!this.isVoiceMode) { + this.sessionLogger.debug( + { requestId }, + "Skipping voice agent processing because voice mode is disabled" + ); + this.setPhase("idle"); + return; + } + + // Set phase to LLM and process (TTS enabled in voice mode for voice agents) this.setPhase("llm"); this.currentStreamPromise = this.processVoiceTurn(result.text); await this.currentStreamPromise; @@ -4560,7 +4603,6 @@ export class Session { this.agentManager.recordUserMessage(agentId, userText); let sawSpeakToolCall = false; - const assistantTextChunks: string[] = []; const iterator = this.agentManager.streamAgent(agentId, prompt); for await (const event of iterator) { if (event.type === "turn_failed") { @@ -4572,9 +4614,6 @@ export class Session { sawSpeakToolCall = true; } } - if (event.item.type === "assistant_message" && event.item.text.trim().length > 0) { - assistantTextChunks.push(event.item.text.trim()); - } } if (event.type === "permission_requested") { if (this.shouldAllowVoicePermission(event.request)) { @@ -4594,26 +4633,10 @@ export class Session { } } - if (!sawSpeakToolCall && assistantTextChunks.length > 0) { - const fallbackText = assistantTextChunks.join(" ").trim(); - await this.ttsManager.generateAndWaitForPlayback( - fallbackText, - (msg) => this.emit(msg), - this.abortController.signal, - true - ); - this.emit({ - type: "activity_log", - payload: { - id: uuidv4(), - timestamp: new Date(), - type: "assistant", - content: fallbackText, - }, - }); + if (!sawSpeakToolCall) { this.sessionLogger.warn( { voiceAssistantAgentId: agentId }, - "Voice agent responded without speak tool; used fallback TTS from assistant text" + "Voice agent turn completed without speak tool call; no audio playback emitted" ); } } diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 610420c1a..4b63da744 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -389,7 +389,14 @@ export class VoiceAssistantWebSocketServer { ? { sessionMessageType: message.message.type } : {}), }; - this.logger.debug(messageSummary, "Received message"); + const isSessionNoise = + message.type === "session" && + (message.message.type === "client_heartbeat" || + message.message.type === "voice_audio_chunk" || + message.message.type === "dictation_stream_chunk"); + if (!isSessionNoise) { + this.logger.debug(messageSummary, "Received message"); + } if (message.type === "ping") { this.sendToClient(ws, { type: "pong" }); diff --git a/packages/server/src/shared/messages.ts b/packages/server/src/shared/messages.ts index 3b8f88252..9acfcb49b 100644 --- a/packages/server/src/shared/messages.ts +++ b/packages/server/src/shared/messages.ts @@ -348,6 +348,7 @@ export const SetVoiceModeMessageSchema = z.object({ type: z.literal("set_voice_mode"), enabled: z.boolean(), voiceAgentId: z.string().optional(), + requestId: z.string().optional(), }); export const SendAgentMessageSchema = z.object({ @@ -571,6 +572,17 @@ export const SetAgentThinkingResponseMessageSchema = z.object({ }), }); +export const SetVoiceModeResponseMessageSchema = z.object({ + type: z.literal("set_voice_mode_response"), + payload: z.object({ + requestId: z.string(), + enabled: z.boolean(), + voiceAgentId: z.string().nullable(), + accepted: z.boolean(), + error: z.string().nullable(), + }), +}); + export const AgentPermissionResponseMessageSchema = z.object({ type: z.literal("agent_permission_response"), agentId: z.string(), @@ -1627,6 +1639,7 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [ FetchAgentsResponseMessageSchema, FetchAgentResponseMessageSchema, SendAgentMessageResponseMessageSchema, + SetVoiceModeResponseMessageSchema, SetAgentModeResponseMessageSchema, SetAgentModelResponseMessageSchema, SetAgentThinkingResponseMessageSchema, @@ -1690,6 +1703,9 @@ export type FetchAgentResponseMessage = z.infer< export type SendAgentMessageResponseMessage = z.infer< typeof SendAgentMessageResponseMessageSchema >; +export type SetVoiceModeResponseMessage = z.infer< + typeof SetVoiceModeResponseMessageSchema +>; export type WaitForFinishResponseMessage = z.infer< typeof WaitForFinishResponseMessageSchema >;