Update files

This commit is contained in:
Mohamed Boudra
2026-02-07 09:43:44 +07:00
parent 3f58937c98
commit 4a9cf9e90f
13 changed files with 427 additions and 157 deletions

View File

@@ -1,5 +1,5 @@
import { useCallback, useMemo, useState, useEffect } from "react";
import { View, Pressable, Text, Platform, Alert } from "react-native";
import { View, Pressable, Text, Platform, Alert, ActivityIndicator } from "react-native";
import { useSafeAreaInsets } from "react-native-safe-area-context";
import Animated, {
useAnimatedStyle,
@@ -57,7 +57,7 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
const trafficLightPadding = useTrafficLightPadding();
const dragHandlers = useTauriDragHandlers();
const { connectionStates } = useDaemonConnections();
const { isVoiceMode, startVoice, stopVoice } = useVoice();
const { isVoiceMode, isVoiceSwitching, startVoice, stopVoice } = useVoice();
const [showVoiceHostPicker, setShowVoiceHostPicker] = useState(false);
// Track user-initiated refresh to avoid showing spinner on background revalidation
@@ -148,6 +148,9 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
const hasAnyConfiguredHosts = connectionStates.size > 0;
const handleToggleVoice = useCallback(() => {
if (isVoiceSwitching) {
return;
}
if (isVoiceMode) {
void stopVoice().catch((error) => {
console.error("[SlidingSidebar] Failed to stop voice", error);
@@ -181,7 +184,7 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
}
setShowVoiceHostPicker(true);
}, [hasAnyConfiguredHosts, isVoiceMode, startVoice, stopVoice, voiceEligibleHosts]);
}, [hasAnyConfiguredHosts, isVoiceMode, isVoiceSwitching, startVoice, stopVoice, voiceEligibleHosts]);
const handleSelectVoiceHost = useCallback(
(serverId: string) => {
@@ -324,11 +327,13 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
accessible
accessibilityLabel="Voice"
accessibilityRole="button"
disabled={isVoiceSwitching}
onPress={handleToggleVoice}
>
{({ hovered }) => (
<AudioLines
size={20}
isVoiceSwitching ? (
<ActivityIndicator
size="small"
color={
isVoiceMode
? theme.colors.foreground
@@ -337,7 +342,19 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
: theme.colors.foregroundMuted
}
/>
)}
) : (
<AudioLines
size={20}
color={
isVoiceMode
? theme.colors.foreground
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
)
)}
</Pressable>
<Pressable
style={styles.footerIconButton}
@@ -448,19 +465,33 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
accessible
accessibilityLabel="Voice"
accessibilityRole="button"
disabled={isVoiceSwitching}
onPress={handleToggleVoice}
>
{({ hovered }) => (
<AudioLines
size={20}
color={
isVoiceMode
? theme.colors.foreground
: hovered
isVoiceSwitching ? (
<ActivityIndicator
size="small"
color={
isVoiceMode
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
) : (
<AudioLines
size={20}
color={
isVoiceMode
? theme.colors.foreground
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
)
)}
</Pressable>
<Pressable

View File

@@ -872,10 +872,7 @@ export function SessionProvider({
const { agentId, event, timestamp } = message.payload;
const parsedTimestamp = new Date(timestamp);
console.log("[Session] agent_stream", { agentId, event, timestamp });
if (event.type === "attention_required") {
console.log("[Session] attention_required", { agentId, shouldNotify: event.shouldNotify, reason: event.reason });
if (event.shouldNotify) {
notifyAgentAttention({
agentId,
@@ -926,11 +923,6 @@ export function SessionProvider({
if (message.type !== "agent_stream_snapshot") return;
const { agentId, events } = message.payload;
console.log("[Session] agent_stream_snapshot", {
agentId,
eventCount: events.length,
});
const hydrated = hydrateStreamState(
events.map(({ event, timestamp }) => ({
event: event as AgentStreamEventPayload,
@@ -1059,15 +1051,8 @@ export function SessionProvider({
});
if (!isFinalChunk) {
console.log(
`[Session] Buffered chunk ${chunkIndex} for group ${playbackGroupId}`
);
return;
}
console.log(
`[Session] Received final chunk for group ${playbackGroupId}, total chunks: ${buffer.length}`
);
buffer.sort((a, b) => a.chunkIndex - b.chunkIndex);
let playbackFailed = false;
@@ -1109,10 +1094,6 @@ export function SessionProvider({
offset += chunk.length;
}
console.log(
`[Session] Playing concatenated audio: ${buffer.length} chunks, ${totalSize} bytes`
);
const audioBlob = {
type: mimeType,
size: totalSize,
@@ -1264,11 +1245,7 @@ export function SessionProvider({
const transcriptText = message.payload.text.trim();
if (!transcriptText) {
console.log(
"[Session] Empty transcription (false positive) - ignoring"
);
} else {
console.log("[Session] Transcription received - stopping playback");
audioPlayer.stop();
setIsPlayingAudio(serverId, false);
setCurrentAssistantMessage(serverId, "");

View File

@@ -12,6 +12,7 @@ const VOICE_VAD_DETECTION_GRACE_PERIOD_MS = 700;
interface VoiceContextValue {
isVoiceMode: boolean;
isVoiceSwitching: boolean;
volume: number;
isMuted: boolean;
isDetecting: boolean;
@@ -57,7 +58,12 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
);
const realtimeSessionRef = useRef<SessionState | null>(null);
const [isVoiceMode, setIsVoiceMode] = useState(false);
const [isVoiceSwitching, setIsVoiceSwitching] = useState(false);
const bargeInPlaybackStopRef = useRef<number | null>(null);
const wasVoiceSocketConnectedRef = useRef(false);
const lastVoiceModeSyncedClientRef = useRef<SessionState["client"] | null>(null);
const voiceTransportReadyRef = useRef(false);
const voiceResyncInFlightRef = useRef(false);
const realtimeAudio = useSpeechmaticsAudio({
onSpeechStart: () => {
@@ -91,6 +97,10 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
console.log("[Voice] Speech ended");
},
onAudioSegment: ({ audioData, isLast }) => {
if (!voiceTransportReadyRef.current) {
console.log("[Voice] Skipping audio segment: voice transport not ready");
return;
}
console.log(
"[Voice] Sending audio segment, length:",
audioData.length,
@@ -134,6 +144,48 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
realtimeSessionRef.current = activeSession;
}, [activeSession]);
useEffect(() => {
const connected = activeSession?.connection.isConnected ?? false;
const client = activeSession?.client ?? null;
if (!connected) {
voiceTransportReadyRef.current = false;
}
if (!isVoiceMode || !activeServerId || !client) {
wasVoiceSocketConnectedRef.current = connected;
if (!isVoiceMode) {
lastVoiceModeSyncedClientRef.current = null;
}
voiceTransportReadyRef.current = false;
return;
}
const connectionRecovered = connected && !wasVoiceSocketConnectedRef.current;
const clientChanged = lastVoiceModeSyncedClientRef.current !== client;
if (connected && (connectionRecovered || clientChanged)) {
if (!voiceResyncInFlightRef.current) {
voiceResyncInFlightRef.current = true;
voiceTransportReadyRef.current = false;
setIsVoiceSwitching(true);
void client.setVoiceMode(true).then(
() => {
console.log("[Voice] Re-synced voice mode after reconnect");
lastVoiceModeSyncedClientRef.current = client;
voiceTransportReadyRef.current = true;
},
(error) => {
console.error("[Voice] Failed to re-sync voice mode:", error);
}
).finally(() => {
voiceResyncInFlightRef.current = false;
setIsVoiceSwitching(false);
});
}
}
wasVoiceSocketConnectedRef.current = connected;
}, [activeServerId, activeSession?.client, activeSession?.connection.isConnected, isVoiceMode]);
const isPlayingAudio = activeSession?.isPlayingAudio ?? false;
useEffect(() => {
@@ -155,56 +207,67 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
throw new Error(`Host ${serverId} is not connected`);
}
setIsVoiceSwitching(true);
voiceTransportReadyRef.current = false;
try {
realtimeSessionRef.current = session;
setActiveServerId(serverId);
await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => {
console.warn("[Voice] Failed to activate keep-awake:", error);
});
await session.audioPlayer?.warmup?.();
await realtimeAudio.start();
setIsVoiceMode(true);
console.log("[Voice] Mode enabled");
if (session?.client) {
await session.client.setVoiceMode(true);
} else {
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
}
await session.audioPlayer?.warmup?.();
await realtimeAudio.start();
voiceTransportReadyRef.current = true;
setIsVoiceMode(true);
lastVoiceModeSyncedClientRef.current = session.client;
console.log("[Voice] Mode enabled");
} catch (error: any) {
console.error("[Voice] Failed to start:", error);
await realtimeAudio.stop().catch(() => undefined);
setActiveServerId((current) => (current === serverId ? null : current));
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
throw error;
} finally {
setIsVoiceSwitching(false);
}
},
[getSession, realtimeAudio]
);
const stopVoice = useCallback(async () => {
setIsVoiceSwitching(true);
voiceTransportReadyRef.current = false;
try {
const session = realtimeSessionRef.current;
session?.audioPlayer?.stop();
if (session?.client) {
await session.client.setVoiceMode(false);
lastVoiceModeSyncedClientRef.current = session.client;
} else {
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
}
await realtimeAudio.stop();
setIsVoiceMode(false);
setActiveServerId(null);
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
console.log("[Voice] Mode disabled");
if (session?.client) {
await session.client.setVoiceMode(false);
} else {
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
}
} catch (error: any) {
console.error("[Voice] Failed to stop:", error);
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
throw error;
} finally {
setIsVoiceSwitching(false);
}
}, [realtimeAudio]);
const value: VoiceContextValue = {
isVoiceMode,
isVoiceSwitching,
volume: realtimeAudio.volume,
isMuted: realtimeAudio.isMuted,
isDetecting: realtimeAudio.isDetecting,

View File

@@ -4,7 +4,6 @@ import { useDaemonConnections } from "@/contexts/daemon-connections-context";
import { useSessionStore } from "@/stores/session-store";
import type { AgentDirectoryEntry } from "@/types/agent-directory";
import type { Agent } from "@/stores/session-store";
import { isPerfLoggingEnabled } from "@/utils/perf";
import { derivePendingPermissionKey, normalizeAgentSnapshot } from "@/utils/agent-snapshots";
export interface AggregatedAgent extends AgentDirectoryEntry {
@@ -44,14 +43,12 @@ export function useAggregatedAgents(): AggregatedAgentsResult {
);
const refreshAll = useCallback(() => {
console.log('[useAggregatedAgents] Manual refresh triggered for all sessions');
for (const [serverId, client] of Object.entries(sessionClients)) {
if (!client) {
continue;
}
void (async () => {
try {
console.log(`[useAggregatedAgents] Refreshing session ${serverId}`);
const agentsList = await client.fetchAgents({
filter: { labels: { ui: "true" } },
});
@@ -131,27 +128,20 @@ export function useAggregatedAgents(): AggregatedAgentsResult {
const hasAnyData = allAgents.length > 0;
// Check if any connection is currently loading
const connectingReasons: string[] = [];
const isConnecting = Array.from(connectionStates.entries()).some(([id, c]) => {
const shortId = id.substring(0, 20);
const isConnecting = Array.from(connectionStates.entries()).some(([, c]) => {
// First-time connection (never received agent list)
if (c.status === 'connecting' && !c.hasEverReceivedAgentList) {
connectingReasons.push(`${shortId}: first-time connecting`);
return true;
}
if (c.status === 'online' && !c.hasEverReceivedAgentList) {
connectingReasons.push(`${shortId}: online but no fetch_agents yet`);
return true;
}
// Reconnecting (have received agent list before)
if (c.status === 'connecting' && c.hasEverReceivedAgentList) {
connectingReasons.push(`${shortId}: reconnecting`);
return true;
}
if (c.status === 'online' && !c.agentListReady && c.hasEverReceivedAgentList) {
connectingReasons.push(`${shortId}: online but agentListReady=false (waiting for fetch_agents)`);
return true;
}
@@ -167,28 +157,6 @@ export function useAggregatedAgents(): AggregatedAgentsResult {
// isLoading: Generic loading flag (either initial or revalidating)
const isLoading = isConnecting;
if (isPerfLoggingEnabled()) {
const connectionStatesArray = Array.from(connectionStates.entries()).map(([id, state]) => ({
id: id.substring(0, 20) + (id.length > 20 ? '...' : ''),
status: state.status,
agentListReady: state.agentListReady,
hasEverReceivedAgentList: state.hasEverReceivedAgentList,
}));
console.log('[useAggregatedAgents] States:', {
hasAnyData,
isConnecting,
isInitialLoad,
isRevalidating,
totalConnectionStates: connectionStates.size,
connectingReasons: connectingReasons.length > 0 ? connectingReasons : 'none',
});
console.log('[useAggregatedAgents] Connection States Detail:',
JSON.stringify(connectionStatesArray, null, 2)
);
}
return {
agents: allAgents,
isLoading,

View File

@@ -210,9 +210,6 @@ export interface GroupedAgents {
const ACTIVE_GRACE_PERIOD_MS = 2 * 24 * 60 * 60 * 1000; // 2 days (temporary for screenshots)
// eslint-disable-next-line no-console
console.log('[agent-grouping] ACTIVE_GRACE_PERIOD_MS:', ACTIVE_GRACE_PERIOD_MS, 'ms =', ACTIVE_GRACE_PERIOD_MS / (1000 * 60 * 60), 'hours');
interface GroupAgentsOptions {
/**
* Optional function to read a remote URL for an agent.
@@ -252,17 +249,6 @@ export function groupAgents(
const isRecentlyActive = ageDiff < ACTIVE_GRACE_PERIOD_MS;
const isActive = isRunningOrAttention || isRecentlyActive;
// eslint-disable-next-line no-console
console.log('[agent-grouping] agent:', agent.title || agent.id, {
status: agent.status,
lastActivityAt: agent.lastActivityAt,
ageDiffHours: ageDiff / (1000 * 60 * 60),
gracePeriodHours: ACTIVE_GRACE_PERIOD_MS / (1000 * 60 * 60),
isRunningOrAttention,
isRecentlyActive,
isActive,
});
if (isActive) {
activeAgents.push(agent);
} else {

View File

@@ -1,21 +1,4 @@
const getIsDevEnvironment = (): boolean => {
const globalDev = (globalThis as { __DEV__?: boolean } | undefined)?.__DEV__;
if (typeof globalDev === "boolean") {
return globalDev;
}
if (typeof process !== "undefined" && process.env?.NODE_ENV) {
return process.env.NODE_ENV !== "production";
}
return false;
};
const shouldDisablePerfLogging =
typeof process !== "undefined" && process.env?.EXPO_PUBLIC_DISABLE_PERF_LOGGING === "1";
const shouldForcePerfLogging =
typeof process !== "undefined" && process.env?.EXPO_PUBLIC_ENABLE_PERF_LOGGING === "1";
const PERF_LOGGING_ENABLED =
(shouldForcePerfLogging || getIsDevEnvironment()) && !shouldDisablePerfLogging;
const PERF_LOGGING_ENABLED = false;
export const isPerfLoggingEnabled = (): boolean => PERF_LOGGING_ENABLED;

View File

@@ -201,6 +201,10 @@ type SpeechModelsListPayload = SpeechModelsListResponse["payload"];
type SpeechModelsDownloadPayload = SpeechModelsDownloadResponse["payload"];
type ListCommandsPayload = ListCommandsResponse["payload"];
type ExecuteCommandPayload = ExecuteCommandResponse["payload"];
type SetVoiceModePayload = Extract<
SessionOutboundMessage,
{ type: "set_voice_mode_response" }
>["payload"];
type AgentPermissionResolvedPayload = AgentPermissionResolvedMessage["payload"];
type ListTerminalsPayload = ListTerminalsResponse["payload"];
type CreateTerminalPayload = CreateTerminalResponse["payload"];
@@ -1285,8 +1289,31 @@ export class DaemonClient {
// Audio / Voice
// ============================================================================
async setVoiceMode(enabled: boolean, voiceAgentId?: string): Promise<void> {
this.sendSessionMessage({ type: "set_voice_mode", enabled, voiceAgentId });
async setVoiceMode(
enabled: boolean,
voiceAgentId?: string
): Promise<SetVoiceModePayload> {
const requestId = this.createRequestId();
const message = SessionInboundMessageSchema.parse({
type: "set_voice_mode",
enabled,
...(voiceAgentId ? { voiceAgentId } : {}),
requestId,
});
return this.sendRequest({
requestId,
message,
timeout: 10000,
select: (msg) => {
if (msg.type !== "set_voice_mode_response") {
return null;
}
if (msg.payload.requestId !== requestId) {
return null;
}
return msg.payload;
},
});
}
async sendVoiceAudioChunk(

View File

@@ -43,5 +43,83 @@ describe("TTSManager", () => {
expect((audioMsgs[1] as any).payload.chunkIndex).toBe(1);
expect((audioMsgs[1] as any).payload.isLastChunk).toBe(true);
});
});
it("splits long text into safe synthesis segments", async () => {
const calls: string[] = [];
const tts: TextToSpeechProvider = {
async synthesizeSpeech(text: string): Promise<{ stream: Readable; format: string }> {
calls.push(text);
return {
stream: Readable.from([Buffer.from("x")]),
format: "pcm;rate=24000",
};
},
};
const manager = new TTSManager("s1", pino({ level: "silent" }), tts);
const abort = new AbortController();
const longText = Array.from({ length: 180 })
.map((_, i) => `Sentence ${i + 1}.`)
.join(" ");
await manager.generateAndWaitForPlayback(
longText,
(msg) => {
if (msg.type === "audio_output") {
manager.confirmAudioPlayed(msg.payload.id);
}
},
abort.signal,
true
);
expect(calls.length).toBeGreaterThan(1);
expect(calls.every((text) => text.length <= 400)).toBe(true);
});
it("does not emit unhandled rejections when stream iteration fails", async () => {
const unhandled: unknown[] = [];
const onUnhandled = (reason: unknown) => {
unhandled.push(reason);
};
process.on("unhandledRejection", onUnhandled);
try {
const tts: TextToSpeechProvider = {
async synthesizeSpeech(): Promise<{ stream: Readable; format: string }> {
const stream = Readable.from(
(async function* () {
yield Buffer.from("a");
throw new Error("stream exploded");
})()
);
return {
stream,
format: "pcm;rate=24000",
};
},
};
const manager = new TTSManager("s1", pino({ level: "silent" }), tts);
const abort = new AbortController();
await expect(
manager.generateAndWaitForPlayback(
"hello",
(msg) => {
if (msg.type === "audio_output") {
manager.confirmAudioPlayed(msg.payload.id);
}
},
abort.signal,
true
)
).rejects.toThrow("stream exploded");
await new Promise((resolve) => setTimeout(resolve, 0));
expect(unhandled).toHaveLength(0);
} finally {
process.off("unhandledRejection", onUnhandled);
}
});
});

View File

@@ -10,6 +10,84 @@ interface PendingPlayback {
streamEnded: boolean;
}
const MAX_TTS_SEGMENT_CHARS = 400;
function splitTextForTts(text: string, maxChars: number): string[] {
const normalized = text.trim().replace(/\s+/g, " ");
if (!normalized) {
throw new Error("Cannot synthesize empty text");
}
if (normalized.length <= maxChars) {
return [normalized];
}
const parts: string[] = [];
const sentenceChunks = normalized.split(/(?<=[.!?])\s+/);
let current = "";
const pushCurrent = () => {
const trimmed = current.trim();
if (trimmed.length > 0) {
parts.push(trimmed);
}
current = "";
};
const appendFragment = (fragment: string) => {
const trimmed = fragment.trim();
if (!trimmed) {
return;
}
if (!current) {
current = trimmed;
return;
}
const candidate = `${current} ${trimmed}`;
if (candidate.length <= maxChars) {
current = candidate;
return;
}
pushCurrent();
current = trimmed;
};
const splitLargeFragment = (fragment: string): string[] => {
const trimmed = fragment.trim();
if (trimmed.length <= maxChars) {
return [trimmed];
}
const out: string[] = [];
let remaining = trimmed;
while (remaining.length > maxChars) {
let idx = remaining.lastIndexOf(" ", maxChars);
if (idx < Math.floor(maxChars * 0.5)) {
idx = maxChars;
}
out.push(remaining.slice(0, idx).trim());
remaining = remaining.slice(idx).trim();
}
if (remaining.length > 0) {
out.push(remaining);
}
return out;
};
for (const sentence of sentenceChunks) {
const fragments = splitLargeFragment(sentence);
for (const fragment of fragments) {
appendFragment(fragment);
}
}
pushCurrent();
return parts;
}
/**
* Per-session TTS manager
* Handles TTS audio generation and playback confirmation tracking
@@ -33,6 +111,37 @@ export class TTSManager {
emitMessage: (msg: SessionOutboundMessage) => void,
abortSignal: AbortSignal,
isVoiceMode: boolean
): Promise<void> {
this.logger.info(
{
isVoiceMode,
textLength: text.length,
text,
},
"TTS input text"
);
const segments = splitTextForTts(text, MAX_TTS_SEGMENT_CHARS);
for (const segment of segments) {
if (abortSignal.aborted) {
this.logger.debug("Aborted before generating segmented audio");
return;
}
await this.generateSegmentAndWaitForPlayback(
segment,
emitMessage,
abortSignal,
isVoiceMode
);
}
}
private async generateSegmentAndWaitForPlayback(
text: string,
emitMessage: (msg: SessionOutboundMessage) => void,
abortSignal: AbortSignal,
isVoiceMode: boolean
): Promise<void> {
if (!this.tts) {
throw new Error("TTS not configured");
@@ -121,11 +230,6 @@ export class TTSManager {
},
});
this.logger.debug(
{ chunkId, isLastChunk: next.done },
"Sent audio chunk"
);
chunkIndex += 1;
if (next.done) {
@@ -151,7 +255,6 @@ export class TTSManager {
} else {
this.logger.error({ err: error }, "Error streaming audio");
this.pendingPlaybacks.delete(audioId);
pendingPlayback.reject(error as Error);
throw error;
}
} finally {

View File

@@ -147,8 +147,16 @@ describe("daemon client E2E", () => {
});
expect(Array.isArray(voiceAgents)).toBe(true);
await expect(ctx.client.setVoiceMode(true)).resolves.toBeUndefined();
await expect(ctx.client.setVoiceMode(false)).resolves.toBeUndefined();
await expect(ctx.client.setVoiceMode(true)).resolves.toMatchObject({
enabled: true,
accepted: true,
error: null,
});
await expect(ctx.client.setVoiceMode(false)).resolves.toMatchObject({
enabled: false,
accepted: true,
error: null,
});
await ctx.client.deleteAgent(randomUUID());
}, 30000);

View File

@@ -879,7 +879,7 @@ export class Session {
break;
case "set_voice_mode":
await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId);
await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId, msg.requestId);
break;
case "send_agent_message_request":
@@ -1259,7 +1259,8 @@ export class Session {
*/
private async handleSetVoiceMode(
enabled: boolean,
voiceAgentId?: string
voiceAgentId?: string,
requestId?: string
): Promise<void> {
if (enabled) {
let normalizedVoiceAgentId: string | null = null;
@@ -1282,6 +1283,18 @@ export class Session {
},
"Voice mode enabled (agent-backed)"
);
if (requestId) {
this.emit({
type: "set_voice_mode_response",
payload: {
requestId,
enabled: true,
voiceAgentId: this.voiceAssistantAgentId,
accepted: true,
error: null,
},
});
}
return;
}
@@ -1290,6 +1303,18 @@ export class Session {
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
"Voice mode disabled (agent-backed)"
);
if (requestId) {
this.emit({
type: "set_voice_mode_response",
payload: {
requestId,
enabled: false,
voiceAgentId: this.voiceAssistantAgentId,
accepted: true,
error: null,
},
});
}
}
private parseVoiceAgentId(rawId: string, source: string): string {
@@ -2471,7 +2496,6 @@ export class Session {
lastActivityAt: string;
appVisible: boolean;
}): void {
this.sessionLogger.debug({ heartbeat: msg }, "Client heartbeat");
this.clientActivity = {
deviceType: msg.deviceType,
focusedAgentId: msg.focusedAgentId,
@@ -4152,6 +4176,12 @@ export class Session {
private async handleAudioChunk(
msg: Extract<SessionInboundMessage, { type: "voice_audio_chunk" }>
): Promise<void> {
if (!this.isVoiceMode) {
this.sessionLogger.warn(
"Received voice_audio_chunk while voice mode is disabled; transcript will be emitted but voice assistant turn is skipped"
);
}
await this.handleVoiceSpeechStart();
const chunkBuffer = Buffer.from(msg.audio, "base64");
@@ -4193,11 +4223,6 @@ export class Session {
this.audioBuffer.totalPCMBytes += chunkBuffer.length;
}
this.sessionLogger.debug(
{ bytes: chunkBuffer.length, chunks: this.audioBuffer.chunks.length, pcmBytes: this.audioBuffer.totalPCMBytes },
`Buffered audio chunk (${chunkBuffer.length} bytes, chunks: ${this.audioBuffer.chunks.length}${this.audioBuffer.isPCM ? `, PCM bytes: ${this.audioBuffer.totalPCMBytes}` : ""})`
);
// In voice mode, only process audio when the user has finished speaking (isLast = true)
// This prevents partial transcriptions from being sent to the LLM
if (this.isVoiceMode) {
@@ -4343,6 +4368,15 @@ export class Session {
});
const transcriptText = result.text.trim();
this.sessionLogger.info(
{
requestId,
isVoiceMode: this.isVoiceMode,
transcriptLength: transcriptText.length,
transcript: transcriptText,
},
"Transcription result"
);
// Emit transcription result
this.emit({
@@ -4408,8 +4442,17 @@ export class Session {
},
});
// Set phase to LLM and process (TTS enabled in voice mode for voice agents)
this.clearSpeechInProgress("transcription complete");
if (!this.isVoiceMode) {
this.sessionLogger.debug(
{ requestId },
"Skipping voice agent processing because voice mode is disabled"
);
this.setPhase("idle");
return;
}
// Set phase to LLM and process (TTS enabled in voice mode for voice agents)
this.setPhase("llm");
this.currentStreamPromise = this.processVoiceTurn(result.text);
await this.currentStreamPromise;
@@ -4560,7 +4603,6 @@ export class Session {
this.agentManager.recordUserMessage(agentId, userText);
let sawSpeakToolCall = false;
const assistantTextChunks: string[] = [];
const iterator = this.agentManager.streamAgent(agentId, prompt);
for await (const event of iterator) {
if (event.type === "turn_failed") {
@@ -4572,9 +4614,6 @@ export class Session {
sawSpeakToolCall = true;
}
}
if (event.item.type === "assistant_message" && event.item.text.trim().length > 0) {
assistantTextChunks.push(event.item.text.trim());
}
}
if (event.type === "permission_requested") {
if (this.shouldAllowVoicePermission(event.request)) {
@@ -4594,26 +4633,10 @@ export class Session {
}
}
if (!sawSpeakToolCall && assistantTextChunks.length > 0) {
const fallbackText = assistantTextChunks.join(" ").trim();
await this.ttsManager.generateAndWaitForPlayback(
fallbackText,
(msg) => this.emit(msg),
this.abortController.signal,
true
);
this.emit({
type: "activity_log",
payload: {
id: uuidv4(),
timestamp: new Date(),
type: "assistant",
content: fallbackText,
},
});
if (!sawSpeakToolCall) {
this.sessionLogger.warn(
{ voiceAssistantAgentId: agentId },
"Voice agent responded without speak tool; used fallback TTS from assistant text"
"Voice agent turn completed without speak tool call; no audio playback emitted"
);
}
}

View File

@@ -389,7 +389,14 @@ export class VoiceAssistantWebSocketServer {
? { sessionMessageType: message.message.type }
: {}),
};
this.logger.debug(messageSummary, "Received message");
const isSessionNoise =
message.type === "session" &&
(message.message.type === "client_heartbeat" ||
message.message.type === "voice_audio_chunk" ||
message.message.type === "dictation_stream_chunk");
if (!isSessionNoise) {
this.logger.debug(messageSummary, "Received message");
}
if (message.type === "ping") {
this.sendToClient(ws, { type: "pong" });

View File

@@ -348,6 +348,7 @@ export const SetVoiceModeMessageSchema = z.object({
type: z.literal("set_voice_mode"),
enabled: z.boolean(),
voiceAgentId: z.string().optional(),
requestId: z.string().optional(),
});
export const SendAgentMessageSchema = z.object({
@@ -571,6 +572,17 @@ export const SetAgentThinkingResponseMessageSchema = z.object({
}),
});
export const SetVoiceModeResponseMessageSchema = z.object({
type: z.literal("set_voice_mode_response"),
payload: z.object({
requestId: z.string(),
enabled: z.boolean(),
voiceAgentId: z.string().nullable(),
accepted: z.boolean(),
error: z.string().nullable(),
}),
});
export const AgentPermissionResponseMessageSchema = z.object({
type: z.literal("agent_permission_response"),
agentId: z.string(),
@@ -1627,6 +1639,7 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
FetchAgentsResponseMessageSchema,
FetchAgentResponseMessageSchema,
SendAgentMessageResponseMessageSchema,
SetVoiceModeResponseMessageSchema,
SetAgentModeResponseMessageSchema,
SetAgentModelResponseMessageSchema,
SetAgentThinkingResponseMessageSchema,
@@ -1690,6 +1703,9 @@ export type FetchAgentResponseMessage = z.infer<
export type SendAgentMessageResponseMessage = z.infer<
typeof SendAgentMessageResponseMessageSchema
>;
export type SetVoiceModeResponseMessage = z.infer<
typeof SetVoiceModeResponseMessageSchema
>;
export type WaitForFinishResponseMessage = z.infer<
typeof WaitForFinishResponseMessageSchema
>;