mirror of
https://github.com/getpaseo/paseo.git
synced 2026-07-29 12:01:31 +00:00
Update files
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import { useCallback, useMemo, useState, useEffect } from "react";
|
||||
import { View, Pressable, Text, Platform, Alert } from "react-native";
|
||||
import { View, Pressable, Text, Platform, Alert, ActivityIndicator } from "react-native";
|
||||
import { useSafeAreaInsets } from "react-native-safe-area-context";
|
||||
import Animated, {
|
||||
useAnimatedStyle,
|
||||
@@ -57,7 +57,7 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
||||
const trafficLightPadding = useTrafficLightPadding();
|
||||
const dragHandlers = useTauriDragHandlers();
|
||||
const { connectionStates } = useDaemonConnections();
|
||||
const { isVoiceMode, startVoice, stopVoice } = useVoice();
|
||||
const { isVoiceMode, isVoiceSwitching, startVoice, stopVoice } = useVoice();
|
||||
const [showVoiceHostPicker, setShowVoiceHostPicker] = useState(false);
|
||||
|
||||
// Track user-initiated refresh to avoid showing spinner on background revalidation
|
||||
@@ -148,6 +148,9 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
||||
const hasAnyConfiguredHosts = connectionStates.size > 0;
|
||||
|
||||
const handleToggleVoice = useCallback(() => {
|
||||
if (isVoiceSwitching) {
|
||||
return;
|
||||
}
|
||||
if (isVoiceMode) {
|
||||
void stopVoice().catch((error) => {
|
||||
console.error("[SlidingSidebar] Failed to stop voice", error);
|
||||
@@ -181,7 +184,7 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
||||
}
|
||||
|
||||
setShowVoiceHostPicker(true);
|
||||
}, [hasAnyConfiguredHosts, isVoiceMode, startVoice, stopVoice, voiceEligibleHosts]);
|
||||
}, [hasAnyConfiguredHosts, isVoiceMode, isVoiceSwitching, startVoice, stopVoice, voiceEligibleHosts]);
|
||||
|
||||
const handleSelectVoiceHost = useCallback(
|
||||
(serverId: string) => {
|
||||
@@ -324,11 +327,13 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
||||
accessible
|
||||
accessibilityLabel="Voice"
|
||||
accessibilityRole="button"
|
||||
disabled={isVoiceSwitching}
|
||||
onPress={handleToggleVoice}
|
||||
>
|
||||
{({ hovered }) => (
|
||||
<AudioLines
|
||||
size={20}
|
||||
isVoiceSwitching ? (
|
||||
<ActivityIndicator
|
||||
size="small"
|
||||
color={
|
||||
isVoiceMode
|
||||
? theme.colors.foreground
|
||||
@@ -337,7 +342,19 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
||||
: theme.colors.foregroundMuted
|
||||
}
|
||||
/>
|
||||
)}
|
||||
) : (
|
||||
<AudioLines
|
||||
size={20}
|
||||
color={
|
||||
isVoiceMode
|
||||
? theme.colors.foreground
|
||||
: hovered
|
||||
? theme.colors.foreground
|
||||
: theme.colors.foregroundMuted
|
||||
}
|
||||
/>
|
||||
)
|
||||
)}
|
||||
</Pressable>
|
||||
<Pressable
|
||||
style={styles.footerIconButton}
|
||||
@@ -448,19 +465,33 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
|
||||
accessible
|
||||
accessibilityLabel="Voice"
|
||||
accessibilityRole="button"
|
||||
disabled={isVoiceSwitching}
|
||||
onPress={handleToggleVoice}
|
||||
>
|
||||
{({ hovered }) => (
|
||||
<AudioLines
|
||||
size={20}
|
||||
color={
|
||||
isVoiceMode
|
||||
? theme.colors.foreground
|
||||
: hovered
|
||||
isVoiceSwitching ? (
|
||||
<ActivityIndicator
|
||||
size="small"
|
||||
color={
|
||||
isVoiceMode
|
||||
? theme.colors.foreground
|
||||
: theme.colors.foregroundMuted
|
||||
}
|
||||
/>
|
||||
: hovered
|
||||
? theme.colors.foreground
|
||||
: theme.colors.foregroundMuted
|
||||
}
|
||||
/>
|
||||
) : (
|
||||
<AudioLines
|
||||
size={20}
|
||||
color={
|
||||
isVoiceMode
|
||||
? theme.colors.foreground
|
||||
: hovered
|
||||
? theme.colors.foreground
|
||||
: theme.colors.foregroundMuted
|
||||
}
|
||||
/>
|
||||
)
|
||||
)}
|
||||
</Pressable>
|
||||
<Pressable
|
||||
|
||||
@@ -872,10 +872,7 @@ export function SessionProvider({
|
||||
const { agentId, event, timestamp } = message.payload;
|
||||
const parsedTimestamp = new Date(timestamp);
|
||||
|
||||
console.log("[Session] agent_stream", { agentId, event, timestamp });
|
||||
|
||||
if (event.type === "attention_required") {
|
||||
console.log("[Session] attention_required", { agentId, shouldNotify: event.shouldNotify, reason: event.reason });
|
||||
if (event.shouldNotify) {
|
||||
notifyAgentAttention({
|
||||
agentId,
|
||||
@@ -926,11 +923,6 @@ export function SessionProvider({
|
||||
if (message.type !== "agent_stream_snapshot") return;
|
||||
const { agentId, events } = message.payload;
|
||||
|
||||
console.log("[Session] agent_stream_snapshot", {
|
||||
agentId,
|
||||
eventCount: events.length,
|
||||
});
|
||||
|
||||
const hydrated = hydrateStreamState(
|
||||
events.map(({ event, timestamp }) => ({
|
||||
event: event as AgentStreamEventPayload,
|
||||
@@ -1059,15 +1051,8 @@ export function SessionProvider({
|
||||
});
|
||||
|
||||
if (!isFinalChunk) {
|
||||
console.log(
|
||||
`[Session] Buffered chunk ${chunkIndex} for group ${playbackGroupId}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[Session] Received final chunk for group ${playbackGroupId}, total chunks: ${buffer.length}`
|
||||
);
|
||||
buffer.sort((a, b) => a.chunkIndex - b.chunkIndex);
|
||||
|
||||
let playbackFailed = false;
|
||||
@@ -1109,10 +1094,6 @@ export function SessionProvider({
|
||||
offset += chunk.length;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[Session] Playing concatenated audio: ${buffer.length} chunks, ${totalSize} bytes`
|
||||
);
|
||||
|
||||
const audioBlob = {
|
||||
type: mimeType,
|
||||
size: totalSize,
|
||||
@@ -1264,11 +1245,7 @@ export function SessionProvider({
|
||||
const transcriptText = message.payload.text.trim();
|
||||
|
||||
if (!transcriptText) {
|
||||
console.log(
|
||||
"[Session] Empty transcription (false positive) - ignoring"
|
||||
);
|
||||
} else {
|
||||
console.log("[Session] Transcription received - stopping playback");
|
||||
audioPlayer.stop();
|
||||
setIsPlayingAudio(serverId, false);
|
||||
setCurrentAssistantMessage(serverId, "");
|
||||
|
||||
@@ -12,6 +12,7 @@ const VOICE_VAD_DETECTION_GRACE_PERIOD_MS = 700;
|
||||
|
||||
interface VoiceContextValue {
|
||||
isVoiceMode: boolean;
|
||||
isVoiceSwitching: boolean;
|
||||
volume: number;
|
||||
isMuted: boolean;
|
||||
isDetecting: boolean;
|
||||
@@ -57,7 +58,12 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
||||
);
|
||||
const realtimeSessionRef = useRef<SessionState | null>(null);
|
||||
const [isVoiceMode, setIsVoiceMode] = useState(false);
|
||||
const [isVoiceSwitching, setIsVoiceSwitching] = useState(false);
|
||||
const bargeInPlaybackStopRef = useRef<number | null>(null);
|
||||
const wasVoiceSocketConnectedRef = useRef(false);
|
||||
const lastVoiceModeSyncedClientRef = useRef<SessionState["client"] | null>(null);
|
||||
const voiceTransportReadyRef = useRef(false);
|
||||
const voiceResyncInFlightRef = useRef(false);
|
||||
|
||||
const realtimeAudio = useSpeechmaticsAudio({
|
||||
onSpeechStart: () => {
|
||||
@@ -91,6 +97,10 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
||||
console.log("[Voice] Speech ended");
|
||||
},
|
||||
onAudioSegment: ({ audioData, isLast }) => {
|
||||
if (!voiceTransportReadyRef.current) {
|
||||
console.log("[Voice] Skipping audio segment: voice transport not ready");
|
||||
return;
|
||||
}
|
||||
console.log(
|
||||
"[Voice] Sending audio segment, length:",
|
||||
audioData.length,
|
||||
@@ -134,6 +144,48 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
||||
realtimeSessionRef.current = activeSession;
|
||||
}, [activeSession]);
|
||||
|
||||
useEffect(() => {
|
||||
const connected = activeSession?.connection.isConnected ?? false;
|
||||
const client = activeSession?.client ?? null;
|
||||
if (!connected) {
|
||||
voiceTransportReadyRef.current = false;
|
||||
}
|
||||
|
||||
if (!isVoiceMode || !activeServerId || !client) {
|
||||
wasVoiceSocketConnectedRef.current = connected;
|
||||
if (!isVoiceMode) {
|
||||
lastVoiceModeSyncedClientRef.current = null;
|
||||
}
|
||||
voiceTransportReadyRef.current = false;
|
||||
return;
|
||||
}
|
||||
|
||||
const connectionRecovered = connected && !wasVoiceSocketConnectedRef.current;
|
||||
const clientChanged = lastVoiceModeSyncedClientRef.current !== client;
|
||||
if (connected && (connectionRecovered || clientChanged)) {
|
||||
if (!voiceResyncInFlightRef.current) {
|
||||
voiceResyncInFlightRef.current = true;
|
||||
voiceTransportReadyRef.current = false;
|
||||
setIsVoiceSwitching(true);
|
||||
void client.setVoiceMode(true).then(
|
||||
() => {
|
||||
console.log("[Voice] Re-synced voice mode after reconnect");
|
||||
lastVoiceModeSyncedClientRef.current = client;
|
||||
voiceTransportReadyRef.current = true;
|
||||
},
|
||||
(error) => {
|
||||
console.error("[Voice] Failed to re-sync voice mode:", error);
|
||||
}
|
||||
).finally(() => {
|
||||
voiceResyncInFlightRef.current = false;
|
||||
setIsVoiceSwitching(false);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
wasVoiceSocketConnectedRef.current = connected;
|
||||
}, [activeServerId, activeSession?.client, activeSession?.connection.isConnected, isVoiceMode]);
|
||||
|
||||
const isPlayingAudio = activeSession?.isPlayingAudio ?? false;
|
||||
|
||||
useEffect(() => {
|
||||
@@ -155,56 +207,67 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
||||
throw new Error(`Host ${serverId} is not connected`);
|
||||
}
|
||||
|
||||
setIsVoiceSwitching(true);
|
||||
voiceTransportReadyRef.current = false;
|
||||
try {
|
||||
realtimeSessionRef.current = session;
|
||||
setActiveServerId(serverId);
|
||||
await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => {
|
||||
console.warn("[Voice] Failed to activate keep-awake:", error);
|
||||
});
|
||||
await session.audioPlayer?.warmup?.();
|
||||
await realtimeAudio.start();
|
||||
setIsVoiceMode(true);
|
||||
console.log("[Voice] Mode enabled");
|
||||
|
||||
if (session?.client) {
|
||||
await session.client.setVoiceMode(true);
|
||||
} else {
|
||||
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
||||
}
|
||||
await session.audioPlayer?.warmup?.();
|
||||
await realtimeAudio.start();
|
||||
voiceTransportReadyRef.current = true;
|
||||
setIsVoiceMode(true);
|
||||
lastVoiceModeSyncedClientRef.current = session.client;
|
||||
console.log("[Voice] Mode enabled");
|
||||
} catch (error: any) {
|
||||
console.error("[Voice] Failed to start:", error);
|
||||
await realtimeAudio.stop().catch(() => undefined);
|
||||
setActiveServerId((current) => (current === serverId ? null : current));
|
||||
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
setIsVoiceSwitching(false);
|
||||
}
|
||||
},
|
||||
[getSession, realtimeAudio]
|
||||
);
|
||||
|
||||
const stopVoice = useCallback(async () => {
|
||||
setIsVoiceSwitching(true);
|
||||
voiceTransportReadyRef.current = false;
|
||||
try {
|
||||
const session = realtimeSessionRef.current;
|
||||
session?.audioPlayer?.stop();
|
||||
if (session?.client) {
|
||||
await session.client.setVoiceMode(false);
|
||||
lastVoiceModeSyncedClientRef.current = session.client;
|
||||
} else {
|
||||
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
||||
}
|
||||
await realtimeAudio.stop();
|
||||
setIsVoiceMode(false);
|
||||
setActiveServerId(null);
|
||||
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||
console.log("[Voice] Mode disabled");
|
||||
|
||||
if (session?.client) {
|
||||
await session.client.setVoiceMode(false);
|
||||
} else {
|
||||
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error("[Voice] Failed to stop:", error);
|
||||
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
setIsVoiceSwitching(false);
|
||||
}
|
||||
}, [realtimeAudio]);
|
||||
|
||||
const value: VoiceContextValue = {
|
||||
isVoiceMode,
|
||||
isVoiceSwitching,
|
||||
volume: realtimeAudio.volume,
|
||||
isMuted: realtimeAudio.isMuted,
|
||||
isDetecting: realtimeAudio.isDetecting,
|
||||
|
||||
@@ -4,7 +4,6 @@ import { useDaemonConnections } from "@/contexts/daemon-connections-context";
|
||||
import { useSessionStore } from "@/stores/session-store";
|
||||
import type { AgentDirectoryEntry } from "@/types/agent-directory";
|
||||
import type { Agent } from "@/stores/session-store";
|
||||
import { isPerfLoggingEnabled } from "@/utils/perf";
|
||||
import { derivePendingPermissionKey, normalizeAgentSnapshot } from "@/utils/agent-snapshots";
|
||||
|
||||
export interface AggregatedAgent extends AgentDirectoryEntry {
|
||||
@@ -44,14 +43,12 @@ export function useAggregatedAgents(): AggregatedAgentsResult {
|
||||
);
|
||||
|
||||
const refreshAll = useCallback(() => {
|
||||
console.log('[useAggregatedAgents] Manual refresh triggered for all sessions');
|
||||
for (const [serverId, client] of Object.entries(sessionClients)) {
|
||||
if (!client) {
|
||||
continue;
|
||||
}
|
||||
void (async () => {
|
||||
try {
|
||||
console.log(`[useAggregatedAgents] Refreshing session ${serverId}`);
|
||||
const agentsList = await client.fetchAgents({
|
||||
filter: { labels: { ui: "true" } },
|
||||
});
|
||||
@@ -131,27 +128,20 @@ export function useAggregatedAgents(): AggregatedAgentsResult {
|
||||
const hasAnyData = allAgents.length > 0;
|
||||
|
||||
// Check if any connection is currently loading
|
||||
const connectingReasons: string[] = [];
|
||||
const isConnecting = Array.from(connectionStates.entries()).some(([id, c]) => {
|
||||
const shortId = id.substring(0, 20);
|
||||
|
||||
const isConnecting = Array.from(connectionStates.entries()).some(([, c]) => {
|
||||
// First-time connection (never received agent list)
|
||||
if (c.status === 'connecting' && !c.hasEverReceivedAgentList) {
|
||||
connectingReasons.push(`${shortId}: first-time connecting`);
|
||||
return true;
|
||||
}
|
||||
if (c.status === 'online' && !c.hasEverReceivedAgentList) {
|
||||
connectingReasons.push(`${shortId}: online but no fetch_agents yet`);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Reconnecting (have received agent list before)
|
||||
if (c.status === 'connecting' && c.hasEverReceivedAgentList) {
|
||||
connectingReasons.push(`${shortId}: reconnecting`);
|
||||
return true;
|
||||
}
|
||||
if (c.status === 'online' && !c.agentListReady && c.hasEverReceivedAgentList) {
|
||||
connectingReasons.push(`${shortId}: online but agentListReady=false (waiting for fetch_agents)`);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -167,28 +157,6 @@ export function useAggregatedAgents(): AggregatedAgentsResult {
|
||||
// isLoading: Generic loading flag (either initial or revalidating)
|
||||
const isLoading = isConnecting;
|
||||
|
||||
if (isPerfLoggingEnabled()) {
|
||||
const connectionStatesArray = Array.from(connectionStates.entries()).map(([id, state]) => ({
|
||||
id: id.substring(0, 20) + (id.length > 20 ? '...' : ''),
|
||||
status: state.status,
|
||||
agentListReady: state.agentListReady,
|
||||
hasEverReceivedAgentList: state.hasEverReceivedAgentList,
|
||||
}));
|
||||
|
||||
console.log('[useAggregatedAgents] States:', {
|
||||
hasAnyData,
|
||||
isConnecting,
|
||||
isInitialLoad,
|
||||
isRevalidating,
|
||||
totalConnectionStates: connectionStates.size,
|
||||
connectingReasons: connectingReasons.length > 0 ? connectingReasons : 'none',
|
||||
});
|
||||
|
||||
console.log('[useAggregatedAgents] Connection States Detail:',
|
||||
JSON.stringify(connectionStatesArray, null, 2)
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
agents: allAgents,
|
||||
isLoading,
|
||||
|
||||
@@ -210,9 +210,6 @@ export interface GroupedAgents {
|
||||
|
||||
const ACTIVE_GRACE_PERIOD_MS = 2 * 24 * 60 * 60 * 1000; // 2 days (temporary for screenshots)
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('[agent-grouping] ACTIVE_GRACE_PERIOD_MS:', ACTIVE_GRACE_PERIOD_MS, 'ms =', ACTIVE_GRACE_PERIOD_MS / (1000 * 60 * 60), 'hours');
|
||||
|
||||
interface GroupAgentsOptions {
|
||||
/**
|
||||
* Optional function to read a remote URL for an agent.
|
||||
@@ -252,17 +249,6 @@ export function groupAgents(
|
||||
const isRecentlyActive = ageDiff < ACTIVE_GRACE_PERIOD_MS;
|
||||
const isActive = isRunningOrAttention || isRecentlyActive;
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('[agent-grouping] agent:', agent.title || agent.id, {
|
||||
status: agent.status,
|
||||
lastActivityAt: agent.lastActivityAt,
|
||||
ageDiffHours: ageDiff / (1000 * 60 * 60),
|
||||
gracePeriodHours: ACTIVE_GRACE_PERIOD_MS / (1000 * 60 * 60),
|
||||
isRunningOrAttention,
|
||||
isRecentlyActive,
|
||||
isActive,
|
||||
});
|
||||
|
||||
if (isActive) {
|
||||
activeAgents.push(agent);
|
||||
} else {
|
||||
|
||||
@@ -1,21 +1,4 @@
|
||||
const getIsDevEnvironment = (): boolean => {
|
||||
const globalDev = (globalThis as { __DEV__?: boolean } | undefined)?.__DEV__;
|
||||
if (typeof globalDev === "boolean") {
|
||||
return globalDev;
|
||||
}
|
||||
if (typeof process !== "undefined" && process.env?.NODE_ENV) {
|
||||
return process.env.NODE_ENV !== "production";
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const shouldDisablePerfLogging =
|
||||
typeof process !== "undefined" && process.env?.EXPO_PUBLIC_DISABLE_PERF_LOGGING === "1";
|
||||
const shouldForcePerfLogging =
|
||||
typeof process !== "undefined" && process.env?.EXPO_PUBLIC_ENABLE_PERF_LOGGING === "1";
|
||||
|
||||
const PERF_LOGGING_ENABLED =
|
||||
(shouldForcePerfLogging || getIsDevEnvironment()) && !shouldDisablePerfLogging;
|
||||
const PERF_LOGGING_ENABLED = false;
|
||||
|
||||
export const isPerfLoggingEnabled = (): boolean => PERF_LOGGING_ENABLED;
|
||||
|
||||
|
||||
@@ -201,6 +201,10 @@ type SpeechModelsListPayload = SpeechModelsListResponse["payload"];
|
||||
type SpeechModelsDownloadPayload = SpeechModelsDownloadResponse["payload"];
|
||||
type ListCommandsPayload = ListCommandsResponse["payload"];
|
||||
type ExecuteCommandPayload = ExecuteCommandResponse["payload"];
|
||||
type SetVoiceModePayload = Extract<
|
||||
SessionOutboundMessage,
|
||||
{ type: "set_voice_mode_response" }
|
||||
>["payload"];
|
||||
type AgentPermissionResolvedPayload = AgentPermissionResolvedMessage["payload"];
|
||||
type ListTerminalsPayload = ListTerminalsResponse["payload"];
|
||||
type CreateTerminalPayload = CreateTerminalResponse["payload"];
|
||||
@@ -1285,8 +1289,31 @@ export class DaemonClient {
|
||||
// Audio / Voice
|
||||
// ============================================================================
|
||||
|
||||
async setVoiceMode(enabled: boolean, voiceAgentId?: string): Promise<void> {
|
||||
this.sendSessionMessage({ type: "set_voice_mode", enabled, voiceAgentId });
|
||||
async setVoiceMode(
|
||||
enabled: boolean,
|
||||
voiceAgentId?: string
|
||||
): Promise<SetVoiceModePayload> {
|
||||
const requestId = this.createRequestId();
|
||||
const message = SessionInboundMessageSchema.parse({
|
||||
type: "set_voice_mode",
|
||||
enabled,
|
||||
...(voiceAgentId ? { voiceAgentId } : {}),
|
||||
requestId,
|
||||
});
|
||||
return this.sendRequest({
|
||||
requestId,
|
||||
message,
|
||||
timeout: 10000,
|
||||
select: (msg) => {
|
||||
if (msg.type !== "set_voice_mode_response") {
|
||||
return null;
|
||||
}
|
||||
if (msg.payload.requestId !== requestId) {
|
||||
return null;
|
||||
}
|
||||
return msg.payload;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async sendVoiceAudioChunk(
|
||||
|
||||
@@ -43,5 +43,83 @@ describe("TTSManager", () => {
|
||||
expect((audioMsgs[1] as any).payload.chunkIndex).toBe(1);
|
||||
expect((audioMsgs[1] as any).payload.isLastChunk).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
it("splits long text into safe synthesis segments", async () => {
|
||||
const calls: string[] = [];
|
||||
const tts: TextToSpeechProvider = {
|
||||
async synthesizeSpeech(text: string): Promise<{ stream: Readable; format: string }> {
|
||||
calls.push(text);
|
||||
return {
|
||||
stream: Readable.from([Buffer.from("x")]),
|
||||
format: "pcm;rate=24000",
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
const manager = new TTSManager("s1", pino({ level: "silent" }), tts);
|
||||
const abort = new AbortController();
|
||||
const longText = Array.from({ length: 180 })
|
||||
.map((_, i) => `Sentence ${i + 1}.`)
|
||||
.join(" ");
|
||||
|
||||
await manager.generateAndWaitForPlayback(
|
||||
longText,
|
||||
(msg) => {
|
||||
if (msg.type === "audio_output") {
|
||||
manager.confirmAudioPlayed(msg.payload.id);
|
||||
}
|
||||
},
|
||||
abort.signal,
|
||||
true
|
||||
);
|
||||
|
||||
expect(calls.length).toBeGreaterThan(1);
|
||||
expect(calls.every((text) => text.length <= 400)).toBe(true);
|
||||
});
|
||||
|
||||
it("does not emit unhandled rejections when stream iteration fails", async () => {
|
||||
const unhandled: unknown[] = [];
|
||||
const onUnhandled = (reason: unknown) => {
|
||||
unhandled.push(reason);
|
||||
};
|
||||
process.on("unhandledRejection", onUnhandled);
|
||||
|
||||
try {
|
||||
const tts: TextToSpeechProvider = {
|
||||
async synthesizeSpeech(): Promise<{ stream: Readable; format: string }> {
|
||||
const stream = Readable.from(
|
||||
(async function* () {
|
||||
yield Buffer.from("a");
|
||||
throw new Error("stream exploded");
|
||||
})()
|
||||
);
|
||||
return {
|
||||
stream,
|
||||
format: "pcm;rate=24000",
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
const manager = new TTSManager("s1", pino({ level: "silent" }), tts);
|
||||
const abort = new AbortController();
|
||||
|
||||
await expect(
|
||||
manager.generateAndWaitForPlayback(
|
||||
"hello",
|
||||
(msg) => {
|
||||
if (msg.type === "audio_output") {
|
||||
manager.confirmAudioPlayed(msg.payload.id);
|
||||
}
|
||||
},
|
||||
abort.signal,
|
||||
true
|
||||
)
|
||||
).rejects.toThrow("stream exploded");
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(unhandled).toHaveLength(0);
|
||||
} finally {
|
||||
process.off("unhandledRejection", onUnhandled);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -10,6 +10,84 @@ interface PendingPlayback {
|
||||
streamEnded: boolean;
|
||||
}
|
||||
|
||||
const MAX_TTS_SEGMENT_CHARS = 400;
|
||||
|
||||
function splitTextForTts(text: string, maxChars: number): string[] {
|
||||
const normalized = text.trim().replace(/\s+/g, " ");
|
||||
if (!normalized) {
|
||||
throw new Error("Cannot synthesize empty text");
|
||||
}
|
||||
|
||||
if (normalized.length <= maxChars) {
|
||||
return [normalized];
|
||||
}
|
||||
|
||||
const parts: string[] = [];
|
||||
const sentenceChunks = normalized.split(/(?<=[.!?])\s+/);
|
||||
|
||||
let current = "";
|
||||
const pushCurrent = () => {
|
||||
const trimmed = current.trim();
|
||||
if (trimmed.length > 0) {
|
||||
parts.push(trimmed);
|
||||
}
|
||||
current = "";
|
||||
};
|
||||
|
||||
const appendFragment = (fragment: string) => {
|
||||
const trimmed = fragment.trim();
|
||||
if (!trimmed) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!current) {
|
||||
current = trimmed;
|
||||
return;
|
||||
}
|
||||
|
||||
const candidate = `${current} ${trimmed}`;
|
||||
if (candidate.length <= maxChars) {
|
||||
current = candidate;
|
||||
return;
|
||||
}
|
||||
|
||||
pushCurrent();
|
||||
current = trimmed;
|
||||
};
|
||||
|
||||
const splitLargeFragment = (fragment: string): string[] => {
|
||||
const trimmed = fragment.trim();
|
||||
if (trimmed.length <= maxChars) {
|
||||
return [trimmed];
|
||||
}
|
||||
|
||||
const out: string[] = [];
|
||||
let remaining = trimmed;
|
||||
while (remaining.length > maxChars) {
|
||||
let idx = remaining.lastIndexOf(" ", maxChars);
|
||||
if (idx < Math.floor(maxChars * 0.5)) {
|
||||
idx = maxChars;
|
||||
}
|
||||
out.push(remaining.slice(0, idx).trim());
|
||||
remaining = remaining.slice(idx).trim();
|
||||
}
|
||||
if (remaining.length > 0) {
|
||||
out.push(remaining);
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
for (const sentence of sentenceChunks) {
|
||||
const fragments = splitLargeFragment(sentence);
|
||||
for (const fragment of fragments) {
|
||||
appendFragment(fragment);
|
||||
}
|
||||
}
|
||||
pushCurrent();
|
||||
|
||||
return parts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-session TTS manager
|
||||
* Handles TTS audio generation and playback confirmation tracking
|
||||
@@ -33,6 +111,37 @@ export class TTSManager {
|
||||
emitMessage: (msg: SessionOutboundMessage) => void,
|
||||
abortSignal: AbortSignal,
|
||||
isVoiceMode: boolean
|
||||
): Promise<void> {
|
||||
this.logger.info(
|
||||
{
|
||||
isVoiceMode,
|
||||
textLength: text.length,
|
||||
text,
|
||||
},
|
||||
"TTS input text"
|
||||
);
|
||||
|
||||
const segments = splitTextForTts(text, MAX_TTS_SEGMENT_CHARS);
|
||||
for (const segment of segments) {
|
||||
if (abortSignal.aborted) {
|
||||
this.logger.debug("Aborted before generating segmented audio");
|
||||
return;
|
||||
}
|
||||
|
||||
await this.generateSegmentAndWaitForPlayback(
|
||||
segment,
|
||||
emitMessage,
|
||||
abortSignal,
|
||||
isVoiceMode
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async generateSegmentAndWaitForPlayback(
|
||||
text: string,
|
||||
emitMessage: (msg: SessionOutboundMessage) => void,
|
||||
abortSignal: AbortSignal,
|
||||
isVoiceMode: boolean
|
||||
): Promise<void> {
|
||||
if (!this.tts) {
|
||||
throw new Error("TTS not configured");
|
||||
@@ -121,11 +230,6 @@ export class TTSManager {
|
||||
},
|
||||
});
|
||||
|
||||
this.logger.debug(
|
||||
{ chunkId, isLastChunk: next.done },
|
||||
"Sent audio chunk"
|
||||
);
|
||||
|
||||
chunkIndex += 1;
|
||||
|
||||
if (next.done) {
|
||||
@@ -151,7 +255,6 @@ export class TTSManager {
|
||||
} else {
|
||||
this.logger.error({ err: error }, "Error streaming audio");
|
||||
this.pendingPlaybacks.delete(audioId);
|
||||
pendingPlayback.reject(error as Error);
|
||||
throw error;
|
||||
}
|
||||
} finally {
|
||||
|
||||
@@ -147,8 +147,16 @@ describe("daemon client E2E", () => {
|
||||
});
|
||||
expect(Array.isArray(voiceAgents)).toBe(true);
|
||||
|
||||
await expect(ctx.client.setVoiceMode(true)).resolves.toBeUndefined();
|
||||
await expect(ctx.client.setVoiceMode(false)).resolves.toBeUndefined();
|
||||
await expect(ctx.client.setVoiceMode(true)).resolves.toMatchObject({
|
||||
enabled: true,
|
||||
accepted: true,
|
||||
error: null,
|
||||
});
|
||||
await expect(ctx.client.setVoiceMode(false)).resolves.toMatchObject({
|
||||
enabled: false,
|
||||
accepted: true,
|
||||
error: null,
|
||||
});
|
||||
|
||||
await ctx.client.deleteAgent(randomUUID());
|
||||
}, 30000);
|
||||
|
||||
@@ -879,7 +879,7 @@ export class Session {
|
||||
break;
|
||||
|
||||
case "set_voice_mode":
|
||||
await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId);
|
||||
await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId, msg.requestId);
|
||||
break;
|
||||
|
||||
case "send_agent_message_request":
|
||||
@@ -1259,7 +1259,8 @@ export class Session {
|
||||
*/
|
||||
private async handleSetVoiceMode(
|
||||
enabled: boolean,
|
||||
voiceAgentId?: string
|
||||
voiceAgentId?: string,
|
||||
requestId?: string
|
||||
): Promise<void> {
|
||||
if (enabled) {
|
||||
let normalizedVoiceAgentId: string | null = null;
|
||||
@@ -1282,6 +1283,18 @@ export class Session {
|
||||
},
|
||||
"Voice mode enabled (agent-backed)"
|
||||
);
|
||||
if (requestId) {
|
||||
this.emit({
|
||||
type: "set_voice_mode_response",
|
||||
payload: {
|
||||
requestId,
|
||||
enabled: true,
|
||||
voiceAgentId: this.voiceAssistantAgentId,
|
||||
accepted: true,
|
||||
error: null,
|
||||
},
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1290,6 +1303,18 @@ export class Session {
|
||||
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
|
||||
"Voice mode disabled (agent-backed)"
|
||||
);
|
||||
if (requestId) {
|
||||
this.emit({
|
||||
type: "set_voice_mode_response",
|
||||
payload: {
|
||||
requestId,
|
||||
enabled: false,
|
||||
voiceAgentId: this.voiceAssistantAgentId,
|
||||
accepted: true,
|
||||
error: null,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private parseVoiceAgentId(rawId: string, source: string): string {
|
||||
@@ -2471,7 +2496,6 @@ export class Session {
|
||||
lastActivityAt: string;
|
||||
appVisible: boolean;
|
||||
}): void {
|
||||
this.sessionLogger.debug({ heartbeat: msg }, "Client heartbeat");
|
||||
this.clientActivity = {
|
||||
deviceType: msg.deviceType,
|
||||
focusedAgentId: msg.focusedAgentId,
|
||||
@@ -4152,6 +4176,12 @@ export class Session {
|
||||
private async handleAudioChunk(
|
||||
msg: Extract<SessionInboundMessage, { type: "voice_audio_chunk" }>
|
||||
): Promise<void> {
|
||||
if (!this.isVoiceMode) {
|
||||
this.sessionLogger.warn(
|
||||
"Received voice_audio_chunk while voice mode is disabled; transcript will be emitted but voice assistant turn is skipped"
|
||||
);
|
||||
}
|
||||
|
||||
await this.handleVoiceSpeechStart();
|
||||
|
||||
const chunkBuffer = Buffer.from(msg.audio, "base64");
|
||||
@@ -4193,11 +4223,6 @@ export class Session {
|
||||
this.audioBuffer.totalPCMBytes += chunkBuffer.length;
|
||||
}
|
||||
|
||||
this.sessionLogger.debug(
|
||||
{ bytes: chunkBuffer.length, chunks: this.audioBuffer.chunks.length, pcmBytes: this.audioBuffer.totalPCMBytes },
|
||||
`Buffered audio chunk (${chunkBuffer.length} bytes, chunks: ${this.audioBuffer.chunks.length}${this.audioBuffer.isPCM ? `, PCM bytes: ${this.audioBuffer.totalPCMBytes}` : ""})`
|
||||
);
|
||||
|
||||
// In voice mode, only process audio when the user has finished speaking (isLast = true)
|
||||
// This prevents partial transcriptions from being sent to the LLM
|
||||
if (this.isVoiceMode) {
|
||||
@@ -4343,6 +4368,15 @@ export class Session {
|
||||
});
|
||||
|
||||
const transcriptText = result.text.trim();
|
||||
this.sessionLogger.info(
|
||||
{
|
||||
requestId,
|
||||
isVoiceMode: this.isVoiceMode,
|
||||
transcriptLength: transcriptText.length,
|
||||
transcript: transcriptText,
|
||||
},
|
||||
"Transcription result"
|
||||
);
|
||||
|
||||
// Emit transcription result
|
||||
this.emit({
|
||||
@@ -4408,8 +4442,17 @@ export class Session {
|
||||
},
|
||||
});
|
||||
|
||||
// Set phase to LLM and process (TTS enabled in voice mode for voice agents)
|
||||
this.clearSpeechInProgress("transcription complete");
|
||||
if (!this.isVoiceMode) {
|
||||
this.sessionLogger.debug(
|
||||
{ requestId },
|
||||
"Skipping voice agent processing because voice mode is disabled"
|
||||
);
|
||||
this.setPhase("idle");
|
||||
return;
|
||||
}
|
||||
|
||||
// Set phase to LLM and process (TTS enabled in voice mode for voice agents)
|
||||
this.setPhase("llm");
|
||||
this.currentStreamPromise = this.processVoiceTurn(result.text);
|
||||
await this.currentStreamPromise;
|
||||
@@ -4560,7 +4603,6 @@ export class Session {
|
||||
this.agentManager.recordUserMessage(agentId, userText);
|
||||
|
||||
let sawSpeakToolCall = false;
|
||||
const assistantTextChunks: string[] = [];
|
||||
const iterator = this.agentManager.streamAgent(agentId, prompt);
|
||||
for await (const event of iterator) {
|
||||
if (event.type === "turn_failed") {
|
||||
@@ -4572,9 +4614,6 @@ export class Session {
|
||||
sawSpeakToolCall = true;
|
||||
}
|
||||
}
|
||||
if (event.item.type === "assistant_message" && event.item.text.trim().length > 0) {
|
||||
assistantTextChunks.push(event.item.text.trim());
|
||||
}
|
||||
}
|
||||
if (event.type === "permission_requested") {
|
||||
if (this.shouldAllowVoicePermission(event.request)) {
|
||||
@@ -4594,26 +4633,10 @@ export class Session {
|
||||
}
|
||||
}
|
||||
|
||||
if (!sawSpeakToolCall && assistantTextChunks.length > 0) {
|
||||
const fallbackText = assistantTextChunks.join(" ").trim();
|
||||
await this.ttsManager.generateAndWaitForPlayback(
|
||||
fallbackText,
|
||||
(msg) => this.emit(msg),
|
||||
this.abortController.signal,
|
||||
true
|
||||
);
|
||||
this.emit({
|
||||
type: "activity_log",
|
||||
payload: {
|
||||
id: uuidv4(),
|
||||
timestamp: new Date(),
|
||||
type: "assistant",
|
||||
content: fallbackText,
|
||||
},
|
||||
});
|
||||
if (!sawSpeakToolCall) {
|
||||
this.sessionLogger.warn(
|
||||
{ voiceAssistantAgentId: agentId },
|
||||
"Voice agent responded without speak tool; used fallback TTS from assistant text"
|
||||
"Voice agent turn completed without speak tool call; no audio playback emitted"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -389,7 +389,14 @@ export class VoiceAssistantWebSocketServer {
|
||||
? { sessionMessageType: message.message.type }
|
||||
: {}),
|
||||
};
|
||||
this.logger.debug(messageSummary, "Received message");
|
||||
const isSessionNoise =
|
||||
message.type === "session" &&
|
||||
(message.message.type === "client_heartbeat" ||
|
||||
message.message.type === "voice_audio_chunk" ||
|
||||
message.message.type === "dictation_stream_chunk");
|
||||
if (!isSessionNoise) {
|
||||
this.logger.debug(messageSummary, "Received message");
|
||||
}
|
||||
|
||||
if (message.type === "ping") {
|
||||
this.sendToClient(ws, { type: "pong" });
|
||||
|
||||
@@ -348,6 +348,7 @@ export const SetVoiceModeMessageSchema = z.object({
|
||||
type: z.literal("set_voice_mode"),
|
||||
enabled: z.boolean(),
|
||||
voiceAgentId: z.string().optional(),
|
||||
requestId: z.string().optional(),
|
||||
});
|
||||
|
||||
export const SendAgentMessageSchema = z.object({
|
||||
@@ -571,6 +572,17 @@ export const SetAgentThinkingResponseMessageSchema = z.object({
|
||||
}),
|
||||
});
|
||||
|
||||
export const SetVoiceModeResponseMessageSchema = z.object({
|
||||
type: z.literal("set_voice_mode_response"),
|
||||
payload: z.object({
|
||||
requestId: z.string(),
|
||||
enabled: z.boolean(),
|
||||
voiceAgentId: z.string().nullable(),
|
||||
accepted: z.boolean(),
|
||||
error: z.string().nullable(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const AgentPermissionResponseMessageSchema = z.object({
|
||||
type: z.literal("agent_permission_response"),
|
||||
agentId: z.string(),
|
||||
@@ -1627,6 +1639,7 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
|
||||
FetchAgentsResponseMessageSchema,
|
||||
FetchAgentResponseMessageSchema,
|
||||
SendAgentMessageResponseMessageSchema,
|
||||
SetVoiceModeResponseMessageSchema,
|
||||
SetAgentModeResponseMessageSchema,
|
||||
SetAgentModelResponseMessageSchema,
|
||||
SetAgentThinkingResponseMessageSchema,
|
||||
@@ -1690,6 +1703,9 @@ export type FetchAgentResponseMessage = z.infer<
|
||||
export type SendAgentMessageResponseMessage = z.infer<
|
||||
typeof SendAgentMessageResponseMessageSchema
|
||||
>;
|
||||
export type SetVoiceModeResponseMessage = z.infer<
|
||||
typeof SetVoiceModeResponseMessageSchema
|
||||
>;
|
||||
export type WaitForFinishResponseMessage = z.infer<
|
||||
typeof WaitForFinishResponseMessageSchema
|
||||
>;
|
||||
|
||||
Reference in New Issue
Block a user