diff --git a/packages/app/src/components/create-agent-modal.tsx b/packages/app/src/components/create-agent-modal.tsx index d6e7e16f8..2fc69fed4 100644 --- a/packages/app/src/components/create-agent-modal.tsx +++ b/packages/app/src/components/create-agent-modal.tsx @@ -20,6 +20,7 @@ import { theme as defaultTheme } from "@/styles/theme"; import { useRecentPaths } from "@/hooks/use-recent-paths"; import { useSession } from "@/contexts/session-context"; import { useRouter } from "expo-router"; +import { generateMessageId } from "@/types/stream"; interface CreateAgentModalProps { isVisible: boolean; @@ -163,7 +164,7 @@ export function CreateAgentModal({ } // Generate request ID - const requestId = crypto.randomUUID(); + const requestId = generateMessageId(); setIsLoading(true); setPendingRequestId(requestId); diff --git a/packages/app/src/contexts/session-context.tsx b/packages/app/src/contexts/session-context.tsx index 33853dcca..603deb051 100644 --- a/packages/app/src/contexts/session-context.tsx +++ b/packages/app/src/contexts/session-context.tsx @@ -260,49 +260,205 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) { }); // Audio output - const unsubAudioOutput = ws.on("audio_output", (message) => { + const unsubAudioOutput = ws.on("audio_output", async (message) => { if (message.type !== "audio_output") return; - const { audio, format, id } = message.payload; + const data = message.payload; - console.log("[Session] Received audio output:", id, "format:", format); + try { + setIsPlayingAudio(true); - // Create Blob-like object for React Native - // React Native doesn't support creating Blobs from binary data - const audioBlob = { - type: format, - size: audio.length, - arrayBuffer: async () => { - // Convert base64 to ArrayBuffer - const binaryString = atob(audio); - const bytes = new Uint8Array(binaryString.length); - for (let i = 0; i < binaryString.length; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - return bytes.buffer; + // Create blob-like object with correct mime type (React Native compatible) + const mimeType = + data.format === "mp3" ? "audio/mpeg" : `audio/${data.format}`; + const base64Audio = data.audio; + + // Create a Blob-like object that works in React Native + const audioBlob = { + type: mimeType, + size: Math.ceil((base64Audio.length * 3) / 4), // Approximate size from base64 + arrayBuffer: async () => { + // Convert base64 to ArrayBuffer + const binaryString = atob(base64Audio); + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + return bytes.buffer; + }, + } as Blob; + + // Play audio + await audioPlayer.play(audioBlob); + + // Send confirmation back to server + const confirmMessage: WSInboundMessage = { + type: "session", + message: { + type: "audio_played", + id: data.id, + }, + }; + ws.send(confirmMessage); + + setIsPlayingAudio(false); + } catch (error: any) { + console.error("[Session] Audio playback error:", error); + setIsPlayingAudio(false); + + // Still send confirmation even on error to prevent server from waiting + const confirmMessage: WSInboundMessage = { + type: "session", + message: { + type: "audio_played", + id: data.id, + }, + }; + ws.send(confirmMessage); + } + }); + + // Activity log handler + const unsubActivity = ws.on("activity_log", (message) => { + if (message.type !== "activity_log") return; + const data = message.payload; + + // Filter out transcription activity logs + if (data.type === "system" && data.content.includes("Transcribing")) { + return; + } + + // Handle tool calls + if (data.type === "tool_call" && data.metadata) { + const { + toolCallId, + toolName, + arguments: args, + } = data.metadata as { + toolCallId: string; + toolName: string; + arguments: unknown; + }; + + setMessages((prev) => [ + ...prev, + { + type: "tool_call", + id: toolCallId, + timestamp: Date.now(), + toolName, + args, + status: "executing", + }, + ]); + return; + } + + // Handle tool results + if (data.type === "tool_result" && data.metadata) { + const { toolCallId, result } = data.metadata as { + toolCallId: string; + result: unknown; + }; + + setMessages((prev) => + prev.map((msg) => + msg.type === "tool_call" && msg.id === toolCallId + ? { ...msg, result, status: "completed" as const } + : msg + ) + ); + return; + } + + // Handle tool errors + if ( + data.type === "error" && + data.metadata && + "toolCallId" in data.metadata + ) { + const { toolCallId, error } = data.metadata as { + toolCallId: string; + error: unknown; + }; + + setMessages((prev) => + prev.map((msg) => + msg.type === "tool_call" && msg.id === toolCallId + ? { ...msg, error, status: "failed" as const } + : msg + ) + ); + } + + // Map activity types to message types + let activityType: "system" | "info" | "success" | "error" = "info"; + if (data.type === "error") activityType = "error"; + + // Add user transcripts as user messages + if (data.type === "transcript") { + setMessages((prev) => [ + ...prev, + { + type: "user", + id: generateMessageId(), + timestamp: Date.now(), + message: data.content, + }, + ]); + return; + } + + // Add assistant messages + if (data.type === "assistant") { + setMessages((prev) => [ + ...prev, + { + type: "assistant", + id: generateMessageId(), + timestamp: Date.now(), + message: data.content, + }, + ]); + setCurrentAssistantMessage(""); + return; + } + + // Add activity log for other types + setMessages((prev) => [ + ...prev, + { + type: "activity", + id: generateMessageId(), + timestamp: Date.now(), + activityType, + message: data.content, + metadata: data.metadata, }, - } as Blob; + ]); + }); - // Play audio - setIsPlayingAudio(true); - audioPlayer.play(audioBlob) - .then(() => { - console.log("[Session] Audio playback complete:", id); - setIsPlayingAudio(false); + // Assistant chunk handler (streaming) + const unsubChunk = ws.on("assistant_chunk", (message) => { + if (message.type !== "assistant_chunk") return; + setCurrentAssistantMessage((prev) => prev + message.payload.chunk); + }); - // Send confirmation to server - const msg: WSInboundMessage = { - type: "session", - message: { - type: "audio_played", - id, - }, - }; - ws.send(msg); - }) - .catch((error) => { - console.error("[Session] Audio playback failed:", error); - setIsPlayingAudio(false); - }); + // Transcription result handler + const unsubTranscription = ws.on("transcription_result", (message) => { + if (message.type !== "transcription_result") return; + + const transcriptText = message.payload.text.trim(); + + if (!transcriptText) { + // Empty transcription - false positive, let playback continue + console.log("[Session] Empty transcription (false positive) - ignoring"); + } else { + // Has content - real speech detected, stop playback + console.log("[Session] Transcription received - stopping playback"); + audioPlayer.stop(); + setIsPlayingAudio(false); + setCurrentAssistantMessage(""); + } }); return () => { @@ -312,6 +468,9 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) { unsubAgentUpdate(); unsubPermissionRequest(); unsubAudioOutput(); + unsubActivity(); + unsubChunk(); + unsubTranscription(); }; }, [ws, audioPlayer, setIsPlayingAudio]); diff --git a/packages/server/src/server/acp/agent-manager.ts b/packages/server/src/server/acp/agent-manager.ts index eabaeed04..fa531c9f7 100644 --- a/packages/server/src/server/acp/agent-manager.ts +++ b/packages/server/src/server/acp/agent-manager.ts @@ -268,6 +268,10 @@ export class AgentManager { throw new Error(`Agent ${agentId} is ${agent.status}`); } + // Reset message IDs for new turn - ensures each prompt gets fresh responses + agent.currentAssistantMessageId = null; + agent.currentThoughtId = null; + // Set session mode if specified if (options?.sessionMode) { await this.setSessionMode(agentId, options.sessionMode);