diff --git a/packages/app/package.json b/packages/app/package.json index a75826a97..289a1d6bb 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -17,7 +17,7 @@ "@react-navigation/bottom-tabs": "^7.4.0", "@react-navigation/elements": "^2.6.3", "@react-navigation/native": "^7.1.8", - "@speechmatics/expo-two-way-audio": "^0.1.2", + "@boudra/expo-two-way-audio": "^0.1.3", "buffer": "^6.0.3", "expo": "^54.0.18", "expo-audio": "~1.0.13", diff --git a/packages/app/src/components/agent-stream-view.tsx b/packages/app/src/components/agent-stream-view.tsx index c0dae2306..dbca06d13 100644 --- a/packages/app/src/components/agent-stream-view.tsx +++ b/packages/app/src/components/agent-stream-view.tsx @@ -1,9 +1,11 @@ -import { useEffect, useRef } from 'react'; +import { useEffect, useRef, useState } from 'react'; import { View, Text, ScrollView, Pressable } from 'react-native'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { StyleSheet, useUnistyles } from 'react-native-unistyles'; +import { BottomSheetModal } from '@gorhom/bottom-sheet'; import type { AgentStatus } from '@server/server/acp/types'; import { AssistantMessage, UserMessage, ActivityLog, ToolCall } from './message'; +import { ToolCallBottomSheet } from './tool-call-bottom-sheet'; import type { StreamItem } from '@/types/stream'; export interface AgentStreamViewProps { @@ -40,13 +42,32 @@ export function AgentStreamView({ onPermissionResponse, }: AgentStreamViewProps) { const scrollViewRef = useRef(null); + const bottomSheetRef = useRef(null); const insets = useSafeAreaInsets(); + const [selectedToolCall, setSelectedToolCall] = useState<{ + toolName: string; + status: 'executing' | 'completed' | 'failed'; + args: any; + result?: any; + error?: any; + } | null>(null); // Auto-scroll to bottom when new items arrive useEffect(() => { scrollViewRef.current?.scrollToEnd({ animated: true }); }, [streamItems]); + function handleOpenToolCallDetails(toolCall: { + toolName: string; + status: 'executing' | 'completed' | 'failed'; + args: any; + result?: any; + error?: any; + }) { + setSelectedToolCall(toolCall); + bottomSheetRef.current?.present(); + } + return ( {/* Content list */} @@ -107,6 +128,12 @@ export function AgentStreamView({ args={item.rawInput} result={item.rawOutput} status={toolStatus} + onOpenDetails={() => handleOpenToolCallDetails({ + toolName: item.title, + status: toolStatus, + args: item.rawInput, + result: item.rawOutput, + })} /> ); } @@ -137,6 +164,15 @@ export function AgentStreamView({ /> ))} + + ); } diff --git a/packages/app/src/components/create-agent-modal.tsx b/packages/app/src/components/create-agent-modal.tsx index ef31660e2..c7aea45f4 100644 --- a/packages/app/src/components/create-agent-modal.tsx +++ b/packages/app/src/components/create-agent-modal.tsx @@ -1,5 +1,12 @@ import { useState, useRef, useEffect, useMemo, useCallback } from "react"; -import { View, Text, Pressable, ScrollView, ActivityIndicator } from "react-native"; +import { + View, + Text, + Pressable, + ScrollView, + ActivityIndicator, + InteractionManager, +} from "react-native"; import { useSafeAreaInsets } from "react-native-safe-area-context"; import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller"; import Animated, { useAnimatedStyle } from "react-native-reanimated"; @@ -57,6 +64,9 @@ export function CreateAgentModal({ const [isLoading, setIsLoading] = useState(false); const [pendingRequestId, setPendingRequestId] = useState(null); + // Use ref instead of state to survive state resets in onDismiss + const pendingNavigationAgentIdRef = useRef(null); + const snapPoints = useMemo(() => ["90%"], []); // Keyboard animation for footer @@ -132,12 +142,11 @@ export function CreateAgentModal({ setIsLoading(false); setPendingRequestId(null); - // Navigate to the agent page BEFORE closing modal - // This prevents race condition on Android where router.push() happens - // while the modal is unmounting, causing NullPointerException - router.push(`/agent/${agentId}`); + // Store the agent ID in ref for navigation after modal dismisses + // Using ref instead of state because state gets reset in onDismiss + pendingNavigationAgentIdRef.current = agentId; - // Close modal after navigation starts + // Close modal - navigation will happen in handleDismiss handleClose(); } }); @@ -145,7 +154,7 @@ export function CreateAgentModal({ return () => { unsubscribe(); }; - }, [pendingRequestId, ws, router]); + }, [pendingRequestId, ws]); async function handleCreate() { if (!workingDir.trim()) { @@ -194,6 +203,8 @@ export function CreateAgentModal({ } function handleDismiss() { + const agentId = pendingNavigationAgentIdRef.current; + // Reset all state setWorkingDir(""); setSelectedMode("plan"); @@ -201,6 +212,14 @@ export function CreateAgentModal({ setIsLoading(false); setPendingRequestId(null); onClose(); + + // Navigate after interactions complete to avoid race with dismiss animation + if (agentId) { + InteractionManager.runAfterInteractions(() => { + router.push(`/agent/${agentId}`); + pendingNavigationAgentIdRef.current = null; + }); + } } return ( diff --git a/packages/app/src/components/message.tsx b/packages/app/src/components/message.tsx index 1aaf3b215..00f69ca0f 100644 --- a/packages/app/src/components/message.tsx +++ b/packages/app/src/components/message.tsx @@ -322,6 +322,7 @@ interface ToolCallProps { result?: any; error?: any; status: 'executing' | 'completed' | 'failed'; + onOpenDetails?: () => void; } const toolCallStylesheet = StyleSheet.create((theme) => ({ @@ -420,8 +421,7 @@ const toolCallStylesheet = StyleSheet.create((theme) => ({ }, })); -export function ToolCall({ toolName, args, result, error, status }: ToolCallProps) { - const [isExpanded, setIsExpanded] = useState(false); +export function ToolCall({ toolName, args, result, error, status, onOpenDetails }: ToolCallProps) { const spinAnim = useRef(new Animated.Value(0)).current; useEffect(() => { @@ -469,18 +469,11 @@ export function ToolCall({ toolName, args, result, error, status }: ToolCallProp return ( setIsExpanded(!isExpanded)} + onPress={onOpenDetails} style={[toolCallStylesheet.pressable, toolCallStylesheet.pressableActive, config.border]} > - - {isExpanded ? ( - - ) : ( - - )} - {toolName} @@ -499,47 +492,6 @@ export function ToolCall({ toolName, args, result, error, status }: ToolCallProp - - {isExpanded && ( - - - - Arguments - - - - {JSON.stringify(args, null, 2)} - - - - - {result !== undefined && ( - - - Result - - - - {JSON.stringify(result, null, 2)} - - - - )} - - {error !== undefined && ( - - - Error - - - - {JSON.stringify(error, null, 2)} - - - - )} - - )} ); diff --git a/packages/app/src/components/orchestrator-messages-view.tsx b/packages/app/src/components/orchestrator-messages-view.tsx index f8558c312..a6f2442da 100644 --- a/packages/app/src/components/orchestrator-messages-view.tsx +++ b/packages/app/src/components/orchestrator-messages-view.tsx @@ -1,12 +1,14 @@ import { View, ScrollView } from "react-native"; import { StyleSheet, useUnistyles } from "react-native-unistyles"; -import { forwardRef } from "react"; +import { forwardRef, useRef, useState } from "react"; +import { BottomSheetModal } from "@gorhom/bottom-sheet"; import { UserMessage, AssistantMessage, ActivityLog, ToolCall, } from "@/components/message"; +import { ToolCallBottomSheet } from "./tool-call-bottom-sheet"; import type { MessageEntry } from "@/contexts/session-context"; interface OrchestratorMessagesViewProps { @@ -18,15 +20,35 @@ interface OrchestratorMessagesViewProps { export const OrchestratorMessagesView = forwardRef( function OrchestratorMessagesView({ messages, currentAssistantMessage, onArtifactClick }, ref) { const { theme } = useUnistyles(); + const bottomSheetRef = useRef(null); + const [selectedToolCall, setSelectedToolCall] = useState<{ + toolName: string; + status: 'executing' | 'completed' | 'failed'; + args: any; + result?: any; + error?: any; + } | null>(null); + + function handleOpenToolCallDetails(toolCall: { + toolName: string; + status: 'executing' | 'completed' | 'failed'; + args: any; + result?: any; + error?: any; + }) { + setSelectedToolCall(toolCall); + bottomSheetRef.current?.present(); + } return ( - + <> + {messages.map((msg) => { if (msg.type === "user") { return ( @@ -85,6 +107,13 @@ export const OrchestratorMessagesView = forwardRef handleOpenToolCallDetails({ + toolName: msg.toolName, + status: msg.status, + args: msg.args, + result: msg.result, + error: msg.error, + })} /> ); } @@ -101,6 +130,16 @@ export const OrchestratorMessagesView = forwardRef )} + + + ); } ); diff --git a/packages/app/src/components/tool-call-bottom-sheet.tsx b/packages/app/src/components/tool-call-bottom-sheet.tsx new file mode 100644 index 000000000..cff7f806e --- /dev/null +++ b/packages/app/src/components/tool-call-bottom-sheet.tsx @@ -0,0 +1,207 @@ +import React, { useCallback, useMemo } from "react"; +import { View, Text, StyleSheet } from "react-native"; +import { + BottomSheetModal, + BottomSheetView, + BottomSheetScrollView, + BottomSheetBackdrop, + BottomSheetBackdropProps, +} from "@gorhom/bottom-sheet"; + +interface ToolCallBottomSheetProps { + bottomSheetRef: React.RefObject; + toolName?: string; + status?: "pending" | "in_progress" | "executing" | "completed" | "failed"; + args?: any; + result?: any; + error?: any; +} + +export function ToolCallBottomSheet({ + bottomSheetRef, + toolName, + status, + args, + result, + error, +}: ToolCallBottomSheetProps) { + const snapPoints = useMemo(() => ["80%"], []); + + const renderBackdrop = useCallback( + (props: BottomSheetBackdropProps) => ( + + ), + [] + ); + + const statusColor = useMemo(() => { + switch (status) { + case "completed": + return "#22c55e"; + case "failed": + return "#ef4444"; + case "executing": + case "in_progress": + return "#3b82f6"; + case "pending": + default: + return "#6b7280"; + } + }, [status]); + + const statusLabel = useMemo(() => { + switch (status) { + case "in_progress": + case "executing": + return "Executing"; + case "completed": + return "Completed"; + case "failed": + return "Failed"; + case "pending": + default: + return "Pending"; + } + }, [status]); + + return ( + + + + {toolName || "Tool Call"} + + {statusLabel} + + + + + {args !== undefined && ( + + Arguments + + + {JSON.stringify(args, null, 2)} + + + + )} + + {result !== undefined && ( + + Result + + + {JSON.stringify(result, null, 2)} + + + + )} + + {error !== undefined && ( + + Error + + + {JSON.stringify(error, null, 2)} + + + + )} + + + + ); +} + +const styles = StyleSheet.create({ + container: { + flex: 1, + backgroundColor: "#1f2937", + }, + handleIndicator: { + backgroundColor: "#4b5563", + }, + background: { + backgroundColor: "#1f2937", + }, + header: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + paddingHorizontal: 20, + paddingVertical: 16, + borderBottomWidth: 1, + borderBottomColor: "#374151", + }, + toolName: { + fontSize: 18, + fontWeight: "600", + color: "#f9fafb", + flex: 1, + }, + statusBadge: { + paddingHorizontal: 8, + paddingVertical: 4, + borderRadius: 4, + marginLeft: 12, + }, + statusText: { + fontSize: 12, + fontWeight: "600", + color: "#fff", + textTransform: "capitalize", + }, + scrollView: { + flex: 1, + }, + scrollContent: { + paddingHorizontal: 20, + paddingVertical: 16, + }, + section: { + marginBottom: 20, + }, + sectionTitle: { + fontSize: 14, + fontWeight: "600", + color: "#9ca3af", + marginBottom: 8, + textTransform: "uppercase", + letterSpacing: 0.5, + }, + jsonContainer: { + backgroundColor: "#111827", + borderRadius: 8, + padding: 12, + borderWidth: 1, + borderColor: "#374151", + }, + jsonText: { + fontFamily: "monospace", + fontSize: 12, + color: "#e5e7eb", + lineHeight: 18, + }, + errorContainer: { + borderColor: "#ef4444", + backgroundColor: "#1f1416", + }, + errorText: { + color: "#fca5a5", + }, +}); diff --git a/packages/app/src/hooks/use-audio-player.ts b/packages/app/src/hooks/use-audio-player.ts index aea818453..a00e5a24a 100644 --- a/packages/app/src/hooks/use-audio-player.ts +++ b/packages/app/src/hooks/use-audio-player.ts @@ -1,5 +1,11 @@ -import { useState, useRef } from 'react'; -import { initialize, playPCMData } from '@speechmatics/expo-two-way-audio'; +import { useState, useRef } from "react"; +import { + initialize, + playPCMData, + stopPlayback, + pausePlayback, + resumePlayback, +} from "@boudra/expo-two-way-audio"; interface QueuedAudio { audioData: Blob; @@ -14,7 +20,7 @@ interface QueuedAudio { function resamplePcm24kTo16k(pcm24k: Uint8Array): Uint8Array { // PCM16 = 2 bytes per sample const samples24k = pcm24k.length / 2; - const samples16k = Math.floor(samples24k * 16000 / 24000); + const samples16k = Math.floor((samples24k * 16000) / 24000); const pcm16k = new Uint8Array(samples16k * 2); const ratio = 24000 / 16000; // 1.5 @@ -27,10 +33,10 @@ function resamplePcm24kTo16k(pcm24k: Uint8Array): Uint8Array { } } - console.log('[AudioPlayer] Resampled PCM:', { + console.log("[AudioPlayer] Resampled PCM:", { input24k: pcm24k.length, output16k: pcm16k.length, - durationMs: (samples16k / 16), + durationMs: samples16k / 16, }); return pcm16k; @@ -44,6 +50,7 @@ export function useAudioPlayer() { const [audioInitialized, setAudioInitialized] = useState(false); const queueRef = useRef([]); const isProcessingQueueRef = useRef(false); + const playbackTimeoutRef = useRef(null); async function play(audioData: Blob): Promise { return new Promise((resolve, reject) => { @@ -100,12 +107,19 @@ export function useAudioPlayer() { // Initialize audio if not already initialized if (!audioInitialized) { - console.log('[AudioPlayer] Initializing audio...'); + console.log("[AudioPlayer] Initializing audio..."); await initialize(); setAudioInitialized(true); - console.log('[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)'); + console.log( + "[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)" + ); } + // Workaround: Resume playback before playing new audio to ensure the audio engine is ready + // This fixes the issue where playback doesn't work after calling stopPlayback() + console.log("[AudioPlayer] Resuming playback engine..."); + resumePlayback(); + // Get PCM data from blob (server now sends PCM format) const arrayBuffer = await audioData.arrayBuffer(); let pcm24k = new Uint8Array(arrayBuffer); @@ -118,22 +132,40 @@ export function useAudioPlayer() { const durationSec = samples / 16000; // 16kHz sample rate const audioSizeKb = (pcm16k.length / 1024).toFixed(2); - console.log('[AudioPlayer] 🔊 Playing audio:', audioSizeKb, 'KB, duration:', durationSec.toFixed(2), 's'); + console.log( + "[AudioPlayer] 🔊 Playing audio:", + audioSizeKb, + "KB, duration:", + durationSec.toFixed(2), + "s" + ); setIsPlaying(true); // Play entire PCM data at once through Speechmatics playPCMData(pcm16k); + // Clear any existing timeout + if (playbackTimeoutRef.current) { + clearTimeout(playbackTimeoutRef.current); + } + // Wait for playback to finish (estimate based on duration) - setTimeout(() => { - console.log('[AudioPlayer] ✅ Playback finished'); + playbackTimeoutRef.current = setTimeout(() => { + console.log("[AudioPlayer] ✅ Playback finished"); setIsPlaying(false); + playbackTimeoutRef.current = null; resolve(durationSec); }, durationSec * 1000); - } catch (error) { - console.error('[AudioPlayer] Error playing audio:', error); + console.error("[AudioPlayer] Error playing audio:", error); + + // Clear timeout on error + if (playbackTimeoutRef.current) { + clearTimeout(playbackTimeoutRef.current); + playbackTimeoutRef.current = null; + } + setIsPlaying(false); reject(error); } @@ -142,7 +174,16 @@ export function useAudioPlayer() { function stop(): void { if (isPlaying) { - console.log('[AudioPlayer] 🛑 Stopping playback (interrupted)'); + console.log("[AudioPlayer] 🛑 Stopping playback (interrupted)"); + + // Stop native playback + stopPlayback(); + + // Clear playback timeout + if (playbackTimeoutRef.current) { + clearTimeout(playbackTimeoutRef.current); + playbackTimeoutRef.current = null; + } } setIsPlaying(false); @@ -150,7 +191,7 @@ export function useAudioPlayer() { // Reject all pending promises in the queue while (queueRef.current.length > 0) { const item = queueRef.current.shift()!; - item.reject(new Error('Playback stopped')); + item.reject(new Error("Playback stopped")); } isProcessingQueueRef.current = false; @@ -160,7 +201,7 @@ export function useAudioPlayer() { // Reject all pending promises in the queue while (queueRef.current.length > 0) { const item = queueRef.current.shift()!; - item.reject(new Error('Queue cleared')); + item.reject(new Error("Queue cleared")); } } diff --git a/packages/app/src/hooks/use-speechmatics-audio.ts b/packages/app/src/hooks/use-speechmatics-audio.ts index 56b9eeeb2..94feb769c 100644 --- a/packages/app/src/hooks/use-speechmatics-audio.ts +++ b/packages/app/src/hooks/use-speechmatics-audio.ts @@ -6,7 +6,7 @@ import { useExpoTwoWayAudioEventListener, type MicrophoneDataCallback, type VolumeLevelCallback, -} from "@speechmatics/expo-two-way-audio"; +} from "@boudra/expo-two-way-audio"; export interface SpeechmaticsAudioConfig { onAudioSegment?: (audioData: string) => void; @@ -200,7 +200,11 @@ export function useSpeechmaticsAudio( // console.log('[SpeechmaticsAudio] Volume:', volumeLevel.toFixed(6), 'Threshold:', VOLUME_THRESHOLD); - if (speechDetected && !isSpeakingRef.current && !speechConfirmedRef.current) { + if ( + speechDetected && + !isSpeakingRef.current && + !speechConfirmedRef.current + ) { // Initial speech detection - start tracking if (speechDetectionStartRef.current === null) { console.log( @@ -233,10 +237,18 @@ export function useSpeechmaticsAudio( config.onSpeechStart?.(); } } - } else if (speechDetected && isSpeakingRef.current && speechConfirmedRef.current) { + } else if ( + speechDetected && + isSpeakingRef.current && + speechConfirmedRef.current + ) { // Continuing confirmed speech silenceStartRef.current = null; - } else if (!speechDetected && !speechConfirmedRef.current && speechDetectionStartRef.current !== null) { + } else if ( + !speechDetected && + !speechConfirmedRef.current && + speechDetectionStartRef.current !== null + ) { // Volume dropped during detection phase - apply grace period if (detectionSilenceStartRef.current === null) { detectionSilenceStartRef.current = Date.now(); @@ -255,7 +267,11 @@ export function useSpeechmaticsAudio( setIsDetecting(false); } } - } else if (!speechDetected && isSpeakingRef.current && speechConfirmedRef.current) { + } else if ( + !speechDetected && + isSpeakingRef.current && + speechConfirmedRef.current + ) { // Potential speech END if (silenceStartRef.current === null) { silenceStartRef.current = Date.now(); @@ -289,7 +305,15 @@ export function useSpeechmaticsAudio( } } }, - [isActive, isMuted, VOLUME_THRESHOLD, SILENCE_DURATION_MS, SPEECH_CONFIRMATION_MS, DETECTION_GRACE_PERIOD_MS, config] + [ + isActive, + isMuted, + VOLUME_THRESHOLD, + SILENCE_DURATION_MS, + SPEECH_CONFIRMATION_MS, + DETECTION_GRACE_PERIOD_MS, + config, + ] ) ); diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 8dcf4830d..b92c53a93 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -259,7 +259,7 @@ export class Session { /** * Maybe trigger title generation for an agent - * Only generates title once after first meaningful activity + * Only generates title once after first user message chunk and some initial agent activity */ private maybeTriggerTitleGeneration(agentId: string): void { // Skip if title generator not initialized @@ -275,8 +275,15 @@ export class Session { // Get agent updates const updates = this.agentManager.getAgentUpdates(agentId); - // Need at least 15 updates before generating title - if (updates.length < 15) { + // Find first user message chunk + const hasUserMessage = updates.some( + (update) => + update.notification.type === "session" && + update.notification.notification.update.sessionUpdate === "user_message_chunk" + ); + + // Need at least one user message and some additional updates (3-5) for context + if (!hasUserMessage || updates.length < 5) { return; }