diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 9d77e1326..351d1aeb7 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -36,3 +36,5 @@ Set a high timeout, like 2 minutes. Try not to bias it, state the problem clearly and let it work it out. `codex exec "prompt"` + +**CRITICAL: ALWAYS RUN TYPECHECK AFTER EVERY CHANGE.** diff --git a/packages/app/assets/images/android-icon-background.png b/packages/app/assets/images/android-icon-background.png index 5ffefc5bb..2cc012d47 100644 Binary files a/packages/app/assets/images/android-icon-background.png and b/packages/app/assets/images/android-icon-background.png differ diff --git a/packages/app/assets/images/android-icon-foreground.png b/packages/app/assets/images/android-icon-foreground.png index 3a9e5016d..27c94262e 100644 Binary files a/packages/app/assets/images/android-icon-foreground.png and b/packages/app/assets/images/android-icon-foreground.png differ diff --git a/packages/app/assets/images/android-icon-monochrome.png b/packages/app/assets/images/android-icon-monochrome.png index 77484ebdb..767aee490 100644 Binary files a/packages/app/assets/images/android-icon-monochrome.png and b/packages/app/assets/images/android-icon-monochrome.png differ diff --git a/packages/app/assets/images/favicon.png b/packages/app/assets/images/favicon.png index 408bd7466..663079133 100644 Binary files a/packages/app/assets/images/favicon.png and b/packages/app/assets/images/favicon.png differ diff --git a/packages/app/assets/images/hammock-icon.png b/packages/app/assets/images/hammock-icon.png new file mode 100644 index 000000000..ae5077dee Binary files /dev/null and b/packages/app/assets/images/hammock-icon.png differ diff --git a/packages/app/assets/images/icon.png b/packages/app/assets/images/icon.png index 7165a53c7..ae5077dee 100644 Binary files a/packages/app/assets/images/icon.png and b/packages/app/assets/images/icon.png differ diff --git a/packages/app/src/app/_layout.tsx b/packages/app/src/app/_layout.tsx index d20194640..7595ebdd1 100644 --- a/packages/app/src/app/_layout.tsx +++ b/packages/app/src/app/_layout.tsx @@ -1,18 +1,61 @@ -import { Stack } from 'expo-router'; -import { SafeAreaProvider } from 'react-native-safe-area-context'; -import { KeyboardProvider } from 'react-native-keyboard-controller'; -import { GestureHandlerRootView } from 'react-native-gesture-handler'; +import { Stack } from "expo-router"; +import { SafeAreaProvider } from "react-native-safe-area-context"; +import { KeyboardProvider } from "react-native-keyboard-controller"; +import { GestureHandlerRootView } from "react-native-gesture-handler"; +import { SessionProvider } from "@/contexts/session-context"; +import { RealtimeProvider } from "@/contexts/realtime-context"; +import { useSettings } from "@/hooks/use-settings"; +import { View, ActivityIndicator } from "react-native"; + +function ProvidersWrapper({ children }: { children: React.ReactNode }) { + const { settings, isLoading } = useSettings(); + + if (isLoading) { + return ( + + + + ); + } + + return ( + + {children} + + ); +} export default function RootLayout() { return ( - - - - - + + + + + + + + + diff --git a/packages/app/src/app/agent/[id].tsx b/packages/app/src/app/agent/[id].tsx new file mode 100644 index 000000000..4776a0634 --- /dev/null +++ b/packages/app/src/app/agent/[id].tsx @@ -0,0 +1,87 @@ +import { View, Text } from "react-native"; +import { useLocalSearchParams } from "expo-router"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller"; +import ReanimatedAnimated, { useAnimatedStyle } from "react-native-reanimated"; +import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import { BackHeader } from "@/components/headers/back-header"; +import { AgentStreamView } from "@/components/agent-stream-view"; +import { GlobalFooter } from "@/components/global-footer"; +import { useSession } from "@/contexts/session-context"; + +export default function AgentScreen() { + const { theme } = useUnistyles(); + const insets = useSafeAreaInsets(); + const { id } = useLocalSearchParams<{ id: string }>(); + const { agents, agentStreamState, pendingPermissions, respondToPermission } = useSession(); + + // Keyboard animation + const { height: keyboardHeight } = useReanimatedKeyboardAnimation(); + const animatedKeyboardStyle = useAnimatedStyle(() => { + "worklet"; + const absoluteHeight = Math.abs(keyboardHeight.value); + const padding = Math.max(0, absoluteHeight - insets.bottom); + return { + paddingBottom: padding, + }; + }); + + const agent = id ? agents.get(id) : undefined; + const streamItems = id ? agentStreamState.get(id) || [] : []; + const agentPermissions = new Map( + Array.from(pendingPermissions.entries()).filter(([_, perm]) => perm.agentId === id) + ); + + if (!agent) { + return ( + + + + Agent not found + + + ); + } + + return ( + + {/* Header */} + + + {/* Content Area with Keyboard Animation */} + + + respondToPermission(requestId, id!, agent.sessionId || "", [optionId]) + } + /> + + {/* Footer */} + + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + container: { + flex: 1, + backgroundColor: theme.colors.background, + }, + content: { + flex: 1, + }, + errorContainer: { + flex: 1, + alignItems: "center", + justifyContent: "center", + }, + errorText: { + fontSize: theme.fontSize.lg, + color: theme.colors.mutedForeground, + }, +})); diff --git a/packages/app/src/app/index.tsx b/packages/app/src/app/index.tsx index c0c02f60f..c5dc68638 100644 --- a/packages/app/src/app/index.tsx +++ b/packages/app/src/app/index.tsx @@ -1,1710 +1,66 @@ -import { useEffect, useState, useRef } from "react"; -import { - View, - Pressable, - Text, - TextInput, - Platform, - KeyboardAvoidingView, - ScrollView, - Animated, - Keyboard, - Modal, -} from "react-native"; -import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller"; -import { GestureDetector, Gesture } from "react-native-gesture-handler"; -import ReanimatedAnimated, { - useAnimatedStyle, - useSharedValue, - withSpring, - withRepeat, - withSequence, - withTiming, - cancelAnimation, - runOnJS, - Easing, -} from "react-native-reanimated"; -import { router } from "expo-router"; -import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; +import { View } from "react-native"; +import { useState } from "react"; import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller"; +import ReanimatedAnimated, { useAnimatedStyle } from "react-native-reanimated"; import { StyleSheet, useUnistyles } from "react-native-unistyles"; -import { theme as defaultTheme, theme } from "../styles/theme"; - -import { useWebSocket } from "@/hooks/use-websocket"; -import { useAudioRecorder } from "@/hooks/use-audio-recorder"; -import { useAudioPlayer } from "@/hooks/use-audio-player"; -import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio"; -import { useSettings } from "@/hooks/use-settings"; -import { ConnectionStatus } from "@/components/connection-status"; -import { - UserMessage, - AssistantMessage, - ActivityLog, - ToolCall, -} from "@/components/message"; -import { ArtifactDrawer, type Artifact } from "@/components/artifact-drawer"; -import { AgentSidebar } from "@/components/agent-sidebar"; -import { AgentStreamView } from "@/components/agent-stream-view"; -import { ConversationSelector } from "@/components/conversation-selector"; -import { VolumeMeter } from "@/components/volume-meter"; -import { reduceStreamUpdate, generateMessageId, type StreamItem } from "@/types/stream"; +import { HomeHeader } from "@/components/headers/home-header"; +import { EmptyState } from "@/components/empty-state"; +import { AgentList } from "@/components/agent-list"; +import { GlobalFooter } from "@/components/global-footer"; import { CreateAgentModal } from "@/components/create-agent-modal"; -import { - Settings, - Mic, - ArrowUp, - Square, - AudioLines, - MicOff, - Plus, - ChevronDown, - Menu, - Home, -} from "lucide-react-native"; -import type { - ActivityLogPayload, - SessionInboundMessage, - WSInboundMessage, -} from "@server/server/messages"; -import type { AgentStatus } from "@server/server/acp/types"; -import type { SessionNotification } from "@agentclientprotocol/sdk"; -import { parseSessionUpdate } from "@/types/agent-activity"; +import { useSession } from "@/contexts/session-context"; -type MessageEntry = - | { - type: "user"; - id: string; - timestamp: number; - message: string; - } - | { - type: "assistant"; - id: string; - timestamp: number; - message: string; - } - | { - type: "activity"; - id: string; - timestamp: number; - activityType: "system" | "info" | "success" | "error"; - message: string; - metadata?: Record; - } - | { - type: "artifact"; - id: string; - timestamp: number; - artifactId: string; - artifactType: string; - title: string; - } - | { - type: "tool_call"; - id: string; - timestamp: number; - toolName: string; - args: any; - result?: any; - error?: any; - status: "executing" | "completed" | "failed"; - }; - -type ViewMode = "orchestrator" | "agent"; - -interface Agent { - id: string; - status: AgentStatus; - createdAt: Date; - type: "claude"; - sessionId?: string; - error?: string; - currentModeId?: string; - availableModes?: Array<{ - id: string; - name: string; - description?: string | null; - }>; - title?: string; - cwd: string; -} - -interface Command { - id: string; - name: string; - workingDirectory: string; - currentCommand: string; - isDead: boolean; - exitCode: number | null; -} - -interface AgentUpdate { - timestamp: Date; - notification: SessionNotification; -} - -interface RealtimeCircleProps { - volume: number; - onClose: () => void; -} - -function RealtimeCircle({ volume, onClose }: RealtimeCircleProps) { +export default function HomeScreen() { const { theme } = useUnistyles(); - - // Base size for the circles - const BASE_SIZE = 80; - const MIN_SCALE = 1; - const MAX_SCALE = 1.8; - - // Create animated value for volume-based scaling - const volumeScale = useSharedValue(MIN_SCALE); - const pulseScale = useSharedValue(1); - - // Update volume scale when volume changes - useEffect(() => { - // Map volume (0-1) to scale range (MIN_SCALE - MAX_SCALE) - const targetScale = MIN_SCALE + volume * (MAX_SCALE - MIN_SCALE); - volumeScale.value = withSpring(targetScale, { - damping: 15, - stiffness: 150, - }); - }, [volume]); - - // Continuous pulsating animation for outer circle - useEffect(() => { - pulseScale.value = withRepeat( - withSequence( - withTiming(1.3, { duration: 1000 }), - withTiming(1, { duration: 1000 }) - ), - -1, - false - ); - }, []); - - // Animated style for inner circle (volume reactive) - const innerCircleStyle = useAnimatedStyle(() => ({ - width: BASE_SIZE, - height: BASE_SIZE, - borderRadius: BASE_SIZE / 2, - transform: [{ scale: volumeScale.value }], - })); - - // Animated style for outer circle (continuous pulse) - const outerCircleStyle = useAnimatedStyle(() => ({ - width: BASE_SIZE * 1.5, - height: BASE_SIZE * 1.5, - borderRadius: (BASE_SIZE * 1.5) / 2, - transform: [{ scale: pulseScale.value }], - opacity: 0.3, - })); - - return ( - - - {/* Outer pulsating circle */} - - - {/* Inner volume-reactive circle */} - - - - ); -} - -export default function VoiceAssistantScreen() { - const { settings, isLoading: settingsLoading } = useSettings(); - const [conversationId, setConversationId] = useState(null); - const ws = useWebSocket(settings.serverUrl, conversationId); - - // Realtime mode state (defined early so we can use it in audioRecorder) - const [isRealtimeMode, setIsRealtimeMode] = useState(false); - const isRealtimeModeRef = useRef(isRealtimeMode); - const pulseAnim = useRef(new Animated.Value(1)).current; - - // Keep ref in sync with state - useEffect(() => { - isRealtimeModeRef.current = isRealtimeMode; - }, [isRealtimeMode]); - const insets = useSafeAreaInsets(); - const audioRecorder = useAudioRecorder(); - const audioPlayer = useAudioPlayer(); + const { agents, createAgent } = useSession(); + const [showCreateModal, setShowCreateModal] = useState(false); // Keyboard animation const { height: keyboardHeight } = useReanimatedKeyboardAnimation(); - const bottomInset = insets.bottom; const animatedKeyboardStyle = useAnimatedStyle(() => { "worklet"; const absoluteHeight = Math.abs(keyboardHeight.value); - const padding = Math.max(0, absoluteHeight - bottomInset); + const padding = Math.max(0, absoluteHeight - insets.bottom); return { paddingBottom: padding, }; }); - // Realtime audio with Speechmatics (echo cancellation) - const realtimeAudio = useSpeechmaticsAudio({ - onSpeechStart: () => { - console.log("[App] Speech detected"); - // Stop audio playback if playing - if (isPlayingAudio) { - audioPlayer.stop(); - } - }, - onSpeechEnd: () => { - console.log("[App] Speech ended"); - }, - onAudioSegment: (base64Audio: string) => { - console.log("[App] Sending audio segment, length:", base64Audio.length); + const hasAgents = agents.size > 0; - // Send audio segment to server (realtime always goes to orchestrator) - try { - ws.send({ - type: "session", - message: { - type: "realtime_audio_chunk", - audio: base64Audio, - format: "audio/wav", - isLast: true, // Complete segment - }, - }); - } catch (error) { - console.error("[App] Failed to send audio segment:", error); - } - }, - onError: (error) => { - console.error("[App] Realtime audio error:", error); - setMessages((prev) => [ - ...prev, - { - type: "activity", - id: generateMessageId(), - timestamp: Date.now(), - activityType: "error", - message: `Realtime audio error: ${error.message}`, - }, - ]); - }, - volumeThreshold: 0.3, - silenceDuration: 2000, - speechConfirmationDuration: 300, - detectionGracePeriod: 200, - }); - - const [messages, setMessages] = useState([]); - const [currentAssistantMessage, setCurrentAssistantMessage] = useState(""); - const [isProcessingAudio, setIsProcessingAudio] = useState(false); - - // Agent stream state - unified chronological stream using reducer pattern - const [agentStreamState, setAgentStreamState] = useState>(new Map()); - const [isPlayingAudio, setIsPlayingAudio] = useState(false); - const [isRecording, setIsRecording] = useState(false); - const [userInput, setUserInput] = useState(""); - - // Artifact state - const [artifacts, setArtifacts] = useState>(new Map()); - const [currentArtifact, setCurrentArtifact] = useState(null); - - // Multi-view navigation state - const [viewMode, setViewMode] = useState("orchestrator"); - const [activeAgentId, setActiveAgentId] = useState(null); - - // Agent creation modal state - const [showCreateAgentModal, setShowCreateAgentModal] = useState(false); - - // Agent sidebar state - const [sidebarOpen, setSidebarOpen] = useState(false); - - // Mode selector modal state - const [showModeSelector, setShowModeSelector] = useState(false); - - // Agent and command state - const [agents, setAgents] = useState>(new Map()); - const [commands, setCommands] = useState>(new Map()); - const [agentUpdates, setAgentUpdates] = useState>( - new Map() - ); - const [pendingPermissions, setPendingPermissions] = useState< - Map< - string, - { - agentId: string; - requestId: string; - sessionId: string; - toolCall: any; - options: Array<{ - kind: string; - name: string; - optionId: string; - }>; - } - > - >(new Map()); - - const scrollViewRef = useRef(null); - - // Pulse animation for speech indicator - useEffect(() => { - if (realtimeAudio.isSpeaking || realtimeAudio.isDetecting) { - Animated.loop( - Animated.sequence([ - Animated.timing(pulseAnim, { - toValue: 1.3, - duration: 400, - useNativeDriver: true, - }), - Animated.timing(pulseAnim, { - toValue: 1, - duration: 400, - useNativeDriver: true, - }), - ]) - ).start(); - } else { - pulseAnim.stopAnimation(); - pulseAnim.setValue(1); - } - }, [realtimeAudio.isSpeaking, realtimeAudio.isDetecting, pulseAnim]); - - // Keep screen awake if setting is enabled (mobile only) - useEffect(() => { - if (Platform.OS === "web") return; - - if (settings.keepScreenOn) { - activateKeepAwakeAsync("voice-assistant"); - } else { - deactivateKeepAwake("voice-assistant"); - } - }, [settings.keepScreenOn]); - - // Auto-scroll to bottom when messages update - useEffect(() => { - scrollViewRef.current?.scrollToEnd({ animated: true }); - }, [messages, currentAssistantMessage]); - - // WebSocket message handlers - useEffect(() => { - // Session state handler - initial agents/commands - const unsubSessionState = ws.on("session_state", (message) => { - if (message.type !== "session_state") return; - const { agents: agentsList, commands: commandsList } = message.payload; - - console.log( - "[App] Session state received:", - agentsList.length, - "agents,", - commandsList.length, - "commands" - ); - - // Update agents (convert createdAt string to Date) - setAgents(new Map(agentsList.map((a) => [a.id, { - ...a, - createdAt: new Date(a.createdAt), - } as Agent]))); - - // Update commands - setCommands(new Map(commandsList.map((c) => [c.id, c as Command]))); - }); - - // Agent created handler - const unsubAgentCreated = ws.on("agent_created", (message) => { - if (message.type !== "agent_created") return; - const { agentId, status, type, currentModeId, availableModes, title, cwd } = - message.payload; - - console.log("[App] Agent created:", agentId, "currentModeId:", currentModeId, "availableModes:", availableModes); - - const agent: Agent = { - id: agentId, - status: status as AgentStatus, - type, - createdAt: new Date(), - title, - cwd, - currentModeId, - availableModes, - }; - - setAgents((prev) => new Map(prev).set(agentId, agent)); - setAgentUpdates((prev) => new Map(prev).set(agentId, [])); - - // Auto-switch to agent view - setActiveAgentId(agentId); - setViewMode("agent"); - }); - - // Agent update handler - // Agent permission request handler - const unsubAgentPermissionRequest = ws.on("agent_permission_request", (message) => { - console.log("[App] agent_permission_request message received:", message.type); - if (message.type !== "agent_permission_request") return; - const { agentId, requestId, sessionId, toolCall, options } = message.payload; - - console.log("[App] Permission request received:", requestId, "for agent:", agentId); - console.log("[App] Permission details:", { agentId, requestId, toolCall, options }); - - // Store pending permission - setPendingPermissions((prev) => { - const updated = new Map(prev); - updated.set(requestId, { - agentId, - requestId, - sessionId, - toolCall, - options, - }); - return updated; - }); - }); - - const unsubAgentUpdate = ws.on("agent_update", (message) => { - if (message.type !== "agent_update") return; - const { agentId, timestamp, notification } = message.payload; - - console.log("[App] Agent update:", agentId); - - // Store raw update for agent stream view - setAgentUpdates((prev) => { - const updated = new Map(prev); - const updates = updated.get(agentId) || []; - updated.set(agentId, [ - ...updates, - { - timestamp: new Date(timestamp), - notification, - }, - ]); - return updated; - }); - - // Reduce stream update using unified reducer pattern - setAgentStreamState((prev) => { - const currentStream = prev.get(agentId) || []; - const nextStream = reduceStreamUpdate(currentStream, notification, new Date(timestamp)); - const updated = new Map(prev); - updated.set(agentId, nextStream); - return updated; - }); - }); - - // Agent status handler - const unsubAgentStatus = ws.on("agent_status", (message) => { - if (message.type !== "agent_status") return; - const { agentId, status, info } = message.payload; - - console.log("[App] Agent status changed:", agentId, status); - - setAgents((prev) => { - const updated = new Map(prev); - const agent = updated.get(agentId); - if (agent) { - updated.set(agentId, { - ...agent, - status: status as AgentStatus, - sessionId: info.sessionId, - error: info.error, - currentModeId: info.currentModeId, - availableModes: info.availableModes, - title: info.title, - cwd: info.cwd, - }); - } - return updated; - }); - }); - - // Activity log handler - const unsubActivity = ws.on("activity_log", (message) => { - if (message.type !== "activity_log") return; - const data = message.payload; - - // Filter out transcription activity logs - if (data.type === "system" && data.content.includes("Transcribing")) { - return; - } - - // Handle tool calls - if (data.type === "tool_call" && data.metadata) { - const { - toolCallId, - toolName, - arguments: args, - } = data.metadata as { - toolCallId: string; - toolName: string; - arguments: unknown; - }; - - setMessages((prev) => [ - ...prev, - { - type: "tool_call", - id: toolCallId, - timestamp: Date.now(), - toolName, - args, - status: "executing", - }, - ]); - return; - } - - // Handle tool results - if (data.type === "tool_result" && data.metadata) { - const { toolCallId, result } = data.metadata as { - toolCallId: string; - result: unknown; - }; - - setMessages((prev) => - prev.map((msg) => - msg.type === "tool_call" && msg.id === toolCallId - ? { ...msg, result, status: "completed" as const } - : msg - ) - ); - return; - } - - // Handle tool errors - if ( - data.type === "error" && - data.metadata && - "toolCallId" in data.metadata - ) { - const { toolCallId, error } = data.metadata as { - toolCallId: string; - error: unknown; - }; - - setMessages((prev) => - prev.map((msg) => - msg.type === "tool_call" && msg.id === toolCallId - ? { ...msg, error, status: "failed" as const } - : msg - ) - ); - } - - // Map activity types to message types - let activityType: "system" | "info" | "success" | "error" = "info"; - if (data.type === "error") activityType = "error"; - - // Add user transcripts as user messages - if (data.type === "transcript") { - setMessages((prev) => [ - ...prev, - { - type: "user", - id: generateMessageId(), - timestamp: Date.now(), - message: data.content, - }, - ]); - return; - } - - // Add assistant messages - if (data.type === "assistant") { - setMessages((prev) => [ - ...prev, - { - type: "assistant", - id: generateMessageId(), - timestamp: Date.now(), - message: data.content, - }, - ]); - setCurrentAssistantMessage(""); - return; - } - - // Add activity log for other types - setMessages((prev) => [ - ...prev, - { - type: "activity", - id: generateMessageId(), - timestamp: Date.now(), - activityType, - message: data.content, - metadata: data.metadata, - }, - ]); - }); - - // Assistant chunk handler (streaming) - const unsubChunk = ws.on("assistant_chunk", (message) => { - if (message.type !== "assistant_chunk") return; - setCurrentAssistantMessage((prev) => prev + message.payload.chunk); - }); - - // Transcription result handler - const unsubTranscription = ws.on("transcription_result", (message) => { - if (message.type !== "transcription_result") return; - - setIsProcessingAudio(false); - - const transcriptText = message.payload.text.trim(); - - if (!transcriptText) { - // Empty transcription - false positive, let playback continue - console.log("[App] Empty transcription (false positive) - ignoring"); - } else { - // Has content - real speech detected, stop playback - console.log("[App] Transcription received - stopping playback"); - audioPlayer.stop(); - setIsPlayingAudio(false); - setCurrentAssistantMessage(""); - } - }); - - // Audio output handler (TTS) - const unsubAudioOutput = ws.on("audio_output", async (message) => { - if (message.type !== "audio_output") return; - const data = message.payload; - - const currentIsRealtimeMode = isRealtimeModeRef.current; - - // Drift protection: Don't play audio generated in different mode - if (data.isRealtimeMode !== currentIsRealtimeMode) { - console.log( - `[App] Skipping audio playback due to mode drift (generated in ${ - data.isRealtimeMode ? "realtime" : "normal" - } mode, currently in ${ - currentIsRealtimeMode ? "realtime" : "normal" - } mode)` - ); - - // Still send confirmation to prevent server from waiting - const confirmMessage: WSInboundMessage = { - type: "session", - message: { - type: "audio_played", - id: data.id, - }, - }; - ws.send(confirmMessage); - return; - } - - // Additional check: Don't play if NOT in realtime mode (shouldn't happen with server-side fix, but defense in depth) - if (!currentIsRealtimeMode) { - console.log("[App] Skipping audio playback - not in realtime mode"); - - // Still send confirmation - const confirmMessage: WSInboundMessage = { - type: "session", - message: { - type: "audio_played", - id: data.id, - }, - }; - ws.send(confirmMessage); - return; - } - - try { - setIsPlayingAudio(true); - - // Create blob-like object with correct mime type (React Native compatible) - const mimeType = - data.format === "mp3" ? "audio/mpeg" : `audio/${data.format}`; - const base64Audio = data.audio; - - // Create a Blob-like object that works in React Native - const audioBlob = { - type: mimeType, - size: Math.ceil((base64Audio.length * 3) / 4), // Approximate size from base64 - arrayBuffer: async () => { - // Convert base64 to ArrayBuffer - const binaryString = atob(base64Audio); - const bytes = new Uint8Array(binaryString.length); - for (let i = 0; i < binaryString.length; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - return bytes.buffer; - }, - } as Blob; - - // Play audio - await audioPlayer.play(audioBlob); - - // Send confirmation back to server (properly typed) - const confirmMessage: WSInboundMessage = { - type: "session", - message: { - type: "audio_played", - id: data.id, - }, - }; - ws.send(confirmMessage); - - setIsPlayingAudio(false); - } catch (error: any) { - console.error("[App] Audio playback error:", error); - setMessages((prev) => [ - ...prev, - { - type: "activity", - id: generateMessageId(), - timestamp: Date.now(), - activityType: "error", - message: `Audio playback failed: ${error.message}`, - }, - ]); - setIsPlayingAudio(false); - } - }); - - // Status handler - const unsubStatus = ws.on("status", (message) => { - if (message.type !== "status") return; - const msg = - "message" in message.payload - ? String(message.payload.message) - : `Status: ${message.payload.status}`; - - setMessages((prev) => [ - ...prev, - { - type: "activity", - id: generateMessageId(), - timestamp: Date.now(), - activityType: "info", - message: msg, - }, - ]); - }); - - // Conversation loaded handler - const unsubConversationLoaded = ws.on("conversation_loaded", (message) => { - if (message.type !== "conversation_loaded") return; - // Don't show message in UI - }); - - // Artifact handler - const unsubArtifact = ws.on("artifact", (message) => { - if (message.type !== "artifact") return; - const artifactData = message.payload; - - console.log( - "[App] Received artifact:", - artifactData.id, - artifactData.type, - artifactData.title - ); - - // Store artifact - setArtifacts((prev) => { - const updated = new Map(prev); - updated.set(artifactData.id, artifactData); - return updated; - }); - - // Add artifact entry to chat history - setMessages((prev) => [ - ...prev, - { - type: "artifact", - id: generateMessageId(), - timestamp: Date.now(), - artifactId: artifactData.id, - artifactType: artifactData.type, - title: artifactData.title, - }, - ]); - - // Show drawer immediately - setCurrentArtifact(artifactData); - }); - - return () => { - unsubSessionState(); - unsubAgentCreated(); - unsubAgentPermissionRequest(); - unsubAgentUpdate(); - unsubAgentStatus(); - unsubActivity(); - unsubChunk(); - unsubTranscription(); - unsubAudioOutput(); - unsubStatus(); - unsubConversationLoaded(); - unsubArtifact(); - }; - }, [ws, audioPlayer]); - - // Voice button handler - async function handleVoicePress() { - if (!ws.isConnected) return; - - // If recording, stop and send - if (isRecording) { - try { - console.log("[App] Stopping recording..."); - const audioBlob = await audioRecorder.stop(); - setIsRecording(false); - - const format = audioBlob.type || "audio/m4a"; - console.log( - `[App] Recording complete: ${audioBlob.size} bytes, format: ${format}` - ); - - setIsProcessingAudio(true); - - // Convert to base64 - const arrayBuffer = await audioBlob.arrayBuffer(); - const base64Audio = btoa( - new Uint8Array(arrayBuffer).reduce( - (data, byte) => data + String.fromCharCode(byte), - "" - ) - ); - - // Route audio based on view mode - let audioMessage: WSInboundMessage; - if (isRealtimeMode) { - // Send as realtime audio chunk to orchestrator (speech-to-speech with TTS) - audioMessage = { - type: "session", - message: { - type: "realtime_audio_chunk", - audio: base64Audio, - format: format, - isLast: true, - }, - }; - } else if (viewMode === "agent" && activeAgentId) { - // Send as agent audio (will be transcribed, no TTS) - audioMessage = { - type: "session", - message: { - type: "send_agent_audio", - agentId: activeAgentId, - audio: base64Audio, - format: format, - isLast: true, - }, - }; - } else { - // Send as regular audio to orchestrator (will be transcribed, no TTS) - audioMessage = { - type: "session", - message: { - type: "realtime_audio_chunk", - audio: base64Audio, - format: format, - isLast: true, - }, - }; - } - - ws.send(audioMessage); - - console.log( - `[App] Sent audio: ${audioBlob.size} bytes, format: ${format}` - ); - } catch (error: any) { - console.error("[App] Recording error:", error); - setMessages((prev) => [ - ...prev, - { - type: "activity", - id: generateMessageId(), - timestamp: Date.now(), - activityType: "error", - message: `Failed to record audio: ${error.message}`, - }, - ]); - setIsRecording(false); - } - } else { - // Start recording - try { - console.log("[App] Starting recording..."); - - // Stop any currently playing audio - audioPlayer.stop(); - setIsPlayingAudio(false); - - await audioRecorder.start(); - setIsRecording(true); - } catch (error: any) { - console.error("[App] Failed to start recording:", error); - setMessages((prev) => [ - ...prev, - { - type: "activity", - id: generateMessageId(), - timestamp: Date.now(), - activityType: "error", - message: `Failed to start recording: ${error.message}`, - }, - ]); - } - } + function handleCreateAgent() { + setShowCreateModal(true); } - // Handle artifact click from activity log - function handleArtifactClick(artifactId: string) { - const artifact = artifacts.get(artifactId); - if (artifact) { - console.log("[App] Opening artifact:", artifactId); - setCurrentArtifact(artifact); - } else { - console.warn("[App] Artifact not found:", artifactId); - } + function handleCreateAgentConfirm(workingDir: string, mode: string) { + createAgent({ cwd: workingDir, autoStart: true }); + setShowCreateModal(false); } - // Close artifact drawer - function handleCloseArtifact() { - setCurrentArtifact(null); - } - - // Handle agent selection - function handleSelectAgent(agentId: string) { - setActiveAgentId(agentId); - setViewMode("agent"); - } - - // Handle back to orchestrator - function handleBackToOrchestrator() { - setActiveAgentId(null); - setViewMode("orchestrator"); - } - - // Agent control handlers - function handleKillAgent(agentId: string) { - console.log("[App] Kill agent:", agentId); - // TODO: Implement kill agent API call - } - - function handleCancelAgent(agentId: string) { - console.log("[App] Cancel agent:", agentId); - // TODO: Implement cancel agent API call - } - - // Agent creation handler - function handleCreateAgent(workingDir: string, mode: string) { - console.log("[App] Creating agent in:", workingDir, "with mode:", mode); - - // Send create agent request to server - const message: WSInboundMessage = { - type: "session", - message: { - type: "create_agent_request", - cwd: workingDir, - initialMode: mode, - }, - }; - ws.send(message); - - // Close modal - setShowCreateAgentModal(false); - - // The agent_created event handler will switch to agent view automatically - } - - // Mode change handler - function handleModeChange(modeId: string) { - if (!activeAgentId) return; - - const message: WSInboundMessage = { - type: "session", - message: { - type: "set_agent_mode", - agentId: activeAgentId, - modeId, - }, - }; - ws.send(message); - setShowModeSelector(false); - } - - // Permission response handler - function handlePermissionResponse(requestId: string, optionId: string) { - const permission = pendingPermissions.get(requestId); - if (!permission) { - console.error("[App] Permission not found:", requestId); - return; - } - - console.log("[App] Responding to permission:", requestId, "with option:", optionId); - - const message: WSInboundMessage = { - type: "session", - message: { - type: "agent_permission_response", - agentId: permission.agentId, - requestId, - optionId, - }, - }; - - ws.send(message); - - // Remove from pending permissions - setPendingPermissions((prev) => { - const updated = new Map(prev); - updated.delete(requestId); - return updated; - }); - } - - // Text message handlers - function handleSendMessage() { - if (!userInput.trim() || !ws.isConnected) return; - - // Stop any currently playing audio - audioPlayer.stop(); - setIsPlayingAudio(false); - - // Route message based on view mode - if (isRealtimeMode) { - // Realtime mode always routes to orchestrator (handled by realtime audio) - ws.sendUserMessage(userInput); - } else if (viewMode === "agent" && activeAgentId) { - // Generate unique message ID for deduplication - const messageId = generateMessageId(); - - // Optimistically add user message to stream - setAgentStreamState((prev) => { - const currentStream = prev.get(activeAgentId) || []; - const nextStream = reduceStreamUpdate( - currentStream, - { - type: "sessionUpdate", - update: { - sessionUpdate: "user_message_chunk", - content: { type: "text", text: userInput }, - messageId, - }, - }, - new Date() - ); - const updated = new Map(prev); - updated.set(activeAgentId, nextStream); - return updated; - }); - - // Send to agent with messageId - const message: WSInboundMessage = { - type: "session", - message: { - type: "send_agent_message", - agentId: activeAgentId, - text: userInput, - messageId, - }, - }; - ws.send(message); - } else { - // Send to orchestrator - ws.sendUserMessage(userInput); - } - - // Clear input and reset streaming state - setUserInput(""); - setCurrentAssistantMessage(""); - } - - function handleCancel() { - console.log("[App] Cancelling operations..."); - - // Stop audio playback - audioPlayer.stop(); - setIsPlayingAudio(false); - - // Clear streaming state - setCurrentAssistantMessage(""); - - // Reset processing state - setIsProcessingAudio(false); - - // Send abort request to server (properly typed) - const abortMessage: WSInboundMessage = { - type: "session", - message: { - type: "abort_request", - }, - }; - ws.send(abortMessage); - - setMessages((prev) => [ - ...prev, - { - type: "activity", - id: generateMessageId(), - timestamp: Date.now(), - activityType: "info", - message: "Operations cancelled", - }, - ]); - } - - // Compute if we're processing - const isInProgress = - isProcessingAudio || isPlayingAudio || currentAssistantMessage.length > 0; - - function handleButtonClick() { - if (isRecording) { - // Stop recording and send the audio - handleVoicePress(); - } else if (isInProgress) { - // Cancel processing/playback - handleCancel(); - } else if (userInput.trim()) { - // Send text message - handleSendMessage(); - } else { - // Start recording - handleVoicePress(); - } - } - - // Realtime mode toggle handler - async function handleRealtimeToggle() { - const newRealtimeMode = !isRealtimeMode; - - if (newRealtimeMode) { - // Start realtime mode - try { - await realtimeAudio.start(); - setIsRealtimeMode(true); - console.log("[App] Realtime mode enabled"); - - // Notify server of mode change - const modeMessage: WSInboundMessage = { - type: "session", - message: { - type: "set_realtime_mode", - enabled: true, - }, - }; - ws.send(modeMessage); - } catch (error: any) { - console.error("[App] Failed to start realtime mode:", error); - } - } else { - // Stop realtime mode - try { - await realtimeAudio.stop(); - setIsRealtimeMode(false); - console.log("[App] Realtime mode disabled"); - - // Notify server of mode change - const modeMessage: WSInboundMessage = { - type: "session", - message: { - type: "set_realtime_mode", - enabled: false, - }, - }; - ws.send(modeMessage); - } catch (error: any) { - console.error("[App] Failed to stop realtime mode:", error); - } - } - } - - // Conversation selection handler - function handleSelectConversation(newConversationId: string | null) { - // Clear all state - setMessages([]); - setAgentStreamState(new Map()); - setCurrentAssistantMessage(""); - setUserInput(""); - setArtifacts(new Map()); - setCurrentArtifact(null); - setAgents(new Map()); - setCommands(new Map()); - setAgentUpdates(new Map()); - setPendingPermissions(new Map()); - setViewMode("orchestrator"); - setActiveAgentId(null); - - // Stop any ongoing operations - if (isRecording) { - audioRecorder.stop().catch(console.error); - setIsRecording(false); - } - audioPlayer.stop(); - setIsPlayingAudio(false); - setIsProcessingAudio(false); - - // Update conversation ID (will trigger WebSocket reconnection) - setConversationId(newConversationId); - } - - // Calculate agent data (used when viewMode === "agent") - const agent = activeAgentId ? agents.get(activeAgentId) : null; - const streamItems = activeAgentId ? (agentStreamState.get(activeAgentId) || []) : []; - - // Edge swipe state - const edgeSwipeTranslateX = useSharedValue(-300); - const isEdgeSwiping = useSharedValue(false); - - // Edge swipe gesture to open sidebar - const edgeSwipeGesture = Gesture.Pan() - .enabled(!sidebarOpen) - .activeOffsetX(10) // require a rightward intent - .failOffsetY([-15, 15]) // ignore primarily vertical swipes - .onStart(() => { - isEdgeSwiping.value = true; - cancelAnimation(edgeSwipeTranslateX); - edgeSwipeTranslateX.value = -300; - }) - .onChange((event) => { - if (!isEdgeSwiping.value) { - return; - } - // Move sidebar from -300 (closed) to 0 (open) as the finger moves - const newX = -300 + event.translationX; - edgeSwipeTranslateX.value = Math.max(-300, Math.min(0, newX)); - }) - .onEnd((event) => { - if (!isEdgeSwiping.value) { - edgeSwipeTranslateX.value = -300; - return; - } - - const shouldOpen = edgeSwipeTranslateX.value > -150 || event.velocityX > 500; - - edgeSwipeTranslateX.value = withTiming(shouldOpen ? 0 : -300, { - duration: 150, - easing: Easing.out(Easing.ease), - }); - - if (shouldOpen) { - runOnJS(setSidebarOpen)(true); - } - - isEdgeSwiping.value = false; - }); - - // Render main view with shared structure return ( - {/* Fixed Header */} - - - - - {/* Menu button to open sidebar */} - { - edgeSwipeTranslateX.value = -300; - setSidebarOpen(true); - }} - style={styles.menuButton} - > - - + {/* Header */} + - {/* Orchestrator button (only show in agent view) */} - {viewMode === "agent" && ( - - - Orchestrator - - )} - - - - - - router.push("/settings")} - style={styles.settingsButton} - > - - - - - - - - {/* Content Area with Keyboard Handling */} - - - {/* Conditionally render content based on view mode */} - {viewMode === "agent" && activeAgentId && agent ? ( - // Agent view - render AgentStreamView - - ) : viewMode === "agent" && activeAgentId && !agent ? ( - // Agent not found - - Agent not found - + {/* Content Area with Keyboard Animation */} + + {hasAgents ? ( + ) : ( - // Orchestrator view - render scrollable messages - - {messages.length === 0 && !currentAssistantMessage && ( - - Hammock - - What would you like to work on? - - - )} - - {messages.map((msg) => { - if (msg.type === "user") { - return ( - - ); - } - - if (msg.type === "assistant") { - return ( - - ); - } - - if (msg.type === "activity") { - return ( - - ); - } - - if (msg.type === "artifact") { - return ( - - ); - } - - if (msg.type === "tool_call") { - return ( - - ); - } - - return null; - })} - - {/* Streaming assistant message */} - {currentAssistantMessage && ( - - )} - + )} - {/* Fixed Footer */} - - {isRealtimeMode ? ( - // Realtime mode - show volume meter and mute button - - - - {/* Debug timer */} - {(realtimeAudio.isDetecting || realtimeAudio.isSpeaking) && ( - - {(realtimeAudio.segmentDuration / 1000).toFixed(1)}s - - )} - - - {/* Mute button */} - realtimeAudio.toggleMute()} - style={[ - styles.realtimeMuteButton, - realtimeAudio.isMuted && styles.realtimeMuteButtonActive, - ]} - > - - - {/* Close button */} - - - - - - ) : ( - // Normal mode - show text input and buttons - - {/* Text input */} - + {/* Footer */} + + - {/* Mode badge and buttons row */} - - {/* Session mode badge - only show in agent view */} - {viewMode === "agent" && activeAgentId && agent && ( - setShowModeSelector(true)} - style={({ pressed }) => [ - styles.modeBadge, - pressed && styles.modeBadgePressed, - ]} - > - - {agent.availableModes?.find(m => m.id === agent.currentModeId)?.name || agent.currentModeId || 'default'} - - - - )} - - {/* Buttons */} - - {userInput.trim().length > 0 ? ( - // Send button when text is entered - - - - ) : ( - // Record and Realtime buttons when no text - <> - {/* Main action button */} - - {isInProgress ? ( - - ) : isRecording ? ( - - ) : ( - - )} - - - {/* Realtime mode button */} - - - - - - - )} - - - - )} - - - - - {/* Artifact drawer */} - - - {/* Create agent modal */} + {/* Create Agent Modal */} setShowCreateAgentModal(false)} - onCreateAgent={handleCreateAgent} - /> - - {/* Mode selector modal */} - setShowModeSelector(false)} - > - setShowModeSelector(false)} - > - - {agent?.availableModes?.map((mode) => { - const isActive = mode.id === agent.currentModeId; - return ( - handleModeChange(mode.id)} - style={[ - styles.modeItem, - isActive && styles.modeItemActive, - ]} - > - {mode.name} - {mode.description && ( - {mode.description} - )} - - ); - })} - - - - - {/* Agent sidebar */} - { - setSidebarOpen(false); - // Reset edge swipe position when closing - edgeSwipeTranslateX.value = -300; - }} - onSelectAgent={handleSelectAgent} - onNewAgent={() => setShowCreateAgentModal(true)} - edgeSwipeTranslateX={edgeSwipeTranslateX} + isVisible={showCreateModal} + onClose={() => setShowCreateModal(false)} + onCreateAgent={handleCreateAgentConfirm} /> ); @@ -1715,260 +71,7 @@ const styles = StyleSheet.create((theme) => ({ flex: 1, backgroundColor: theme.colors.background, }, - agentNotFoundContainer: { + content: { flex: 1, - alignItems: "center", - justifyContent: "center", - padding: theme.spacing[6], - }, - agentNotFoundText: { - color: theme.colors.mutedForeground, - fontSize: theme.fontSize.lg, - }, - header: { - backgroundColor: theme.colors.background, - }, - contentArea: { - flex: 1, - minHeight: 0, - }, - headerRow: { - flexDirection: "row", - alignItems: "center", - justifyContent: "space-between", - paddingHorizontal: theme.spacing[6], - paddingBottom: theme.spacing[4], - borderBottomWidth: theme.borderWidth[1], - borderBottomColor: theme.colors.border, - }, - headerLeft: { - flex: 1, - flexDirection: "row", - alignItems: "center", - gap: theme.spacing[3], - }, - menuButton: { - padding: theme.spacing[2], - }, - orchestratorButton: { - flexDirection: "row", - alignItems: "center", - gap: theme.spacing[2], - paddingVertical: theme.spacing[2], - paddingHorizontal: theme.spacing[3], - backgroundColor: theme.colors.muted, - borderRadius: theme.borderRadius.md, - }, - orchestratorButtonText: { - color: theme.colors.foreground, - fontSize: theme.fontSize.sm, - fontWeight: theme.fontWeight.medium, - }, - headerRight: { - flexDirection: "row", - alignItems: "center", - gap: theme.spacing[2], - }, - settingsButton: { - backgroundColor: theme.colors.muted, - padding: theme.spacing[3], - borderRadius: theme.borderRadius.lg, - }, - scrollView: { - flex: 1, - minHeight: 0, - }, - scrollContent: { - paddingTop: theme.spacing[6], - paddingBottom: theme.spacing[4], - flexGrow: 1, - }, - emptyStateContainer: { - flex: 1, - justifyContent: "center", - alignItems: "center", - }, - emptyState: { - alignItems: "center", - justifyContent: "center", - paddingHorizontal: theme.spacing[6], - }, - emptyStateTitle: { - fontSize: theme.fontSize["4xl"], - fontWeight: "700", - color: theme.colors.foreground, - marginBottom: theme.spacing[2], - }, - emptyStateSubtitle: { - fontSize: theme.fontSize.lg, - color: theme.colors.mutedForeground, - textAlign: "center", - }, - inputAreaWrapper: { - borderTopRightRadius: theme.borderRadius["2xl"], - borderTopLeftRadius: theme.borderRadius["2xl"], - borderTopWidth: theme.borderWidth[1], - borderTopColor: theme.colors.border, - backgroundColor: theme.colors.muted, - }, - inputArea: {}, - textInput: { - paddingTop: theme.spacing[4], - paddingHorizontal: theme.spacing[4], - borderRadius: theme.borderRadius["2xl"], - paddingVertical: theme.spacing[3], - backgroundColor: "transparent", - color: theme.colors.foreground, - fontSize: theme.fontSize.lg, - marginBottom: theme.spacing[3], - maxHeight: 128, - }, - controlsRow: { - flexDirection: "row", - alignItems: "center", - gap: theme.spacing[2], - marginBottom: theme.spacing[1], - paddingHorizontal: theme.spacing[4], - paddingBottom: theme.spacing[2], - }, - buttonRow: { - flexDirection: "row", - alignItems: "center", - gap: theme.spacing[2], - marginLeft: "auto", - }, - realtimeButton: { - width: 40, - height: 40, - borderRadius: theme.borderRadius.full, - alignItems: "center", - justifyContent: "center", - backgroundColor: theme.colors.accentForeground, - }, - realtimeButtonActive: { - backgroundColor: theme.colors.foreground, - }, - mainButton: { - width: 40, - height: 40, - borderRadius: theme.borderRadius.full, - alignItems: "center", - justifyContent: "center", - backgroundColor: "transparent", - }, - mainButtonRecording: { - backgroundColor: theme.colors.palette.red[500], - }, - sendButton: { - width: 40, - height: 40, - borderRadius: theme.borderRadius.full, - alignItems: "center", - justifyContent: "center", - backgroundColor: theme.colors.palette.blue[600], - }, - mainButtonInProgress: { - backgroundColor: theme.colors.palette.red[600], - }, - mainButtonWithText: { - backgroundColor: theme.colors.palette.blue[600], - }, - buttonDisabled: { - opacity: theme.opacity[50], - }, - realtimeModeButtons: { - flexDirection: "row", - alignItems: "center", - justifyContent: "center", - gap: theme.spacing[3], - paddingTop: theme.spacing[4], - }, - realtimeMuteButton: { - width: 48, - height: 48, - borderRadius: theme.borderRadius.full, - alignItems: "center", - justifyContent: "center", - backgroundColor: theme.colors.muted, - borderWidth: theme.borderWidth[2], - borderColor: theme.colors.border, - }, - realtimeMuteButtonActive: { - backgroundColor: theme.colors.palette.red[500], - borderColor: theme.colors.palette.red[600], - }, - realtimeCloseButton: { - width: 48, - height: 48, - borderRadius: theme.borderRadius.full, - alignItems: "center", - justifyContent: "center", - backgroundColor: theme.colors.palette.red[600], - }, - debugTimer: { - marginTop: theme.spacing[2], - color: theme.colors.mutedForeground, - fontSize: theme.fontSize.sm, - fontFamily: "monospace", - }, - modeBadge: { - flexDirection: 'row', - alignItems: 'center', - gap: theme.spacing[2], - paddingHorizontal: theme.spacing[3], - paddingVertical: theme.spacing[2], - backgroundColor: theme.colors.palette.blue[950], - borderRadius: theme.borderRadius.full, - borderWidth: theme.borderWidth[1], - borderColor: theme.colors.palette.blue[800], - }, - modeBadgePressed: { - backgroundColor: theme.colors.palette.blue[900], - borderColor: theme.colors.palette.blue[700], - }, - modeBadgeText: { - color: theme.colors.palette.blue[400], - fontSize: theme.fontSize.xs, - fontWeight: theme.fontWeight.semibold, - textTransform: 'capitalize', - }, - modalOverlay: { - flex: 1, - backgroundColor: 'rgba(0,0,0,0.5)', - justifyContent: 'center', - alignItems: 'center', - }, - modeSelectorContent: { - backgroundColor: theme.colors.card, - borderRadius: theme.borderRadius.lg, - padding: theme.spacing[4], - minWidth: 280, - maxWidth: 320, - }, - modeItem: { - padding: theme.spacing[4], - borderRadius: theme.borderRadius.md, - marginBottom: theme.spacing[2], - backgroundColor: theme.colors.muted, - }, - modeItemActive: { - backgroundColor: theme.colors.primary, - }, - modeName: { - color: theme.colors.foreground, - fontSize: theme.fontSize.base, - fontWeight: theme.fontWeight.semibold, - marginBottom: theme.spacing[1], - }, - modeNameActive: { - color: theme.colors.primaryForeground, - }, - modeDescription: { - color: theme.colors.mutedForeground, - fontSize: theme.fontSize.sm, - }, - modeDescriptionActive: { - color: theme.colors.primaryForeground, - opacity: theme.opacity[80], }, })); diff --git a/packages/app/src/app/orchestrator.tsx b/packages/app/src/app/orchestrator.tsx new file mode 100644 index 000000000..e9665339c --- /dev/null +++ b/packages/app/src/app/orchestrator.tsx @@ -0,0 +1,66 @@ +import { View } from "react-native"; +import { useRef, useState } from "react"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller"; +import ReanimatedAnimated, { useAnimatedStyle } from "react-native-reanimated"; +import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import { BackHeader } from "@/components/headers/back-header"; +import { OrchestratorMessagesView } from "@/components/orchestrator-messages-view"; +import { GlobalFooter } from "@/components/global-footer"; +import { useSession } from "@/contexts/session-context"; +import type { ScrollView } from "react-native"; +import type { Artifact } from "@/components/artifact-drawer"; + +export default function OrchestratorScreen() { + const { theme } = useUnistyles(); + const insets = useSafeAreaInsets(); + const { messages, currentAssistantMessage } = useSession(); + const scrollViewRef = useRef(null); + const [currentArtifact, setCurrentArtifact] = useState(null); + + // Keyboard animation + const { height: keyboardHeight } = useReanimatedKeyboardAnimation(); + const animatedKeyboardStyle = useAnimatedStyle(() => { + "worklet"; + const absoluteHeight = Math.abs(keyboardHeight.value); + const padding = Math.max(0, absoluteHeight - insets.bottom); + return { + paddingBottom: padding, + }; + }); + + function handleArtifactClick(artifactId: string) { + // TODO: Implement artifact drawer + console.log("[Orchestrator] Artifact clicked:", artifactId); + } + + return ( + + {/* Header */} + + + {/* Content Area with Keyboard Animation */} + + + + {/* Footer */} + + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + container: { + flex: 1, + backgroundColor: theme.colors.background, + }, + content: { + flex: 1, + }, +})); diff --git a/packages/app/src/app/settings.tsx b/packages/app/src/app/settings.tsx index 2b60283ef..0e7bc9ba5 100644 --- a/packages/app/src/app/settings.tsx +++ b/packages/app/src/app/settings.tsx @@ -10,10 +10,10 @@ import { ActivityIndicator, } from "react-native"; import { router } from "expo-router"; -import { useSafeAreaInsets } from "react-native-safe-area-context"; import { StyleSheet } from "react-native-unistyles"; import { useSettings } from "@/hooks/use-settings"; import { theme as defaultTheme } from "@/styles/theme"; +import { BackHeader } from "@/components/headers/back-header"; const styles = StyleSheet.create((theme) => ({ loadingContainer: { @@ -30,27 +30,6 @@ const styles = StyleSheet.create((theme) => ({ flex: 1, backgroundColor: theme.colors.background, }, - header: { - paddingHorizontal: theme.spacing[6], - paddingBottom: theme.spacing[4], - borderBottomWidth: theme.borderWidth[1], - borderBottomColor: theme.colors.border, - }, - headerRow: { - flexDirection: "row", - alignItems: "center", - justifyContent: "space-between", - }, - headerTitle: { - color: theme.colors.foreground, - fontSize: theme.fontSize["3xl"], - fontWeight: theme.fontWeight.bold, - }, - cancelButton: { - color: theme.colors.palette.blue[500], - fontSize: theme.fontSize.base, - fontWeight: theme.fontWeight.semibold, - }, scrollView: { flex: 1, }, @@ -233,7 +212,6 @@ const styles = StyleSheet.create((theme) => ({ export default function SettingsScreen() { const { settings, isLoading, updateSettings, resetSettings } = useSettings(); - const insets = useSafeAreaInsets(); const [serverUrl, setServerUrl] = useState(settings.serverUrl); const [useSpeaker, setUseSpeaker] = useState(settings.useSpeaker); @@ -337,24 +315,6 @@ export default function SettingsScreen() { ); } - function handleCancel() { - if (hasChanges) { - Alert.alert( - "Discard Changes", - "You have unsaved changes. Are you sure you want to go back?", - [ - { text: "Stay", style: "cancel" }, - { - text: "Discard", - style: "destructive", - onPress: () => router.back(), - }, - ] - ); - } else { - router.back(); - } - } async function handleTestConnection() { if (!validateServerUrl(serverUrl)) { @@ -418,15 +378,7 @@ export default function SettingsScreen() { return ( - {/* Header */} - - - Settings - - Cancel - - - + diff --git a/packages/app/src/components/agent-input-area.tsx b/packages/app/src/components/agent-input-area.tsx new file mode 100644 index 000000000..6c1804d67 --- /dev/null +++ b/packages/app/src/components/agent-input-area.tsx @@ -0,0 +1,246 @@ +import { View, TextInput, Pressable, Text } from "react-native"; +import { useState } from "react"; +import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import { Mic, ArrowUp, AudioLines, Square, ChevronDown } from "lucide-react-native"; +import { useSession } from "@/contexts/session-context"; +import { useRealtime } from "@/contexts/realtime-context"; +import { useAudioRecorder } from "@/hooks/use-audio-recorder"; +import { ModeSelectorModal } from "./mode-selector-modal"; + +interface AgentInputAreaProps { + agentId: string; +} + +export function AgentInputArea({ agentId }: AgentInputAreaProps) { + const { theme } = useUnistyles(); + const { agents, ws, sendAgentMessage, setAgentMode } = useSession(); + const { startRealtime } = useRealtime(); + const audioRecorder = useAudioRecorder(); + + const [userInput, setUserInput] = useState(""); + const [isRecording, setIsRecording] = useState(false); + const [isProcessing, setIsProcessing] = useState(false); + const [showModeSelector, setShowModeSelector] = useState(false); + + const agent = agents.get(agentId); + + async function handleSendMessage() { + if (!userInput.trim() || !ws.isConnected) return; + + const message = userInput.trim(); + setUserInput(""); + setIsProcessing(true); + + try { + sendAgentMessage(agentId, message); + } catch (error) { + console.error("[AgentInput] Failed to send message:", error); + } finally { + setIsProcessing(false); + } + } + + async function handleVoicePress() { + if (isRecording) { + // Stop recording + try { + setIsRecording(false); + const audioData = await audioRecorder.stop(); + + if (audioData) { + setIsProcessing(true); + // TODO: Send audio to agent + // For now, just log it + console.log("[AgentInput] Audio recorded:", audioData.size, "bytes"); + setIsProcessing(false); + } + } catch (error) { + console.error("[AgentInput] Failed to stop recording:", error); + setIsRecording(false); + setIsProcessing(false); + } + } else { + // Start recording + try { + await audioRecorder.start(); + setIsRecording(true); + } catch (error) { + console.error("[AgentInput] Failed to start recording:", error); + } + } + } + + function handleModeChange(modeId: string) { + setAgentMode(agentId, modeId); + } + + const hasText = userInput.trim().length > 0; + + return ( + + {/* Text input */} + + + {/* Controls row */} + + {/* Mode badge - only show if agent has modes */} + {agent && agent.availableModes && agent.availableModes.length > 0 && ( + setShowModeSelector(true)} + style={({ pressed }) => [ + styles.modeBadge, + pressed && styles.modeBadgePressed, + ]} + > + + {agent.availableModes?.find(m => m.id === agent.currentModeId)?.name || agent.currentModeId || 'default'} + + + + )} + + {/* Buttons */} + + {hasText ? ( + // Send button when text is entered + + + + ) : ( + // Voice and Realtime buttons when no text + <> + {/* Voice recording button */} + + {isRecording ? ( + + ) : ( + + )} + + + {/* Realtime button */} + + + + + )} + + + + {/* Mode selector modal */} + setShowModeSelector(false)} + /> + + ); +} + +const styles = StyleSheet.create((theme) => ({ + container: { + padding: theme.spacing[4], + gap: theme.spacing[3], + }, + textInput: { + minHeight: 44, + maxHeight: 120, + paddingHorizontal: theme.spacing[4], + paddingVertical: theme.spacing[3], + backgroundColor: theme.colors.muted, + borderRadius: theme.borderRadius.lg, + color: theme.colors.foreground, + fontSize: theme.fontSize.base, + }, + controlsRow: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + gap: theme.spacing[2], + }, + modeBadge: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + paddingHorizontal: theme.spacing[3], + paddingVertical: theme.spacing[2], + backgroundColor: theme.colors.muted, + borderRadius: theme.borderRadius.full, + }, + modeBadgePressed: { + backgroundColor: theme.colors.accent, + }, + modeBadgeText: { + color: theme.colors.mutedForeground, + fontSize: theme.fontSize.xs, + fontWeight: theme.fontWeight.semibold, + textTransform: "capitalize", + }, + buttonRow: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + marginLeft: "auto", + }, + sendButton: { + width: 40, + height: 40, + borderRadius: theme.borderRadius.full, + backgroundColor: theme.colors.palette.blue[600], + alignItems: "center", + justifyContent: "center", + }, + voiceButton: { + width: 40, + height: 40, + borderRadius: theme.borderRadius.full, + backgroundColor: "transparent", + alignItems: "center", + justifyContent: "center", + }, + voiceButtonRecording: { + backgroundColor: theme.colors.destructive, + }, + realtimeButton: { + width: 40, + height: 40, + borderRadius: theme.borderRadius.full, + backgroundColor: theme.colors.accentForeground, + alignItems: "center", + justifyContent: "center", + }, + buttonDisabled: { + opacity: 0.5, + }, +})); diff --git a/packages/app/src/components/agent-list.tsx b/packages/app/src/components/agent-list.tsx new file mode 100644 index 000000000..5eaaef888 --- /dev/null +++ b/packages/app/src/components/agent-list.tsx @@ -0,0 +1,148 @@ +import { View, Text, Pressable, ScrollView } from "react-native"; +import { router } from "expo-router"; +import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import type { Agent } from "@/contexts/session-context"; +import type { AgentStatus } from "@server/server/acp/types"; + +interface AgentListProps { + agents: Map; +} + +function getStatusColor(status: AgentStatus): string { + switch (status) { + case "initializing": + return "#FFA500"; + case "ready": + return "#2563EB"; + case "processing": + return "#FACC15"; + case "completed": + return "#10B981"; + case "failed": + return "#EF4444"; + case "killed": + return "#6B7280"; + default: + return "#6B7280"; + } +} + +function getStatusLabel(status: AgentStatus): string { + switch (status) { + case "initializing": + return "Initializing"; + case "ready": + return "Ready"; + case "processing": + return "Processing"; + case "completed": + return "Completed"; + case "failed": + return "Failed"; + case "killed": + return "Killed"; + default: + return "Unknown"; + } +} + +export function AgentList({ agents }: AgentListProps) { + const { theme } = useUnistyles(); + const agentArray = Array.from(agents.values()); + + function handleAgentPress(agentId: string) { + router.push(`/agent/${agentId}`); + } + + return ( + + {agentArray.map((agent) => { + const statusColor = getStatusColor(agent.status); + const statusLabel = getStatusLabel(agent.status); + + return ( + handleAgentPress(agent.id)} + > + + + {agent.title || "New Agent"} + + + + + {agent.cwd} + + + + + + {statusLabel} + + + + + + ); + })} + + ); +} + +const styles = StyleSheet.create((theme) => ({ + container: { + flex: 1, + paddingHorizontal: theme.spacing[4], + paddingTop: theme.spacing[4], + }, + agentItem: { + paddingVertical: theme.spacing[4], + paddingHorizontal: theme.spacing[4], + borderRadius: theme.borderRadius.lg, + marginBottom: theme.spacing[2], + backgroundColor: theme.colors.muted, + }, + agentContent: { + flex: 1, + }, + agentTitle: { + fontSize: theme.fontSize.base, + fontWeight: theme.fontWeight.semibold, + color: theme.colors.foreground, + marginBottom: theme.spacing[1], + }, + directoryRow: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + agentDirectory: { + flex: 1, + fontSize: theme.fontSize.sm, + color: theme.colors.mutedForeground, + }, + statusBadge: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[1], + }, + statusDot: { + width: 6, + height: 6, + borderRadius: theme.borderRadius.full, + }, + statusText: { + fontSize: theme.fontSize.xs, + fontWeight: theme.fontWeight.semibold, + }, +})); diff --git a/packages/app/src/components/agent-sidebar.tsx b/packages/app/src/components/agent-sidebar.tsx index 70d331d06..f1c92f3a1 100644 --- a/packages/app/src/components/agent-sidebar.tsx +++ b/packages/app/src/components/agent-sidebar.tsx @@ -11,6 +11,7 @@ import Animated, { cancelAnimation, runOnJS, Easing, + type SharedValue, } from "react-native-reanimated"; import type { AgentStatus } from "@server/server/acp/types"; @@ -28,7 +29,7 @@ interface AgentSidebarProps { onClose: () => void; onSelectAgent: (agentId: string) => void; onNewAgent: () => void; - edgeSwipeTranslateX?: Animated.SharedValue | null; + edgeSwipeTranslateX?: SharedValue | null; } function getStatusColor(status: AgentStatus): string { @@ -50,6 +51,25 @@ function getStatusColor(status: AgentStatus): string { } } +function getStatusLabel(status: AgentStatus): string { + switch (status) { + case "initializing": + return "Initializing"; + case "ready": + return "Ready"; + case "processing": + return "Processing"; + case "completed": + return "Completed"; + case "failed": + return "Failed"; + case "killed": + return "Killed"; + default: + return "Unknown"; + } +} + export function AgentSidebar({ isOpen, agents, @@ -202,6 +222,7 @@ export function AgentSidebar({ {agents.map((agent) => { const isActive = agent.id === activeAgentId; const statusColor = getStatusColor(agent.status); + const statusLabel = getStatusLabel(agent.status); return ( handleAgentSelect(agent.id)} > - - {/* Status Indicator */} - + + + {agent.title || "New Agent"} + - {/* Agent Title */} - - - {agent.title || "New Agent"} - - - {/* Agent Directory */} + {agent.cwd} + + + + + {statusLabel} + + @@ -308,16 +331,6 @@ const styles = StyleSheet.create({ borderRadius: 8, marginBottom: 4, }, - agentInfo: { - flexDirection: "row", - alignItems: "center", - gap: 12, - }, - statusDot: { - width: 8, - height: 8, - borderRadius: 4, - }, agentContent: { flex: 1, }, @@ -326,9 +339,29 @@ const styles = StyleSheet.create({ fontWeight: "500", marginBottom: 2, }, + directoryRow: { + flexDirection: "row", + alignItems: "center", + gap: 8, + }, agentDirectory: { + flex: 1, fontSize: 12, }, + statusBadge: { + flexDirection: "row", + alignItems: "center", + gap: 4, + }, + statusDot: { + width: 6, + height: 6, + borderRadius: 3, + }, + statusText: { + fontSize: 11, + fontWeight: "500", + }, emptyState: { paddingVertical: 32, alignItems: "center", diff --git a/packages/app/src/components/empty-state.tsx b/packages/app/src/components/empty-state.tsx new file mode 100644 index 000000000..361ebc3c7 --- /dev/null +++ b/packages/app/src/components/empty-state.tsx @@ -0,0 +1,59 @@ +import { View, Text, Pressable } from "react-native"; +import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import { Plus } from "lucide-react-native"; + +interface EmptyStateProps { + onCreateAgent: () => void; +} + +export function EmptyState({ onCreateAgent }: EmptyStateProps) { + const { theme } = useUnistyles(); + + return ( + + Hammock + + What would you like to work on? + + + + New agent + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + container: { + flex: 1, + alignItems: "center", + justifyContent: "center", + paddingHorizontal: theme.spacing[6], + }, + title: { + fontSize: theme.fontSize["4xl"], + fontWeight: "700", + color: theme.colors.foreground, + marginBottom: theme.spacing[2], + }, + subtitle: { + fontSize: theme.fontSize.lg, + color: theme.colors.mutedForeground, + textAlign: "center", + marginBottom: theme.spacing[8], + }, + button: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + paddingVertical: theme.spacing[3], + paddingHorizontal: theme.spacing[6], + backgroundColor: theme.colors.primary, + borderRadius: theme.borderRadius.lg, + }, + buttonText: { + fontSize: theme.fontSize.base, + fontWeight: theme.fontWeight.semibold, + color: theme.colors.primaryForeground, + }, +})); diff --git a/packages/app/src/components/global-footer.tsx b/packages/app/src/components/global-footer.tsx new file mode 100644 index 000000000..c7168faf7 --- /dev/null +++ b/packages/app/src/components/global-footer.tsx @@ -0,0 +1,86 @@ +import { View, Pressable } from "react-native"; +import { usePathname } from "expo-router"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import { AudioLines } from "lucide-react-native"; +import { useRealtime } from "@/contexts/realtime-context"; +import { useSession } from "@/contexts/session-context"; +import { RealtimeControls } from "./realtime-controls"; +import { AgentInputArea } from "./agent-input-area"; + +interface GlobalFooterProps { + agentId?: string; +} + +export function GlobalFooter({ agentId }: GlobalFooterProps) { + const { theme } = useUnistyles(); + const insets = useSafeAreaInsets(); + const pathname = usePathname(); + const { isRealtimeMode, startRealtime } = useRealtime(); + const { ws } = useSession(); + + // Determine current screen type + const isAgentScreen = pathname?.startsWith("/agent/"); + const isOrchestratorScreen = pathname === "/orchestrator"; + const isHomeScreen = pathname === "/"; + + // If realtime is active, always show realtime controls + if (isRealtimeMode) { + return ( + + + + ); + } + + // If on agent screen, show full input area + if (isAgentScreen && agentId) { + return ( + + + + ); + } + + // For home and orchestrator screens, show centered realtime button + return ( + + + + + + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + container: { + backgroundColor: theme.colors.background, + borderTopWidth: theme.borderWidth[1], + borderTopColor: theme.colors.border, + }, + centeredButtonContainer: { + padding: theme.spacing[6], + alignItems: "center", + justifyContent: "center", + }, + centeredRealtimeButton: { + width: 56, + height: 56, + borderRadius: theme.borderRadius.full, + backgroundColor: theme.colors.palette.blue[600], + alignItems: "center", + justifyContent: "center", + }, + buttonDisabled: { + opacity: 0.5, + }, +})); diff --git a/packages/app/src/components/headers/back-header.tsx b/packages/app/src/components/headers/back-header.tsx new file mode 100644 index 000000000..38160ff5f --- /dev/null +++ b/packages/app/src/components/headers/back-header.tsx @@ -0,0 +1,76 @@ +import { View, Pressable, Text } from "react-native"; +import { router } from "expo-router"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import { ArrowLeft } from "lucide-react-native"; + +interface BackHeaderProps { + title?: string; +} + +export function BackHeader({ title }: BackHeaderProps) { + const { theme } = useUnistyles(); + const insets = useSafeAreaInsets(); + + return ( + + + + {/* Left side - Back button */} + + router.back()} + style={styles.backButton} + > + + + {title && ( + + {title} + + )} + + + {/* Right side - Empty for now */} + + + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + header: { + backgroundColor: theme.colors.background, + }, + headerRow: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + paddingHorizontal: theme.spacing[3], + paddingBottom: theme.spacing[2], + borderBottomWidth: theme.borderWidth[1], + borderBottomColor: theme.colors.border, + }, + headerLeft: { + flex: 1, + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[3], + }, + headerRight: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + backButton: { + padding: theme.spacing[3], + borderRadius: theme.borderRadius.lg, + }, + title: { + flex: 1, + fontSize: theme.fontSize.lg, + fontWeight: theme.fontWeight.semibold, + color: theme.colors.foreground, + }, +})); diff --git a/packages/app/src/components/headers/home-header.tsx b/packages/app/src/components/headers/home-header.tsx new file mode 100644 index 000000000..c9219efb9 --- /dev/null +++ b/packages/app/src/components/headers/home-header.tsx @@ -0,0 +1,79 @@ +import { View, Pressable } from "react-native"; +import { router } from "expo-router"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import { Settings, MessageSquare, Plus } from "lucide-react-native"; + +interface HomeHeaderProps { + onCreateAgent: () => void; +} + +export function HomeHeader({ onCreateAgent }: HomeHeaderProps) { + const { theme } = useUnistyles(); + const insets = useSafeAreaInsets(); + + return ( + + + + {/* Left side - Settings */} + + router.push("/settings")} + style={styles.iconButton} + > + + + + + {/* Right side - Activity and New Agent */} + + router.push("/orchestrator")} + style={styles.iconButton} + > + + + + + + + + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + header: { + backgroundColor: theme.colors.background, + }, + headerRow: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + paddingHorizontal: theme.spacing[3], + paddingBottom: theme.spacing[2], + borderBottomWidth: theme.borderWidth[1], + borderBottomColor: theme.colors.border, + }, + headerLeft: { + flex: 1, + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + headerRight: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + iconButton: { + padding: theme.spacing[3], + borderRadius: theme.borderRadius.lg, + backgroundColor: theme.colors.muted, + }, +})); diff --git a/packages/app/src/components/mode-selector-modal.tsx b/packages/app/src/components/mode-selector-modal.tsx new file mode 100644 index 000000000..c4b6085e3 --- /dev/null +++ b/packages/app/src/components/mode-selector-modal.tsx @@ -0,0 +1,110 @@ +import { View, Text, Modal, Pressable } from "react-native"; +import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import type { Agent } from "@/contexts/session-context"; + +interface ModeSelectorModalProps { + visible: boolean; + agent: Agent | null; + onModeChange: (modeId: string) => void; + onClose: () => void; +} + +export function ModeSelectorModal({ + visible, + agent, + onModeChange, + onClose, +}: ModeSelectorModalProps) { + const { theme } = useUnistyles(); + + return ( + + + + {agent?.availableModes?.map((mode) => { + const isActive = mode.id === agent.currentModeId; + return ( + { + onModeChange(mode.id); + onClose(); + }} + style={[ + styles.modeItem, + isActive && styles.modeItemActive, + ]} + > + + {mode.name} + + {mode.description && ( + + {mode.description} + + )} + + ); + })} + + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + modalOverlay: { + flex: 1, + backgroundColor: "rgba(0,0,0,0.5)", + justifyContent: "center", + alignItems: "center", + }, + modeSelectorContent: { + backgroundColor: theme.colors.card, + borderRadius: theme.borderRadius.lg, + padding: theme.spacing[4], + minWidth: 280, + maxWidth: 320, + }, + modeItem: { + padding: theme.spacing[4], + borderRadius: theme.borderRadius.md, + marginBottom: theme.spacing[2], + backgroundColor: theme.colors.muted, + }, + modeItemActive: { + backgroundColor: theme.colors.primary, + }, + modeName: { + color: theme.colors.foreground, + fontSize: theme.fontSize.base, + fontWeight: theme.fontWeight.semibold, + marginBottom: theme.spacing[1], + }, + modeNameActive: { + color: theme.colors.primaryForeground, + }, + modeDescription: { + color: theme.colors.mutedForeground, + fontSize: theme.fontSize.sm, + }, + modeDescriptionActive: { + color: theme.colors.primaryForeground, + opacity: 0.8, + }, +})); diff --git a/packages/app/src/components/orchestrator-messages-view.tsx b/packages/app/src/components/orchestrator-messages-view.tsx new file mode 100644 index 000000000..f8558c312 --- /dev/null +++ b/packages/app/src/components/orchestrator-messages-view.tsx @@ -0,0 +1,117 @@ +import { View, ScrollView } from "react-native"; +import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import { forwardRef } from "react"; +import { + UserMessage, + AssistantMessage, + ActivityLog, + ToolCall, +} from "@/components/message"; +import type { MessageEntry } from "@/contexts/session-context"; + +interface OrchestratorMessagesViewProps { + messages: MessageEntry[]; + currentAssistantMessage: string; + onArtifactClick: (artifactId: string) => void; +} + +export const OrchestratorMessagesView = forwardRef( + function OrchestratorMessagesView({ messages, currentAssistantMessage, onArtifactClick }, ref) { + const { theme } = useUnistyles(); + + return ( + + {messages.map((msg) => { + if (msg.type === "user") { + return ( + + ); + } + + if (msg.type === "assistant") { + return ( + + ); + } + + if (msg.type === "activity") { + return ( + + ); + } + + if (msg.type === "artifact") { + return ( + + ); + } + + if (msg.type === "tool_call") { + return ( + + ); + } + + return null; + })} + + {/* Streaming assistant message */} + {currentAssistantMessage && ( + + )} + + ); + } +); + +const styles = StyleSheet.create((theme) => ({ + scrollView: { + flex: 1, + }, + scrollContent: { + paddingHorizontal: theme.spacing[4], + paddingTop: theme.spacing[4], + paddingBottom: theme.spacing[6], + }, +})); diff --git a/packages/app/src/components/realtime-controls.tsx b/packages/app/src/components/realtime-controls.tsx new file mode 100644 index 000000000..2e6dd7a4e --- /dev/null +++ b/packages/app/src/components/realtime-controls.tsx @@ -0,0 +1,110 @@ +import { View, Text, Pressable } from "react-native"; +import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import { MicOff, Square } from "lucide-react-native"; +import { VolumeMeter } from "./volume-meter"; +import { useRealtime } from "@/contexts/realtime-context"; + +export function RealtimeControls() { + const { theme } = useUnistyles(); + const { + volume, + isMuted, + isDetecting, + isSpeaking, + segmentDuration, + stopRealtime, + toggleMute, + } = useRealtime(); + + return ( + + + + {/* Debug timer */} + {(isDetecting || isSpeaking) && ( + + {(segmentDuration / 1000).toFixed(1)}s + + )} + + + {/* Mute button */} + + + + {/* Stop button */} + + + + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + container: { + minHeight: 200, + padding: theme.spacing[4], + }, + volumeContainer: { + flex: 1, + justifyContent: "center", + alignItems: "center", + }, + buttons: { + flexDirection: "row", + alignItems: "center", + justifyContent: "center", + gap: theme.spacing[3], + paddingTop: theme.spacing[4], + }, + muteButton: { + width: 48, + height: 48, + borderRadius: theme.borderRadius.full, + alignItems: "center", + justifyContent: "center", + backgroundColor: theme.colors.muted, + borderWidth: theme.borderWidth[2], + borderColor: theme.colors.border, + }, + muteButtonActive: { + backgroundColor: theme.colors.palette.red[500], + borderColor: theme.colors.palette.red[600], + }, + stopButton: { + width: 48, + height: 48, + borderRadius: theme.borderRadius.full, + alignItems: "center", + justifyContent: "center", + backgroundColor: theme.colors.palette.red[600], + }, + debugTimer: { + marginTop: theme.spacing[2], + color: theme.colors.mutedForeground, + fontSize: theme.fontSize.sm, + fontFamily: "monospace", + }, +})); diff --git a/packages/app/src/contexts/realtime-context.tsx b/packages/app/src/contexts/realtime-context.tsx new file mode 100644 index 000000000..c530d6880 --- /dev/null +++ b/packages/app/src/contexts/realtime-context.tsx @@ -0,0 +1,144 @@ +import { createContext, useContext, useState, ReactNode, useCallback } from "react"; +import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio"; +import { useSession } from "./session-context"; +import { generateMessageId } from "@/types/stream"; +import type { WSInboundMessage } from "@server/server/messages"; + +interface RealtimeContextValue { + isRealtimeMode: boolean; + volume: number; + isMuted: boolean; + isDetecting: boolean; + isSpeaking: boolean; + segmentDuration: number; + startRealtime: () => Promise; + stopRealtime: () => Promise; + toggleMute: () => void; +} + +const RealtimeContext = createContext(null); + +export function useRealtime() { + const context = useContext(RealtimeContext); + if (!context) { + throw new Error("useRealtime must be used within RealtimeProvider"); + } + return context; +} + +interface RealtimeProviderProps { + children: ReactNode; +} + +export function RealtimeProvider({ children }: RealtimeProviderProps) { + const { ws, audioPlayer, isPlayingAudio, setMessages } = useSession(); + const [isRealtimeMode, setIsRealtimeMode] = useState(false); + + const realtimeAudio = useSpeechmaticsAudio({ + onSpeechStart: () => { + console.log("[Realtime] Speech detected"); + // Stop audio playback if playing + if (isPlayingAudio) { + audioPlayer.stop(); + } + }, + onSpeechEnd: () => { + console.log("[Realtime] Speech ended"); + }, + onAudioSegment: (base64Audio: string) => { + console.log("[Realtime] Sending audio segment, length:", base64Audio.length); + + // Send audio segment to server (realtime always goes to orchestrator) + try { + ws.send({ + type: "session", + message: { + type: "realtime_audio_chunk", + audio: base64Audio, + format: "audio/wav", + isLast: true, // Complete segment + }, + }); + } catch (error) { + console.error("[Realtime] Failed to send audio segment:", error); + } + }, + onError: (error) => { + console.error("[Realtime] Audio error:", error); + setMessages((prev) => [ + ...prev, + { + type: "activity", + id: generateMessageId(), + timestamp: Date.now(), + activityType: "error", + message: `Realtime audio error: ${error.message}`, + }, + ]); + }, + volumeThreshold: 0.3, + silenceDuration: 2000, + speechConfirmationDuration: 300, + detectionGracePeriod: 200, + }); + + const startRealtime = useCallback(async () => { + try { + await realtimeAudio.start(); + setIsRealtimeMode(true); + console.log("[Realtime] Mode enabled"); + + // Notify server + const modeMessage: WSInboundMessage = { + type: "session", + message: { + type: "set_realtime_mode", + enabled: true, + }, + }; + ws.send(modeMessage); + } catch (error: any) { + console.error("[Realtime] Failed to start:", error); + throw error; + } + }, [realtimeAudio, ws]); + + const stopRealtime = useCallback(async () => { + try { + await realtimeAudio.stop(); + setIsRealtimeMode(false); + console.log("[Realtime] Mode disabled"); + + // Notify server + const modeMessage: WSInboundMessage = { + type: "session", + message: { + type: "set_realtime_mode", + enabled: false, + }, + }; + ws.send(modeMessage); + } catch (error: any) { + console.error("[Realtime] Failed to stop:", error); + throw error; + } + }, [realtimeAudio, ws]); + + const value: RealtimeContextValue = { + isRealtimeMode, + volume: realtimeAudio.volume, + isMuted: realtimeAudio.isMuted, + isDetecting: realtimeAudio.isDetecting, + isSpeaking: realtimeAudio.isSpeaking, + segmentDuration: realtimeAudio.segmentDuration, + startRealtime, + stopRealtime, + toggleMute: realtimeAudio.toggleMute, + }; + + return ( + + {children} + + ); +} diff --git a/packages/app/src/contexts/session-context.tsx b/packages/app/src/contexts/session-context.tsx new file mode 100644 index 000000000..99b6cbde8 --- /dev/null +++ b/packages/app/src/contexts/session-context.tsx @@ -0,0 +1,391 @@ +import { createContext, useContext, useState, useRef, ReactNode, useCallback, useEffect } from "react"; +import { useWebSocket, type UseWebSocketReturn } from "@/hooks/use-websocket"; +import { useAudioPlayer } from "@/hooks/use-audio-player"; +import { reduceStreamUpdate, generateMessageId, type StreamItem } from "@/types/stream"; +import type { + ActivityLogPayload, + SessionInboundMessage, + WSInboundMessage, +} from "@server/server/messages"; +import type { AgentStatus, AgentUpdate, AgentNotification } from "@server/server/acp/types"; +import { parseSessionUpdate } from "@/types/agent-activity"; +import { ScrollView } from "react-native"; + +export type MessageEntry = + | { + type: "user"; + id: string; + timestamp: number; + message: string; + } + | { + type: "assistant"; + id: string; + timestamp: number; + message: string; + } + | { + type: "activity"; + id: string; + timestamp: number; + activityType: "system" | "info" | "success" | "error"; + message: string; + metadata?: Record; + } + | { + type: "artifact"; + id: string; + timestamp: number; + artifactId: string; + artifactType: string; + title: string; + } + | { + type: "tool_call"; + id: string; + timestamp: number; + toolName: string; + args: any; + result?: any; + error?: any; + status: "executing" | "completed" | "failed"; + }; + +export interface Agent { + id: string; + status: AgentStatus; + createdAt: Date; + type: "claude"; + sessionId?: string; + error?: string; + currentModeId?: string; + availableModes?: Array<{ + id: string; + name: string; + description?: string | null; + }>; + title?: string; + cwd: string; +} + +export interface Command { + id: string; + name: string; + workingDirectory: string; + currentCommand: string; + isDead: boolean; + exitCode: number | null; +} + +export interface PendingPermission { + agentId: string; + requestId: string; + sessionId: string; + toolCall: any; + options: Array<{ + kind: string; + name: string; + optionId: string; + }>; +} + +interface SessionContextValue { + // WebSocket + ws: UseWebSocketReturn; + + // Audio + audioPlayer: ReturnType; + isPlayingAudio: boolean; + setIsPlayingAudio: (playing: boolean) => void; + + // Messages and stream state + messages: MessageEntry[]; + setMessages: (messages: MessageEntry[] | ((prev: MessageEntry[]) => MessageEntry[])) => void; + currentAssistantMessage: string; + setCurrentAssistantMessage: (message: string) => void; + agentStreamState: Map; + setAgentStreamState: (state: Map | ((prev: Map) => Map)) => void; + + // Agents and commands + agents: Map; + setAgents: (agents: Map | ((prev: Map) => Map)) => void; + commands: Map; + setCommands: (commands: Map | ((prev: Map) => Map)) => void; + agentUpdates: Map; + setAgentUpdates: (updates: Map | ((prev: Map) => Map)) => void; + + // Permissions + pendingPermissions: Map; + setPendingPermissions: (perms: Map | ((prev: Map) => Map)) => void; + + // Helpers + sendAgentMessage: (agentId: string, message: string) => void; + createAgent: (options: { cwd: string; autoStart?: boolean }) => void; + setAgentMode: (agentId: string, modeId: string) => void; + respondToPermission: (requestId: string, agentId: string, sessionId: string, selectedOptionIds: string[]) => void; +} + +const SessionContext = createContext(null); + +export function useSession() { + const context = useContext(SessionContext); + if (!context) { + throw new Error("useSession must be used within SessionProvider"); + } + return context; +} + +interface SessionProviderProps { + children: ReactNode; + serverUrl: string; +} + +export function SessionProvider({ children, serverUrl }: SessionProviderProps) { + const ws = useWebSocket(serverUrl); + const audioPlayer = useAudioPlayer(); + + const [isPlayingAudio, setIsPlayingAudio] = useState(false); + const [messages, setMessages] = useState([]); + const [currentAssistantMessage, setCurrentAssistantMessage] = useState(""); + const [agentStreamState, setAgentStreamState] = useState>(new Map()); + + const [agents, setAgents] = useState>(new Map()); + const [commands, setCommands] = useState>(new Map()); + const [agentUpdates, setAgentUpdates] = useState>(new Map()); + const [pendingPermissions, setPendingPermissions] = useState>(new Map()); + + // WebSocket message handlers + useEffect(() => { + // Session state - initial agents/commands + const unsubSessionState = ws.on("session_state", (message) => { + if (message.type !== "session_state") return; + const { agents: agentsList, commands: commandsList } = message.payload; + + console.log("[Session] Session state:", agentsList.length, "agents,", commandsList.length, "commands"); + + setAgents(new Map(agentsList.map((a) => [a.id, { + ...a, + createdAt: new Date(a.createdAt), + } as Agent]))); + + setCommands(new Map(commandsList.map((c) => [c.id, c as Command]))); + }); + + // Agent created + const unsubAgentCreated = ws.on("agent_created", (message) => { + if (message.type !== "agent_created") return; + const { agentId, status, type, currentModeId, availableModes, title, cwd } = message.payload; + + console.log("[Session] Agent created:", agentId); + + const agent: Agent = { + id: agentId, + status: status as AgentStatus, + type, + createdAt: new Date(), + title, + cwd, + currentModeId, + availableModes, + }; + + setAgents((prev) => new Map(prev).set(agentId, agent)); + setAgentStreamState((prev) => new Map(prev).set(agentId, [])); + }); + + // Agent status update (mode changes, title changes, etc.) + const unsubAgentStatus = ws.on("agent_status", (message) => { + if (message.type !== "agent_status") return; + const { agentId, info } = message.payload; + + console.log("[Session] Agent status update:", agentId, "mode:", info.currentModeId); + + setAgents((prev) => { + const existingAgent = prev.get(agentId); + if (!existingAgent) return prev; + + const updatedAgent: Agent = { + ...existingAgent, + status: info.status as AgentStatus, + sessionId: info.sessionId, + error: info.error, + currentModeId: info.currentModeId, + availableModes: info.availableModes, + title: info.title, + cwd: info.cwd, + }; + + return new Map(prev).set(agentId, updatedAgent); + }); + }); + + // Agent update + const unsubAgentUpdate = ws.on("agent_update", (message) => { + if (message.type !== "agent_update") return; + const { agentId, notification } = message.payload; + + const update: AgentUpdate = { + agentId, + timestamp: new Date(), + notification, + }; + + setAgentUpdates((prev) => { + const agentHistory = prev.get(agentId) || []; + return new Map(prev).set(agentId, [...agentHistory, update]); + }); + + // Update stream state using reducer + setAgentStreamState((prev) => { + const currentStream = prev.get(agentId) || []; + const newStream = reduceStreamUpdate(currentStream, notification, new Date()); + return new Map(prev).set(agentId, newStream); + }); + }); + + // Permission request + const unsubPermissionRequest = ws.on("agent_permission_request", (message) => { + if (message.type !== "agent_permission_request") return; + const { agentId, requestId, sessionId, toolCall, options } = message.payload; + + console.log("[Session] Permission request:", requestId, "for agent:", agentId); + + setPendingPermissions((prev) => new Map(prev).set(requestId, { + agentId, + requestId, + sessionId, + toolCall, + options, + })); + }); + + return () => { + unsubSessionState(); + unsubAgentCreated(); + unsubAgentStatus(); + unsubAgentUpdate(); + unsubPermissionRequest(); + }; + }, [ws]); + + const sendAgentMessage = useCallback((agentId: string, message: string) => { + // Generate unique message ID for deduplication + const messageId = generateMessageId(); + + // Optimistically add user message to stream + setAgentStreamState((prev) => { + const currentStream = prev.get(agentId) || []; + + // Create AgentNotification structure that matches server format + const notification: AgentNotification = { + type: 'session', + notification: { + sessionId: '', + update: { + sessionUpdate: "user_message_chunk", + content: { type: 'text', text: message }, + messageId, + }, + }, + }; + + // Use reduceStreamUpdate to properly create the StreamItem + const newStream = reduceStreamUpdate(currentStream, notification, new Date()); + + const updated = new Map(prev); + updated.set(agentId, newStream); + return updated; + }); + + // Send to agent with messageId + const msg: WSInboundMessage = { + type: "session", + message: { + type: "send_agent_message", + agentId, + text: message, + messageId, + }, + }; + ws.send(msg); + }, [ws]); + + const createAgent = useCallback((options: { cwd: string; autoStart?: boolean }) => { + const msg: WSInboundMessage = { + type: "session", + message: { + type: "create_agent_request", + ...options, + }, + }; + ws.send(msg); + }, [ws]); + + const setAgentMode = useCallback((agentId: string, modeId: string) => { + const msg: WSInboundMessage = { + type: "session", + message: { + type: "set_agent_mode", + agentId, + modeId, + }, + }; + ws.send(msg); + }, [ws]); + + const respondToPermission = useCallback(( + requestId: string, + agentId: string, + sessionId: string, + selectedOptionIds: string[] + ) => { + const msg: WSInboundMessage = { + type: "session", + message: { + type: "agent_permission_response", + agentId, + requestId, + optionId: selectedOptionIds[0], + }, + }; + ws.send(msg); + + // Remove from pending + setPendingPermissions((prev) => { + const next = new Map(prev); + next.delete(requestId); + return next; + }); + }, [ws]); + + const value: SessionContextValue = { + ws, + audioPlayer, + isPlayingAudio, + setIsPlayingAudio, + messages, + setMessages, + currentAssistantMessage, + setCurrentAssistantMessage, + agentStreamState, + setAgentStreamState, + agents, + setAgents, + commands, + setCommands, + agentUpdates, + setAgentUpdates, + pendingPermissions, + setPendingPermissions, + sendAgentMessage, + createAgent, + setAgentMode, + respondToPermission, + }; + + return ( + + {children} + + ); +} diff --git a/packages/app/src/types/stream.ts b/packages/app/src/types/stream.ts index bd9151fae..4155767ba 100644 --- a/packages/app/src/types/stream.ts +++ b/packages/app/src/types/stream.ts @@ -1,4 +1,4 @@ -import type { SessionNotification } from '@agentclientprotocol/sdk'; +import type { AgentNotification } from '@server/server/acp/types'; /** * Simple hash function for deterministic ID generation @@ -147,10 +147,16 @@ type ParsedNotification = | null; /** - * Parse SessionNotification into typed ParsedNotification + * Parse AgentNotification into typed ParsedNotification + * Only processes session notifications, ignoring permission and status notifications */ -function parseNotification(notification: SessionNotification | any): ParsedNotification { - const update = (notification as any).update; +function parseNotification(notification: AgentNotification): ParsedNotification { + // Only process session notifications (discriminated union) + if (notification.type !== 'session') { + return null; + } + + const update = notification.notification.update; if (!update || !update.sessionUpdate) { return null; @@ -159,26 +165,38 @@ function parseNotification(notification: SessionNotification | any): ParsedNotif const kind = update.sessionUpdate; switch (kind) { - case 'user_message_chunk': + case 'user_message_chunk': { + const content = update.content; + const text = content && content.type === 'text' ? content.text : ''; + const messageId = 'messageId' in update ? update.messageId : undefined; return { kind: 'user_message_chunk', - text: update.content?.text || '', - messageId: update.messageId, + text, + messageId, }; + } - case 'agent_message_chunk': + case 'agent_message_chunk': { + const content = update.content; + const text = content && content.type === 'text' ? content.text : ''; + const messageId = 'messageId' in update ? update.messageId : undefined; return { kind: 'agent_message_chunk', - text: update.content?.text || '', - messageId: update.messageId, + text, + messageId, }; + } - case 'agent_thought_chunk': + case 'agent_thought_chunk': { + const content = update.content; + const text = content && content.type === 'text' ? content.text : ''; + const messageId = 'messageId' in update ? update.messageId : undefined; return { kind: 'agent_thought_chunk', - text: update.content?.text || '', - messageId: update.messageId, + text, + messageId, }; + } case 'tool_call': return { @@ -243,7 +261,7 @@ function parseNotification(notification: SessionNotification | any): ParsedNotif */ export function reduceStreamUpdate( state: StreamItem[], - notification: SessionNotification | any, + notification: AgentNotification, timestamp: Date ): StreamItem[] { const parsed = parseNotification(notification); @@ -407,7 +425,7 @@ export function reduceStreamUpdate( * Hydrate stream state from batch of notifications */ export function hydrateStreamState( - notifications: Array<{ timestamp: Date; notification: SessionNotification }> + notifications: Array<{ timestamp: Date; notification: AgentNotification }> ): StreamItem[] { return notifications.reduce( (state, { notification, timestamp }) => reduceStreamUpdate(state, notification, timestamp), diff --git a/packages/server/src/server/acp/types.ts b/packages/server/src/server/acp/types.ts index 5a910275a..37d981012 100644 --- a/packages/server/src/server/acp/types.ts +++ b/packages/server/src/server/acp/types.ts @@ -2,14 +2,17 @@ import type { SessionNotification, RequestPermissionRequest } from "@agentclient /** * Extended update types with messageId for proper deduplication + * messageId is optional since some sources may not provide it */ -type AgentMessageChunkWithId = Extract & { messageId: string }; -type AgentThoughtChunkWithId = Extract & { messageId: string }; +type UserMessageChunkWithId = Extract & { messageId?: string }; +type AgentMessageChunkWithId = Extract & { messageId?: string }; +type AgentThoughtChunkWithId = Extract & { messageId?: string }; export type EnrichedSessionUpdate = + | UserMessageChunkWithId | AgentMessageChunkWithId | AgentThoughtChunkWithId - | Exclude; + | Exclude; export interface EnrichedSessionNotification extends Omit { update: EnrichedSessionUpdate; diff --git a/packages/server/src/server/messages.ts b/packages/server/src/server/messages.ts index a3b521f1a..06ba89fa5 100644 --- a/packages/server/src/server/messages.ts +++ b/packages/server/src/server/messages.ts @@ -196,7 +196,8 @@ export const AgentUpdateMessageSchema = z.object({ payload: z.object({ agentId: z.string(), timestamp: z.date(), - notification: z.any(), // SessionNotification from ACP - complex type, using any for simplicity + // Runtime validation with z.any(), TypeScript enforces AgentNotification type + notification: z.any(), }), });