diff --git a/packages/app/src/app/_layout.tsx b/packages/app/src/app/_layout.tsx index 3e8435120..894f0cded 100644 --- a/packages/app/src/app/_layout.tsx +++ b/packages/app/src/app/_layout.tsx @@ -5,7 +5,7 @@ import { KeyboardProvider } from "react-native-keyboard-controller"; import { GestureHandlerRootView, Gesture, GestureDetector } from "react-native-gesture-handler"; import { BottomSheetModalProvider } from "@gorhom/bottom-sheet"; import { PortalProvider } from "@gorhom/portal"; -import { RealtimeProvider } from "@/contexts/realtime-context"; +import { VoiceProvider } from "@/contexts/voice-context"; import { useAppSettings } from "@/hooks/use-settings"; import { useFaviconStatus } from "@/hooks/use-favicon-status"; import { View, ActivityIndicator, Text } from "react-native"; @@ -226,10 +226,10 @@ function ProvidersWrapper({ children }: { children: ReactNode }) { } return ( - + {children} - + ); } diff --git a/packages/app/src/components/agent-input-area.tsx b/packages/app/src/components/agent-input-area.tsx index aa5533462..9e4fbd029 100644 --- a/packages/app/src/components/agent-input-area.tsx +++ b/packages/app/src/components/agent-input-area.tsx @@ -1,26 +1,20 @@ import { View, Pressable, - Platform, Text, ActivityIndicator, } from "react-native"; import { useState, useEffect, useRef, useCallback, useMemo } from "react"; import { StyleSheet, useUnistyles } from "react-native-unistyles"; -import { ArrowUp, AudioLines, Square, Pencil } from "lucide-react-native"; -import Animated, { - useAnimatedStyle, - FadeIn, - FadeOut, -} from "react-native-reanimated"; +import { ArrowUp, AudioLines, MicOff, Square, Pencil } from "lucide-react-native"; +import Animated, { useAnimatedStyle } from "react-native-reanimated"; import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller"; import { useSafeAreaInsets } from "react-native-safe-area-context"; -import { useRealtime } from "@/contexts/realtime-context"; +import { useVoice } from "@/contexts/voice-context"; import { useIsFocused } from "@react-navigation/native"; import { FOOTER_HEIGHT, MAX_CONTENT_WIDTH } from "@/constants/layout"; import { generateMessageId } from "@/types/stream"; import { AgentStatusBar } from "./agent-status-bar"; -import { RealtimeControls } from "./realtime-controls"; import { useImageAttachmentPicker } from "@/hooks/use-image-attachment-picker"; import { useSessionStore } from "@/stores/session-store"; import { useDraftStore } from "@/stores/draft-store"; @@ -57,15 +51,6 @@ interface AgentInputAreaProps { } const EMPTY_ARRAY: readonly QueuedMessage[] = []; -// Android currently crashes inside ViewGroup.dispatchDraw when running Reanimated -// entering/exiting animations (see react-native-reanimated#8422), so guard them. -const SHOULD_DISABLE_ENTRY_EXIT_ANIMATIONS = Platform.OS === "android"; -const REALTIME_FADE_IN = SHOULD_DISABLE_ENTRY_EXIT_ANIMATIONS - ? undefined - : FadeIn.duration(250); -const REALTIME_FADE_OUT = SHOULD_DISABLE_ENTRY_EXIT_ANIMATIONS - ? undefined - : FadeOut.duration(250); export function AgentInputArea({ agentId, @@ -106,7 +91,7 @@ export function AgentInputArea({ const setQueuedMessages = useSessionStore((state) => state.setQueuedMessages); - const { startRealtime, stopRealtime, isRealtimeMode } = useRealtime(); + const { isVoiceMode, isMuted: isVoiceMuted, toggleMute: toggleVoiceMute } = useVoice(); const [internalInput, setInternalInput] = useState(""); const userInput = value ?? internalInput; @@ -345,21 +330,6 @@ export function AgentInputArea({ }; }); - async function handleRealtimePress() { - try { - if (isRealtimeMode) { - await stopRealtime(); - } else { - if (!isConnected || !serverId) { - return; - } - await startRealtime(serverId); - } - } catch (error) { - console.error("[AgentInput] Failed to toggle realtime mode:", error); - } - } - function handleCancelAgent() { if (!agent || agent.status !== "running" || isCancellingAgent) { return; @@ -474,29 +444,7 @@ export function AgentInputArea({ )} - ) : ( - - {isRealtimeMode ? ( - - ) : ( - - )} - - ); + ) : null; const leftContent = ; @@ -508,17 +456,6 @@ export function AgentInputArea({ keyboardAnimatedStyle, ]} > - {/* Realtime controls - only when active */} - {isRealtimeMode && ( - - - - )} - {/* Input area */} @@ -564,6 +501,35 @@ export function AgentInputArea({ /> )} + {/* Voice quick mute indicator */} + {isVoiceMode && ( + + + {isVoiceMuted ? ( + + ) : ( + + )} + + {isVoiceMuted ? "Voice muted" : "Voice on"} + + + + )} + {/* MessageInput handles everything: text, dictation, attachments, all buttons */} ({ height: theme.borderWidth[1], backgroundColor: theme.colors.border, }, - realtimeControlsContainer: { - height: FOOTER_HEIGHT, - }, inputAreaContainer: { position: "relative", minHeight: FOOTER_HEIGHT, @@ -620,17 +583,6 @@ const styles = StyleSheet.create(((theme: Theme) => ({ maxWidth: MAX_CONTENT_WIDTH, gap: theme.spacing[3], }, - realtimeButton: { - width: 34, - height: 34, - borderRadius: theme.borderRadius.full, - backgroundColor: theme.colors.accentForeground, - alignItems: "center", - justifyContent: "center", - }, - realtimeButtonActive: { - backgroundColor: theme.colors.palette.blue[600], - }, cancelButton: { width: 34, height: 34, @@ -642,6 +594,33 @@ const styles = StyleSheet.create(((theme: Theme) => ({ buttonDisabled: { opacity: 0.5, }, + voiceIndicatorRow: { + flexDirection: "row", + justifyContent: "flex-end", + }, + voiceIndicatorPill: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + paddingHorizontal: theme.spacing[3], + height: 32, + borderRadius: theme.borderRadius.full, + backgroundColor: theme.colors.surface2, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.border, + }, + voiceIndicatorPillMuted: { + backgroundColor: theme.colors.palette.red[600], + borderColor: theme.colors.palette.red[800], + }, + voiceIndicatorText: { + color: theme.colors.foreground, + fontSize: theme.fontSize.xs, + fontWeight: theme.fontWeight.medium, + }, + voiceIndicatorTextMuted: { + color: theme.colors.surface0, + }, queueContainer: { flexDirection: "column", gap: theme.spacing[2], diff --git a/packages/app/src/components/home-footer.tsx b/packages/app/src/components/home-footer.tsx deleted file mode 100644 index 19e46c53d..000000000 --- a/packages/app/src/components/home-footer.tsx +++ /dev/null @@ -1,297 +0,0 @@ -import { useCallback, useMemo, useState } from "react"; -import { View, Pressable, Text, Platform, Modal, Alert } from "react-native"; -import { useRouter } from "expo-router"; -import { useSafeAreaInsets } from "react-native-safe-area-context"; -import { StyleSheet, useUnistyles } from "react-native-unistyles"; -import { AudioLines, Users, Plus } from "lucide-react-native"; -import { useRealtime } from "@/contexts/realtime-context"; -import { useDaemonConnections } from "@/contexts/daemon-connections-context"; -import { FOOTER_HEIGHT } from "@/constants/layout"; -import { RealtimeControls } from "./realtime-controls"; -import Animated, { - FadeIn, - FadeOut, - useAnimatedStyle, -} from "react-native-reanimated"; -import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller"; - -export function HomeFooter() { - const { theme } = useUnistyles(); - const insets = useSafeAreaInsets(); - const router = useRouter(); - const { isRealtimeMode, startRealtime } = useRealtime(); - const { connectionStates } = useDaemonConnections(); - const [showRealtimeHostPicker, setShowRealtimeHostPicker] = useState(false); - // Guard Reanimated entry/exit transitions on Android to avoid ViewGroup.dispatchDraw crashes - // tracked in react-native-reanimated#8422. - const shouldDisableEntryExitAnimations = Platform.OS === "android"; - const realtimeFadeIn = shouldDisableEntryExitAnimations ? undefined : FadeIn.duration(250); - const realtimeFadeOut = shouldDisableEntryExitAnimations ? undefined : FadeOut.duration(250); - - const { height: keyboardHeight } = useReanimatedKeyboardAnimation(); - const bottomInset = insets.bottom; - - const keyboardAnimatedStyle = useAnimatedStyle( - () => { - "worklet"; - const absoluteHeight = Math.abs(keyboardHeight.value); - const shift = Math.max(0, absoluteHeight - bottomInset); - return { - transform: [{ translateY: -shift }], - }; - }, - [bottomInset], - ); - - const realtimeEligibleHosts = useMemo(() => { - return Array.from(connectionStates.values()).filter((entry) => entry.status === "online"); - }, [connectionStates]); - const hasAnyConfiguredHosts = connectionStates.size > 0; - - const handleStartRealtime = useCallback(() => { - if (realtimeEligibleHosts.length === 0) { - if (!hasAnyConfiguredHosts) { - Alert.alert( - "No hosts available", - "Add a host in Settings before starting realtime mode." - ); - return; - } - Alert.alert( - "Hosts reconnecting", - "Every host is offline right now. Paseo reconnects automatically—try realtime again once one comes online." - ); - return; - } - if (realtimeEligibleHosts.length === 1) { - void startRealtime(realtimeEligibleHosts[0].daemon.id).catch((error) => { - console.error("[HomeFooter] Failed to start realtime", error); - Alert.alert("Realtime failed", "Unable to start realtime mode for this host."); - }); - return; - } - setShowRealtimeHostPicker(true); - }, [hasAnyConfiguredHosts, realtimeEligibleHosts, startRealtime]); - - const handleSelectRealtimeHost = useCallback( - (daemonId: string) => { - setShowRealtimeHostPicker(false); - void startRealtime(daemonId).catch((error) => { - console.error("[HomeFooter] Failed to start realtime", error); - Alert.alert("Realtime failed", "Unable to start realtime mode for this host."); - }); - }, - [startRealtime] - ); - - const handleDismissHostPicker = useCallback(() => { - setShowRealtimeHostPicker(false); - }, []); - - // For home and orchestrator screens, show action buttons with realtime stacked on top - const nonAgentFooterHeight = isRealtimeMode - ? FOOTER_HEIGHT * 2 + insets.bottom - : FOOTER_HEIGHT + insets.bottom; - - const iconSize = 24; - const iconStyle = { width: iconSize, height: iconSize }; - - return ( - <> - - - {/* Realtime controls - only visible when active */} - {isRealtimeMode && ( - - - - )} - - {/* Action menu */} - - router.push("/agents")} - style={({ pressed }) => [ - styles.footerButton, - pressed && styles.buttonPressed, - ]} - > - - - - Agents - - - { - console.log("[HomeFooter] New Agent button pressed"); - router.push("/agent" as any); - }} - style={({ pressed }) => [ - styles.footerButton, - pressed && styles.buttonPressed, - ]} - > - - - - New agent - - - [ - styles.footerButton, - realtimeEligibleHosts.length === 0 && styles.buttonDisabled, - pressed && realtimeEligibleHosts.length === 0 && styles.buttonPressed, - ]} - > - - - - Realtime - - - - - - - - - - Choose a host - {realtimeEligibleHosts.map((entry) => ( - handleSelectRealtimeHost(entry.daemon.id)} - > - {entry.daemon.label} - - ))} - - Cancel - - - - - - ); -} - -const styles = StyleSheet.create((theme) => ({ - container: { - backgroundColor: theme.colors.surface0, - borderTopWidth: theme.borderWidth[1], - borderTopColor: theme.colors.border, - }, - nonAgentContent: { - flexDirection: "column", - }, - realtimeSection: { - height: FOOTER_HEIGHT, - borderBottomWidth: theme.borderWidth[1], - borderBottomColor: theme.colors.border, - }, - actionButtonContainer: { - flexDirection: "row", - padding: theme.spacing[4], - gap: theme.spacing[3], - height: FOOTER_HEIGHT, - }, - footerButton: { - flex: 1, - flexDirection: "column", - alignItems: "center", - justifyContent: "center", - paddingVertical: theme.spacing[3], - gap: theme.spacing[1], - }, - footerIconWrapper: { - width: 28, - height: 28, - alignItems: "center", - justifyContent: "center", - }, - footerButtonText: { - color: theme.colors.foreground, - fontSize: theme.fontSize.xs, - fontWeight: theme.fontWeight.normal, - }, - buttonDisabled: { - opacity: 0.5, - }, - buttonPressed: { - opacity: 0.5, - }, - hostPickerOverlay: { - flex: 1, - backgroundColor: "rgba(0,0,0,0.5)", - justifyContent: "flex-end", - }, - hostPickerBackdrop: { - flex: 1, - }, - hostPickerContainer: { - backgroundColor: theme.colors.surface2, - padding: theme.spacing[4], - borderTopLeftRadius: theme.borderRadius.xl, - borderTopRightRadius: theme.borderRadius.xl, - gap: theme.spacing[3], - }, - hostPickerTitle: { - fontSize: theme.fontSize.lg, - fontWeight: theme.fontWeight.semibold, - color: theme.colors.foreground, - textAlign: "center", - }, - hostPickerButton: { - paddingVertical: theme.spacing[3], - borderRadius: theme.borderRadius.lg, - backgroundColor: theme.colors.surface2, - }, - hostPickerButtonText: { - color: theme.colors.foreground, - fontSize: theme.fontSize.sm, - textAlign: "center", - }, - hostPickerCancel: { - paddingVertical: theme.spacing[3], - }, - hostPickerCancelText: { - color: theme.colors.foregroundMuted, - fontSize: theme.fontSize.sm, - textAlign: "center", - }, -})); diff --git a/packages/app/src/components/message-input.tsx b/packages/app/src/components/message-input.tsx index 23a431cdd..c68be2662 100644 --- a/packages/app/src/components/message-input.tsx +++ b/packages/app/src/components/message-input.tsx @@ -28,6 +28,7 @@ import { useDictation } from "@/hooks/use-dictation"; import { DictationOverlay } from "./dictation-controls"; import type { DaemonClientV2 } from "@server/client/daemon-client-v2"; import { usePanelStore } from "@/stores/panel-store"; +import { useVoiceOptional } from "@/contexts/voice-context"; export interface ImageAttachment { uri: string; @@ -118,6 +119,7 @@ export const MessageInput = forwardRef( ref ) { const { theme } = useUnistyles(); + const voice = useVoiceOptional(); const toggleAgentList = usePanelStore((state) => state.toggleAgentList); const [inputHeight, setInputHeight] = useState(MIN_INPUT_HEIGHT); const textInputRef = useRef< @@ -294,9 +296,12 @@ export const MessageInput = forwardRef( if (isDictating) { await cancelDictation(); } else { + if (voice?.isVoiceMode) { + await voice.stopVoice(); + } await startDictation(); } - }, [isDictating, cancelDictation, startDictation]); + }, [isDictating, cancelDictation, startDictation, voice]); const handleCancelRecording = useCallback(async () => { await cancelDictation(); diff --git a/packages/app/src/components/realtime-controls.tsx b/packages/app/src/components/realtime-controls.tsx deleted file mode 100644 index 6009f6c5b..000000000 --- a/packages/app/src/components/realtime-controls.tsx +++ /dev/null @@ -1,110 +0,0 @@ -import { View, Text, Pressable } from "react-native"; -import { StyleSheet, useUnistyles } from "react-native-unistyles"; -import { MicOff, Square } from "lucide-react-native"; -import { VolumeMeter } from "./volume-meter"; -import { useRealtime } from "@/contexts/realtime-context"; -import { FOOTER_HEIGHT } from "@/constants/layout"; - -const CONTROL_BUTTON_SIZE = 48; -const VERTICAL_PADDING = (FOOTER_HEIGHT - CONTROL_BUTTON_SIZE) / 2; - -export function RealtimeControls() { - const { theme } = useUnistyles(); - const { - volume, - isMuted, - isDetecting, - isSpeaking, - segmentDuration, - stopRealtime, - toggleMute, - } = useRealtime(); - - function handleStop() { - stopRealtime(); - } - - return ( - - - - - - {/* Mute button */} - - - - {/* Stop button */} - - - - - - ); -} - -const styles = StyleSheet.create((theme) => ({ - container: { - flexDirection: "row", - alignItems: "center", - justifyContent: "space-between", - paddingHorizontal: theme.spacing[4], - paddingVertical: VERTICAL_PADDING, - height: FOOTER_HEIGHT, - }, - volumeContainer: { - flex: 1, - justifyContent: "center", - alignItems: "flex-start", - }, - buttons: { - flexDirection: "row", - alignItems: "center", - justifyContent: "center", - gap: theme.spacing[3], - }, - muteButton: { - width: 48, - height: 48, - borderRadius: theme.borderRadius.full, - alignItems: "center", - justifyContent: "center", - backgroundColor: theme.colors.surface2, - borderWidth: theme.borderWidth[2], - borderColor: theme.colors.border, - }, - muteButtonActive: { - backgroundColor: theme.colors.palette.red[500], - borderColor: theme.colors.palette.red[600], - }, - stopButton: { - width: 48, - height: 48, - borderRadius: theme.borderRadius.full, - alignItems: "center", - justifyContent: "center", - backgroundColor: theme.colors.palette.red[600], - }, -})); diff --git a/packages/app/src/components/sliding-sidebar.tsx b/packages/app/src/components/sliding-sidebar.tsx index 3fd918c4c..7d4a75682 100644 --- a/packages/app/src/components/sliding-sidebar.tsx +++ b/packages/app/src/components/sliding-sidebar.tsx @@ -1,5 +1,5 @@ import { useCallback, useMemo, useState, useEffect } from "react"; -import { View, Pressable, Text, Platform } from "react-native"; +import { View, Pressable, Text, Platform, Modal, Alert } from "react-native"; import { useSafeAreaInsets } from "react-native-safe-area-context"; import Animated, { useAnimatedStyle, @@ -9,13 +9,16 @@ import Animated, { } from "react-native-reanimated"; import { Gesture, GestureDetector } from "react-native-gesture-handler"; import { StyleSheet, UnistylesRuntime, useUnistyles } from "react-native-unistyles"; -import { Plus, Settings, Users } from "lucide-react-native"; +import { Plus, Settings, Users, AudioLines } from "lucide-react-native"; import { router } from "expo-router"; import { usePanelStore } from "@/stores/panel-store"; import { GroupedAgentList } from "./grouped-agent-list"; import { useAggregatedAgents } from "@/hooks/use-aggregated-agents"; import { useSidebarAnimation } from "@/contexts/sidebar-animation-context"; import { useTauriDragHandlers, useTrafficLightPadding } from "@/utils/tauri-window"; +import { useVoice } from "@/contexts/voice-context"; +import { useDaemonConnections } from "@/contexts/daemon-connections-context"; +import { VoicePanel } from "./voice-panel"; const DESKTOP_SIDEBAR_WIDTH = 320; const SIDEBAR_AGENT_LIMIT = 15; @@ -48,6 +51,9 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) { } = useSidebarAnimation(); const trafficLightPadding = useTrafficLightPadding(); const dragHandlers = useTauriDragHandlers(); + const { connectionStates } = useDaemonConnections(); + const { isVoiceMode, startVoice, stopVoice } = useVoice(); + const [showVoiceHostPicker, setShowVoiceHostPicker] = useState(false); // Track user-initiated refresh to avoid showing spinner on background revalidation const [isManualRefresh, setIsManualRefresh] = useState(false); @@ -124,6 +130,62 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) { router.push("/agents"); }, [backdropOpacity, closeToAgent, isMobile, translateX, windowWidth]); + const voiceEligibleHosts = useMemo(() => { + return Array.from(connectionStates.values()).filter((entry) => entry.status === "online"); + }, [connectionStates]); + const hasAnyConfiguredHosts = connectionStates.size > 0; + + const handleToggleVoice = useCallback(() => { + if (isVoiceMode) { + void stopVoice().catch((error) => { + console.error("[SlidingSidebar] Failed to stop voice", error); + Alert.alert("Voice failed", "Unable to stop voice mode."); + }); + return; + } + + if (voiceEligibleHosts.length === 0) { + if (!hasAnyConfiguredHosts) { + Alert.alert( + "No hosts available", + "Add a host in Settings before starting Voice mode.", + [{ text: "Open Settings", onPress: () => router.push("/settings") }, { text: "OK" }] + ); + return; + } + Alert.alert( + "Hosts reconnecting", + "Every host is offline right now. Paseo reconnects automatically—try Voice mode again once one comes online." + ); + return; + } + + if (voiceEligibleHosts.length === 1) { + void startVoice(voiceEligibleHosts[0].daemon.id).catch((error) => { + console.error("[SlidingSidebar] Failed to start voice", error); + Alert.alert("Voice failed", "Unable to start Voice mode for this host."); + }); + return; + } + + setShowVoiceHostPicker(true); + }, [hasAnyConfiguredHosts, isVoiceMode, startVoice, stopVoice, voiceEligibleHosts]); + + const handleSelectVoiceHost = useCallback( + (daemonId: string) => { + setShowVoiceHostPicker(false); + void startVoice(daemonId).catch((error) => { + console.error("[SlidingSidebar] Failed to start voice", error); + Alert.alert("Voice failed", "Unable to start Voice mode for this host."); + }); + }, + [startVoice] + ); + + const handleDismissVoiceHostPicker = useCallback(() => { + setShowVoiceHostPicker(false); + }, []); + // Close gesture (swipe left to close when sidebar is open) // Only activates on leftward swipe, fails on rightward or vertical movement // This mirrors the explorer-sidebar pattern for the right sidebar @@ -224,6 +286,8 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) { parentGestureRef={closeGestureRef} /> + {isVoiceMode ? : null} + {/* Footer */} )} + + + {({ hovered }) => ( + + )} + )} + + + + + + + Choose a host + {voiceEligibleHosts.map((entry) => ( + handleSelectVoiceHost(entry.daemon.id)} + > + {entry.daemon.label} + + ))} + + Cancel + + + + ); } @@ -289,6 +399,8 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) { selectedAgentId={selectedAgentId} /> + {isVoiceMode ? : null} + {/* Footer */} )} - - {({ hovered }) => ( - - )} - + + + {({ hovered }) => ( + + )} + + + {({ hovered }) => ( + + )} + + ); @@ -376,6 +508,11 @@ const styles = StyleSheet.create((theme) => ({ borderTopWidth: 1, borderTopColor: theme.colors.border, }, + footerIconRow: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[3], + }, footerButton: { flexDirection: "row", alignItems: "center", @@ -395,4 +532,56 @@ const styles = StyleSheet.create((theme) => ({ footerButtonTextHovered: { color: theme.colors.foreground, }, + hostPickerOverlay: { + flex: 1, + justifyContent: "center", + alignItems: "center", + backgroundColor: "rgba(0, 0, 0, 0.5)", + padding: theme.spacing[4], + }, + hostPickerBackdrop: { + ...StyleSheet.absoluteFillObject, + }, + hostPickerContainer: { + width: "100%", + maxWidth: 360, + backgroundColor: theme.colors.surface0, + borderRadius: theme.borderRadius["2xl"], + padding: theme.spacing[4], + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.border, + gap: theme.spacing[2], + }, + hostPickerTitle: { + fontSize: theme.fontSize.base, + fontWeight: theme.fontWeight.semibold, + color: theme.colors.foreground, + marginBottom: theme.spacing[2], + }, + hostPickerButton: { + paddingVertical: theme.spacing[3], + paddingHorizontal: theme.spacing[3], + borderRadius: theme.borderRadius.lg, + backgroundColor: theme.colors.surface2, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.border, + }, + hostPickerButtonText: { + color: theme.colors.foreground, + fontSize: theme.fontSize.sm, + }, + hostPickerCancel: { + marginTop: theme.spacing[2], + paddingVertical: theme.spacing[3], + paddingHorizontal: theme.spacing[3], + borderRadius: theme.borderRadius.lg, + backgroundColor: theme.colors.surface0, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.border, + alignItems: "center", + }, + hostPickerCancelText: { + color: theme.colors.foregroundMuted, + fontSize: theme.fontSize.sm, + }, })); diff --git a/packages/app/src/components/voice-panel.tsx b/packages/app/src/components/voice-panel.tsx new file mode 100644 index 000000000..0043eaad3 --- /dev/null +++ b/packages/app/src/components/voice-panel.tsx @@ -0,0 +1,139 @@ +import { View, Text, Pressable } from "react-native"; +import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import { MicOff, Square, AudioLines } from "lucide-react-native"; +import { VolumeMeter } from "./volume-meter"; +import { useVoice } from "@/contexts/voice-context"; +import { useDaemonConnections } from "@/contexts/daemon-connections-context"; + +export function VoicePanel() { + const { theme } = useUnistyles(); + const { connectionStates } = useDaemonConnections(); + const { + volume, + isMuted, + isDetecting, + isSpeaking, + stopVoice, + toggleMute, + activeServerId, + } = useVoice(); + + const activeHost = activeServerId ? connectionStates.get(activeServerId) ?? null : null; + const hostLabel = activeHost?.daemon.label ?? null; + const hostStatus = activeHost?.status ?? null; + + return ( + + + + + Voice + + + {hostLabel ? `Host: ${hostLabel}` : "Host: unknown"} + {hostStatus ? ` (${hostStatus})` : ""} + + + + + + + + + + + + + void stopVoice()} + accessibilityRole="button" + accessibilityLabel="Stop voice mode" + style={[styles.iconButton, styles.iconButtonStop]} + > + + + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + container: { + marginHorizontal: theme.spacing[4], + marginBottom: theme.spacing[3], + borderRadius: theme.borderRadius["2xl"], + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.border, + backgroundColor: theme.colors.surface2, + paddingVertical: theme.spacing[3], + paddingHorizontal: theme.spacing[3], + gap: theme.spacing[3], + }, + headerRow: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + gap: theme.spacing[3], + }, + titleRow: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + flexShrink: 0, + }, + titleText: { + color: theme.colors.foreground, + fontSize: theme.fontSize.sm, + fontWeight: theme.fontWeight.semibold, + }, + hostText: { + flex: 1, + textAlign: "right", + color: theme.colors.foregroundMuted, + fontSize: theme.fontSize.xs, + }, + meterRow: { + justifyContent: "center", + }, + actionsRow: { + flexDirection: "row", + alignItems: "center", + justifyContent: "flex-end", + gap: theme.spacing[2], + }, + iconButton: { + width: 40, + height: 40, + borderRadius: theme.borderRadius.full, + alignItems: "center", + justifyContent: "center", + backgroundColor: theme.colors.surface0, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.border, + }, + iconButtonMuted: { + backgroundColor: theme.colors.palette.red[500], + borderColor: theme.colors.palette.red[600], + }, + iconButtonStop: { + backgroundColor: theme.colors.palette.red[600], + borderColor: theme.colors.palette.red[800], + }, +})); diff --git a/packages/app/src/contexts/realtime-context.tsx b/packages/app/src/contexts/voice-context.tsx similarity index 74% rename from packages/app/src/contexts/realtime-context.tsx rename to packages/app/src/contexts/voice-context.tsx index b4d315988..6afce3a7e 100644 --- a/packages/app/src/contexts/realtime-context.tsx +++ b/packages/app/src/contexts/voice-context.tsx @@ -7,34 +7,38 @@ import { randomUUID } from "expo-crypto"; const VOICE_CONVERSATION_ID_STORAGE_KEY = "@paseo:voice-conversation-id"; -interface RealtimeContextValue { - isRealtimeMode: boolean; +interface VoiceContextValue { + isVoiceMode: boolean; volume: number; isMuted: boolean; isDetecting: boolean; isSpeaking: boolean; segmentDuration: number; - startRealtime: (serverId: string) => Promise; - stopRealtime: () => Promise; + startVoice: (serverId: string) => Promise; + stopVoice: () => Promise; toggleMute: () => void; activeServerId: string | null; } -const RealtimeContext = createContext(null); +const VoiceContext = createContext(null); -export function useRealtime() { - const context = useContext(RealtimeContext); +export function useVoice() { + const context = useContext(VoiceContext); if (!context) { - throw new Error("useRealtime must be used within RealtimeProvider"); + throw new Error("useVoice must be used within VoiceProvider"); } return context; } -interface RealtimeProviderProps { +export function useVoiceOptional(): VoiceContextValue | null { + return useContext(VoiceContext); +} + +interface VoiceProviderProps { children: ReactNode; } -export function RealtimeProvider({ children }: RealtimeProviderProps) { +export function VoiceProvider({ children }: VoiceProviderProps) { const getSession = useSessionStore((state) => state.getSession); const [activeServerId, setActiveServerId] = useState(null); const activeSession = useSessionStore( @@ -49,12 +53,12 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) { ) ); const realtimeSessionRef = useRef(null); - const [isRealtimeMode, setIsRealtimeMode] = useState(false); + const [isVoiceMode, setIsVoiceMode] = useState(false); const bargeInPlaybackStopRef = useRef(null); const realtimeAudio = useSpeechmaticsAudio({ onSpeechStart: () => { - console.log("[Realtime] Speech detected"); + console.log("[Voice] Speech detected"); // Stop audio playback if playing const session = realtimeSessionRef.current; const sessionAudioPlayer = session?.audioPlayer ?? null; @@ -72,20 +76,20 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) { try { if (sessionClient) { void sessionClient.abortRequest().catch((error) => { - console.error("[Realtime] Failed to send abort_request:", error); + console.error("[Voice] Failed to send abort_request:", error); }); } - console.log("[Realtime] Sent abort_request before streaming audio"); + console.log("[Voice] Sent abort_request before streaming audio"); } catch (error) { - console.error("[Realtime] Failed to send abort_request:", error); + console.error("[Voice] Failed to send abort_request:", error); } }, onSpeechEnd: () => { - console.log("[Realtime] Speech ended"); + console.log("[Voice] Speech ended"); }, onAudioSegment: ({ audioData, isLast }) => { console.log( - "[Realtime] Sending audio segment, length:", + "[Voice] Sending audio segment, length:", audioData.length, "isLast:", isLast @@ -96,25 +100,25 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) { try { if (session?.client) { void session.client - .sendRealtimeAudioChunk( + .sendVoiceAudioChunk( audioData, "audio/pcm;rate=16000;bits=16", isLast ) .catch((error) => { - console.error("[Realtime] Failed to send audio segment:", error); + console.error("[Voice] Failed to send audio segment:", error); }); } } catch (error) { - console.error("[Realtime] Failed to send audio segment:", error); + console.error("[Voice] Failed to send audio segment:", error); } }, onError: (error) => { - console.error("[Realtime] Audio error:", error); + console.error("[Voice] Audio error:", error); const session = realtimeSessionRef.current; if (session?.client) { // Send error through websocket instead of directly manipulating messages - console.error("[Realtime] Cannot handle error - setMessages not available from SessionState"); + console.error("[Voice] Cannot handle error - setMessages not available from SessionState"); } }, volumeThreshold: 0.3, @@ -149,7 +153,7 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) { } }, [isPlayingAudio]); - const startRealtime = useCallback( + const startVoice = useCallback( async (serverId: string) => { const session = getSession(serverId) ?? null; if (!session) { @@ -160,8 +164,8 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) { realtimeSessionRef.current = session; setActiveServerId(serverId); await realtimeAudio.start(); - setIsRealtimeMode(true); - console.log("[Realtime] Mode enabled"); + setIsVoiceMode(true); + console.log("[Voice] Mode enabled"); if (session?.client) { let voiceConversationId = @@ -175,10 +179,10 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) { } await session.client.setVoiceConversation(true, voiceConversationId); } else { - console.warn("[Realtime] setRealtimeMode skipped: daemon unavailable"); + console.warn("[Voice] setVoiceConversation skipped: daemon unavailable"); } } catch (error: any) { - console.error("[Realtime] Failed to start:", error); + console.error("[Voice] Failed to start:", error); setActiveServerId((current) => (current === serverId ? null : current)); throw error; } @@ -186,42 +190,42 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) { [getSession, realtimeAudio] ); - const stopRealtime = useCallback(async () => { + const stopVoice = useCallback(async () => { try { const session = realtimeSessionRef.current; session?.audioPlayer?.stop(); await realtimeAudio.stop(); - setIsRealtimeMode(false); + setIsVoiceMode(false); setActiveServerId(null); - console.log("[Realtime] Mode disabled"); + console.log("[Voice] Mode disabled"); if (session?.client) { await session.client.setVoiceConversation(false); } else { - console.warn("[Realtime] setRealtimeMode skipped: daemon unavailable"); + console.warn("[Voice] setVoiceConversation skipped: daemon unavailable"); } } catch (error: any) { - console.error("[Realtime] Failed to stop:", error); + console.error("[Voice] Failed to stop:", error); throw error; } }, [realtimeAudio]); - const value: RealtimeContextValue = { - isRealtimeMode, + const value: VoiceContextValue = { + isVoiceMode, volume: realtimeAudio.volume, isMuted: realtimeAudio.isMuted, isDetecting: realtimeAudio.isDetecting, isSpeaking: realtimeAudio.isSpeaking, segmentDuration: realtimeAudio.segmentDuration, - startRealtime, - stopRealtime, + startVoice, + stopVoice, toggleMute: realtimeAudio.toggleMute, activeServerId, }; return ( - + {children} - + ); } diff --git a/packages/app/src/hooks/use-audio-player.web.ts b/packages/app/src/hooks/use-audio-player.web.ts index 53884a62e..7c833703c 100644 --- a/packages/app/src/hooks/use-audio-player.web.ts +++ b/packages/app/src/hooks/use-audio-player.web.ts @@ -7,7 +7,7 @@ export interface AudioPlayerOptions { /** * Web shim for the native two-way audio player. - * We currently don't support realtime voice playback on web. + * We currently don't support Voice mode playback on web. */ export function useAudioPlayer(_options?: AudioPlayerOptions) { return useMemo( diff --git a/packages/server/agent-prompt.md b/packages/server/agent-prompt.md index 1a7194bbe..07b8bbb4a 100644 --- a/packages/server/agent-prompt.md +++ b/packages/server/agent-prompt.md @@ -6,6 +6,15 @@ You are a **voice-controlled** assistant. The user speaks to you via phone and hears your responses via TTS. +### Voice Message Envelope + +Some user utterances will be wrapped in an XML tag like: + +`...` + +- Treat the inner text as what the user said (STT output). +- `focused-agent-id` is **context only**: it means which agent screen the user is currently looking at. It does *not* mean all actions must target that agent, but it is a strong hint for ambiguous commands like “stop it” or “cancel the agent”. + **Critical constraints:** - User typically codes from their **phone** using voice diff --git a/packages/server/src/client/daemon-client-v2.ts b/packages/server/src/client/daemon-client-v2.ts index 1cc400a4e..1e49511e2 100644 --- a/packages/server/src/client/daemon-client-v2.ts +++ b/packages/server/src/client/daemon-client-v2.ts @@ -1132,12 +1132,12 @@ export class DaemonClientV2 { this.sendSessionMessage({ type: "set_voice_conversation", enabled, voiceConversationId }); } - async sendRealtimeAudioChunk( + async sendVoiceAudioChunk( audio: string, format: string, isLast: boolean ): Promise { - this.sendSessionMessage({ type: "realtime_audio_chunk", audio, format, isLast }); + this.sendSessionMessage({ type: "voice_audio_chunk", audio, format, isLast }); } startDictationStream(dictationId: string, format: string): Promise { diff --git a/packages/server/src/server/agent/tts-manager.ts b/packages/server/src/server/agent/tts-manager.ts index f12b69519..2b0491a52 100644 --- a/packages/server/src/server/agent/tts-manager.ts +++ b/packages/server/src/server/agent/tts-manager.ts @@ -32,7 +32,7 @@ export class TTSManager { text: string, emitMessage: (msg: SessionOutboundMessage) => void, abortSignal: AbortSignal, - isRealtimeMode: boolean + isVoiceMode: boolean ): Promise { if (!this.tts) { throw new Error("TTS not configured"); @@ -117,7 +117,7 @@ export class TTSManager { isLastChunk: next.done, audio: chunkBuffer.toString("base64"), format, - isRealtimeMode, + isVoiceMode, }, }); diff --git a/packages/server/src/server/agent/wait-for-agent-tracker.ts b/packages/server/src/server/agent/wait-for-agent-tracker.ts index 80a4a1073..a26c53763 100644 --- a/packages/server/src/server/agent/wait-for-agent-tracker.ts +++ b/packages/server/src/server/agent/wait-for-agent-tracker.ts @@ -4,7 +4,7 @@ export type WaitForAgentCanceler = (agentId: string, reason?: string) => boolean /** * Tracks long-running wait_for_agent tool calls so they can be cancelled - * explicitly (e.g., when a realtime barge-in aborts the current turn). + * explicitly (e.g., when a voice barge-in aborts the current turn). */ export class WaitForAgentTracker { private waiters = new Map void>>(); diff --git a/packages/server/src/server/daemon-client-v2.e2e.test.ts b/packages/server/src/server/daemon-client-v2.e2e.test.ts index d959154ff..82314baa1 100644 --- a/packages/server/src/server/daemon-client-v2.e2e.test.ts +++ b/packages/server/src/server/daemon-client-v2.e2e.test.ts @@ -544,7 +544,7 @@ describe("daemon client v2 E2E", () => { ); test( - "realtime mode buffers audio until isLast and emits transcription_result", + "voice mode buffers audio until isLast and emits transcription_result", async () => { requireEnv("OPENAI_API_KEY"); @@ -613,7 +613,7 @@ describe("daemon client v2 E2E", () => { const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16 const firstChunk = pcm16.subarray(0, Math.min(chunkBytes, pcm16.length)); - await ctx.client.sendRealtimeAudioChunk(firstChunk.toString("base64"), format, false); + await ctx.client.sendVoiceAudioChunk(firstChunk.toString("base64"), format, false); await earlyTranscription .then(() => { throw new Error("Expected no transcription_result before isLast=true"); @@ -623,7 +623,7 @@ describe("daemon client v2 E2E", () => { for (let offset = chunkBytes; offset < pcm16.length; offset += chunkBytes) { const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes)); const isLast = offset + chunkBytes >= pcm16.length; - await ctx.client.sendRealtimeAudioChunk(chunk.toString("base64"), format, isLast); + await ctx.client.sendVoiceAudioChunk(chunk.toString("base64"), format, isLast); } const outcome = await Promise.race([ diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 640d9d194..2e1d97627 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -260,8 +260,8 @@ export class Session { private processingPhase: ProcessingPhase = "idle"; private currentStreamPromise: Promise | null = null; - // Realtime mode state - private isRealtimeMode = false; + // Voice mode state + private isVoiceMode = false; private speechInProgress = false; private voiceConversationId: string | null = null; @@ -369,6 +369,28 @@ export class Session { this.sessionLogger.info("Session created"); } + private escapeXmlText(value: string): string { + return value + .replace(/&/g, "&") + .replace(//g, ">"); + } + + private escapeXmlAttribute(value: string): string { + return this.escapeXmlText(value) + .replace(/"/g, """) + .replace(/'/g, "'"); + } + + private formatVoiceTranscriptionForLLM(text: string): string { + const trimmed = text.trim(); + const focusedAgentId = this.clientActivity?.focusedAgentId ?? null; + const focusedAttr = focusedAgentId + ? ` focused-agent-id="${this.escapeXmlAttribute(focusedAgentId)}"` + : ""; + return `${this.escapeXmlText(trimmed)}`; + } + /** * Get the client's current activity state */ @@ -735,7 +757,7 @@ export class Session { await this.handleUserText(msg.text); break; - case "realtime_audio_chunk": + case "voice_audio_chunk": await this.handleAudioChunk(msg); break; @@ -1214,7 +1236,7 @@ export class Session { } /** - * Handle realtime mode toggle + * Handle voice mode toggle */ private async handleSetVoiceConversation( enabled: boolean, @@ -1226,7 +1248,7 @@ export class Session { return; } - this.isRealtimeMode = true; + this.isVoiceMode = true; this.voiceConversationId = voiceConversationId; const loaded = await this.voiceConversationStore.load( @@ -1242,7 +1264,7 @@ export class Session { return; } - this.isRealtimeMode = false; + this.isVoiceMode = false; const idToPersist = this.voiceConversationId; if (idToPersist) { try { @@ -3875,8 +3897,8 @@ export class Session { // Add to conversation this.messages.push({ role: "user", content: text }); - // Process through LLM (TTS enabled in realtime mode for voice conversations) - this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode); + // Process through LLM (TTS enabled in voice mode for voice conversations) + this.currentStreamPromise = this.processWithLLM(this.isVoiceMode); await this.currentStreamPromise; } @@ -3884,9 +3906,9 @@ export class Session { * Handle audio chunk for buffering and transcription */ private async handleAudioChunk( - msg: Extract + msg: Extract ): Promise { - await this.handleRealtimeSpeechStart(); + await this.handleVoiceSpeechStart(); const chunkBuffer = Buffer.from(msg.audio, "base64"); const chunkFormat = msg.format || "audio/wav"; @@ -3932,19 +3954,19 @@ export class Session { `Buffered audio chunk (${chunkBuffer.length} bytes, chunks: ${this.audioBuffer.chunks.length}${this.audioBuffer.isPCM ? `, PCM bytes: ${this.audioBuffer.totalPCMBytes}` : ""})` ); - // In realtime mode, only process audio when the user has finished speaking (isLast = true) + // In voice mode, only process audio when the user has finished speaking (isLast = true) // This prevents partial transcriptions from being sent to the LLM - if (this.isRealtimeMode) { + if (this.isVoiceMode) { if (!msg.isLast) { - this.sessionLogger.debug("Realtime mode: buffering audio, waiting for speech end"); + this.sessionLogger.debug("Voice mode: buffering audio, waiting for speech end"); return; } - this.sessionLogger.debug("Realtime mode: speech ended, processing complete audio"); + this.sessionLogger.debug("Voice mode: speech ended, processing complete audio"); } - // In non-realtime mode, use streaming threshold to process chunks + // In non-voice mode, use streaming threshold to process chunks const reachedStreamingThreshold = - !this.isRealtimeMode && + !this.isVoiceMode && this.audioBuffer.isPCM && this.audioBuffer.totalPCMBytes >= MIN_STREAMING_SEGMENT_BYTES; @@ -4073,7 +4095,7 @@ export class Session { const requestId = uuidv4(); const result = await this.sttManager.transcribe(audio, format, { requestId, - label: this.isRealtimeMode ? "realtime" : "buffered", + label: this.isVoiceMode ? "voice" : "buffered", }); const transcriptText = result.text.trim(); @@ -4143,12 +4165,17 @@ export class Session { }); // Add to conversation - this.messages.push({ role: "user", content: result.text }); + this.messages.push({ + role: "user", + content: this.isVoiceMode + ? this.formatVoiceTranscriptionForLLM(result.text) + : result.text, + }); - // Set phase to LLM and process (TTS enabled in realtime mode for voice conversations) + // Set phase to LLM and process (TTS enabled in voice mode for voice conversations) this.clearSpeechInProgress("transcription complete"); this.setPhase("llm"); - this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode); + this.currentStreamPromise = this.processWithLLM(this.isVoiceMode); await this.currentStreamPromise; this.setPhase("idle"); } catch (error: any) { @@ -4180,7 +4207,7 @@ export class Session { if (textBuffer.length > 0) { // TTS handling (capture mode at generation time for drift protection) if (enableTTS && !this.speechInProgress) { - const modeAtGeneration = this.isRealtimeMode; + const modeAtGeneration = this.isVoiceMode; pendingTTS = this.ttsManager.generateAndWaitForPlayback( textBuffer, (msg) => this.emit(msg), @@ -4572,7 +4599,7 @@ export class Session { /** * Mark speech detection start and abort any active playback/LLM */ - private async handleRealtimeSpeechStart(): Promise { + private async handleVoiceSpeechStart(): Promise { if (this.speechInProgress) { return; } @@ -4582,13 +4609,13 @@ export class Session { const hadActiveStream = Boolean(this.currentStreamPromise); this.speechInProgress = true; - this.sessionLogger.debug("Realtime speech chunk detected – aborting playback and LLM"); - this.ttsManager.cancelPendingPlaybacks("realtime speech detected"); + this.sessionLogger.debug("Voice speech detected – aborting playback and LLM"); + this.ttsManager.cancelPendingPlaybacks("voice speech detected"); if (this.pendingAudioSegments.length > 0) { this.sessionLogger.debug( { segmentCount: this.pendingAudioSegments.length }, - `Dropping ${this.pendingAudioSegments.length} buffered audio segment(s) due to realtime speech` + `Dropping ${this.pendingAudioSegments.length} buffered audio segment(s) due to voice speech` ); this.pendingAudioSegments = []; } diff --git a/packages/server/src/shared/messages.ts b/packages/server/src/shared/messages.ts index 2f7221c00..a2c8f4dfe 100644 --- a/packages/server/src/shared/messages.ts +++ b/packages/server/src/shared/messages.ts @@ -288,8 +288,8 @@ export const UserTextMessageSchema = z.object({ text: z.string(), }); -export const RealtimeAudioChunkMessageSchema = z.object({ - type: z.literal("realtime_audio_chunk"), +export const VoiceAudioChunkMessageSchema = z.object({ + type: z.literal("voice_audio_chunk"), audio: z.string(), // base64 encoded format: z.string(), isLast: z.boolean(), @@ -804,7 +804,7 @@ export const KillTerminalRequestSchema = z.object({ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ UserTextMessageSchema, - RealtimeAudioChunkMessageSchema, + VoiceAudioChunkMessageSchema, AbortRequestMessageSchema, AudioPlayedMessageSchema, FetchAgentsRequestMessageSchema, @@ -900,7 +900,7 @@ export const AudioOutputMessageSchema = z.object({ audio: z.string(), // base64 encoded format: z.string(), id: z.string(), - isRealtimeMode: z.boolean(), // Mode when audio was generated (for drift protection) + isVoiceMode: z.boolean(), // Mode when audio was generated (for drift protection) groupId: z.string().optional(), // Logical utterance id chunkIndex: z.number().int().nonnegative().optional(), isLastChunk: z.boolean().optional(), @@ -1643,7 +1643,7 @@ export type ActivityLogPayload = z.infer; // Type exports for inbound message types export type UserTextMessage = z.infer; -export type RealtimeAudioChunkMessage = z.infer; +export type VoiceAudioChunkMessage = z.infer; export type FetchAgentsRequestMessage = z.infer; export type FetchAgentRequestMessage = z.infer; export type SendAgentMessageRequest = z.infer;