puter/paseo
1
0
Fork 0
mirror of https://github.com/getpaseo/paseo.git synced 2026-07-29 12:01:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9 from boudra/real-time/voice-agent-ui-refresh

Browse Source 
refactor: rename realtime to voice throughout app and server
This commit is contained in:
Mohamed Boudra
2026-02-03 23:44:33 +07:00
committed by GitHub
parent 201fbcc0c6 cf9c1c23d3
commit 03f383b908
16 changed files with 526 additions and 581 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/app/src/app/_layout.tsx
View File

@@ -5,7 +5,7 @@ import { KeyboardProvider } from "react-native-keyboard-controller";
import { GestureHandlerRootView, Gesture, GestureDetector } from "react-native-gesture-handler";
import { BottomSheetModalProvider } from "@gorhom/bottom-sheet";
import { PortalProvider } from "@gorhom/portal";
import { RealtimeProvider } from "@/contexts/realtime-context";
import { VoiceProvider } from "@/contexts/voice-context";
import { useAppSettings } from "@/hooks/use-settings";
import { useFaviconStatus } from "@/hooks/use-favicon-status";
import { View, ActivityIndicator, Text } from "react-native";
@@ -284,10 +284,10 @@ function ProvidersWrapper({ children }: { children: ReactNode }) {
}
return (
<RealtimeProvider>
<VoiceProvider>
<OfferLinkListener upsertDaemonFromOfferUrl={upsertDaemonFromOfferUrl} />
{children}
</RealtimeProvider>
</VoiceProvider>
);
}

145
packages/app/src/components/agent-input-area.tsx
View File

@@ -1,26 +1,20 @@
import {
View,
Pressable,
Platform,
Text,
ActivityIndicator,
} from "react-native";
import { useState, useEffect, useRef, useCallback, useMemo } from "react";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { ArrowUp, AudioLines, Square, Pencil } from "lucide-react-native";
import Animated, {
useAnimatedStyle,
FadeIn,
FadeOut,
} from "react-native-reanimated";
import { ArrowUp, AudioLines, MicOff, Square, Pencil } from "lucide-react-native";
import Animated, { useAnimatedStyle } from "react-native-reanimated";
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
import { useSafeAreaInsets } from "react-native-safe-area-context";
import { useRealtime } from "@/contexts/realtime-context";
import { useVoice } from "@/contexts/voice-context";
import { useIsFocused } from "@react-navigation/native";
import { FOOTER_HEIGHT, MAX_CONTENT_WIDTH } from "@/constants/layout";
import { generateMessageId } from "@/types/stream";
import { AgentStatusBar } from "./agent-status-bar";
import { RealtimeControls } from "./realtime-controls";
import { useImageAttachmentPicker } from "@/hooks/use-image-attachment-picker";
import { useSessionStore } from "@/stores/session-store";
import { useDraftStore } from "@/stores/draft-store";
@@ -57,15 +51,6 @@ interface AgentInputAreaProps {
}
const EMPTY_ARRAY: readonly QueuedMessage[] = [];
// Android currently crashes inside ViewGroup.dispatchDraw when running Reanimated
// entering/exiting animations (see react-native-reanimated#8422), so guard them.
const SHOULD_DISABLE_ENTRY_EXIT_ANIMATIONS = Platform.OS === "android";
const REALTIME_FADE_IN = SHOULD_DISABLE_ENTRY_EXIT_ANIMATIONS
? undefined
: FadeIn.duration(250);
const REALTIME_FADE_OUT = SHOULD_DISABLE_ENTRY_EXIT_ANIMATIONS
? undefined
: FadeOut.duration(250);
export function AgentInputArea({
agentId,
@@ -106,7 +91,7 @@ export function AgentInputArea({
const setQueuedMessages = useSessionStore((state) => state.setQueuedMessages);
const { startRealtime, stopRealtime, isRealtimeMode } = useRealtime();
const { isVoiceMode, isMuted: isVoiceMuted, toggleMute: toggleVoiceMute } = useVoice();
const [internalInput, setInternalInput] = useState("");
const userInput = value ?? internalInput;
@@ -345,21 +330,6 @@ export function AgentInputArea({
};
});
async function handleRealtimePress() {
try {
if (isRealtimeMode) {
await stopRealtime();
} else {
if (!isConnected || !serverId) {
return;
}
await startRealtime(serverId);
}
} catch (error) {
console.error("[AgentInput] Failed to toggle realtime mode:", error);
}
}
function handleCancelAgent() {
if (!agent || agent.status !== "running" || isCancellingAgent) {
return;
@@ -474,29 +444,7 @@ export function AgentInputArea({
<Square size={18} color="white" fill="white" />
)}
</Pressable>
) : (
<Pressable
onPress={handleRealtimePress}
disabled={!isConnected && !isRealtimeMode}
style={[
styles.realtimeButton as any,
(isRealtimeMode ? styles.realtimeButtonActive : undefined) as any,
(!isConnected && !isRealtimeMode
? styles.buttonDisabled
: undefined) as any,
]}
>
{isRealtimeMode ? (
<Square
size={18}
color={theme.colors.background}
fill={theme.colors.background}
/>
) : (
<AudioLines size={20} color={theme.colors.background} />
)}
</Pressable>
);
) : null;
const leftContent = <AgentStatusBar agentId={agentId} serverId={serverId} />;
@@ -508,17 +456,6 @@ export function AgentInputArea({
keyboardAnimatedStyle,
]}
>
{/* Realtime controls - only when active */}
{isRealtimeMode && (
<Animated.View
style={styles.realtimeControlsContainer}
entering={REALTIME_FADE_IN}
exiting={REALTIME_FADE_OUT}
>
<RealtimeControls />
</Animated.View>
)}
{/* Input area */}
<View style={styles.inputAreaContainer}>
<View style={styles.inputAreaContent}>
@@ -564,6 +501,35 @@ export function AgentInputArea({
/>
)}
{/* Voice quick mute indicator */}
{isVoiceMode && (
<View style={styles.voiceIndicatorRow}>
<Pressable
onPress={toggleVoiceMute}
accessibilityRole="button"
accessibilityLabel={isVoiceMuted ? "Unmute voice" : "Mute voice"}
style={[
styles.voiceIndicatorPill,
isVoiceMuted && styles.voiceIndicatorPillMuted,
]}
>
{isVoiceMuted ? (
<MicOff size={14} color={theme.colors.surface0} />
) : (
<AudioLines size={14} color={theme.colors.foreground} />
)}
<Text
style={[
styles.voiceIndicatorText,
isVoiceMuted && styles.voiceIndicatorTextMuted,
]}
>
{isVoiceMuted ? "Voice muted" : "Voice on"}
</Text>
</Pressable>
</View>
)}
{/* MessageInput handles everything: text, dictation, attachments, all buttons */}
<MessageInput
ref={messageInputRef}
@@ -578,7 +544,7 @@ export function AgentInputArea({
client={client}
placeholder="Message agent..."
autoFocus={autoFocus}
disabled={isRealtimeMode || isSubmitLoading}
disabled={isSubmitLoading}
isScreenFocused={isScreenFocused}
leftContent={leftContent}
rightContent={rightContent}
@@ -603,9 +569,6 @@ const styles = StyleSheet.create(((theme: Theme) => ({
height: theme.borderWidth[1],
backgroundColor: theme.colors.border,
},
realtimeControlsContainer: {
height: FOOTER_HEIGHT,
},
inputAreaContainer: {
position: "relative",
minHeight: FOOTER_HEIGHT,
@@ -620,17 +583,6 @@ const styles = StyleSheet.create(((theme: Theme) => ({
maxWidth: MAX_CONTENT_WIDTH,
gap: theme.spacing[3],
},
realtimeButton: {
width: 34,
height: 34,
borderRadius: theme.borderRadius.full,
backgroundColor: theme.colors.accentForeground,
alignItems: "center",
justifyContent: "center",
},
realtimeButtonActive: {
backgroundColor: theme.colors.palette.blue[600],
},
cancelButton: {
width: 34,
height: 34,
@@ -642,6 +594,33 @@ const styles = StyleSheet.create(((theme: Theme) => ({
buttonDisabled: {
opacity: 0.5,
},
voiceIndicatorRow: {
flexDirection: "row",
justifyContent: "flex-end",
},
voiceIndicatorPill: {
flexDirection: "row",
alignItems: "center",
gap: theme.spacing[2],
paddingHorizontal: theme.spacing[3],
height: 32,
borderRadius: theme.borderRadius.full,
backgroundColor: theme.colors.surface2,
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
},
voiceIndicatorPillMuted: {
backgroundColor: theme.colors.palette.red[600],
borderColor: theme.colors.palette.red[800],
},
voiceIndicatorText: {
color: theme.colors.foreground,
fontSize: theme.fontSize.xs,
fontWeight: theme.fontWeight.medium,
},
voiceIndicatorTextMuted: {
color: theme.colors.surface0,
},
queueContainer: {
flexDirection: "column",
gap: theme.spacing[2],

297
packages/app/src/components/home-footer.tsx
View File

@@ -1,297 +0,0 @@
import { useCallback, useMemo, useState } from "react";
import { View, Pressable, Text, Platform, Modal, Alert } from "react-native";
import { useRouter } from "expo-router";
import { useSafeAreaInsets } from "react-native-safe-area-context";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { AudioLines, Users, Plus } from "lucide-react-native";
import { useRealtime } from "@/contexts/realtime-context";
import { useDaemonConnections } from "@/contexts/daemon-connections-context";
import { FOOTER_HEIGHT } from "@/constants/layout";
import { RealtimeControls } from "./realtime-controls";
import Animated, {
FadeIn,
FadeOut,
useAnimatedStyle,
} from "react-native-reanimated";
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
export function HomeFooter() {
const { theme } = useUnistyles();
const insets = useSafeAreaInsets();
const router = useRouter();
const { isRealtimeMode, startRealtime } = useRealtime();
const { connectionStates } = useDaemonConnections();
const [showRealtimeHostPicker, setShowRealtimeHostPicker] = useState(false);
// Guard Reanimated entry/exit transitions on Android to avoid ViewGroup.dispatchDraw crashes
// tracked in react-native-reanimated#8422.
const shouldDisableEntryExitAnimations = Platform.OS === "android";
const realtimeFadeIn = shouldDisableEntryExitAnimations ? undefined : FadeIn.duration(250);
const realtimeFadeOut = shouldDisableEntryExitAnimations ? undefined : FadeOut.duration(250);
const { height: keyboardHeight } = useReanimatedKeyboardAnimation();
const bottomInset = insets.bottom;
const keyboardAnimatedStyle = useAnimatedStyle(
() => {
"worklet";
const absoluteHeight = Math.abs(keyboardHeight.value);
const shift = Math.max(0, absoluteHeight - bottomInset);
return {
transform: [{ translateY: -shift }],
};
},
[bottomInset],
);
const realtimeEligibleHosts = useMemo(() => {
return Array.from(connectionStates.values()).filter((entry) => entry.status === "online");
}, [connectionStates]);
const hasAnyConfiguredHosts = connectionStates.size > 0;
const handleStartRealtime = useCallback(() => {
if (realtimeEligibleHosts.length === 0) {
if (!hasAnyConfiguredHosts) {
Alert.alert(
"No hosts available",
"Add a host in Settings before starting realtime mode."
);
return;
}
Alert.alert(
"Hosts reconnecting",
"Every host is offline right now. Paseo reconnects automatically—try realtime again once one comes online."
);
return;
}
if (realtimeEligibleHosts.length === 1) {
void startRealtime(realtimeEligibleHosts[0].daemon.id).catch((error) => {
console.error("[HomeFooter] Failed to start realtime", error);
Alert.alert("Realtime failed", "Unable to start realtime mode for this host.");
});
return;
}
setShowRealtimeHostPicker(true);
}, [hasAnyConfiguredHosts, realtimeEligibleHosts, startRealtime]);
const handleSelectRealtimeHost = useCallback(
(daemonId: string) => {
setShowRealtimeHostPicker(false);
void startRealtime(daemonId).catch((error) => {
console.error("[HomeFooter] Failed to start realtime", error);
Alert.alert("Realtime failed", "Unable to start realtime mode for this host.");
});
},
[startRealtime]
);
const handleDismissHostPicker = useCallback(() => {
setShowRealtimeHostPicker(false);
}, []);
// For home and orchestrator screens, show action buttons with realtime stacked on top
const nonAgentFooterHeight = isRealtimeMode
? FOOTER_HEIGHT * 2 + insets.bottom
: FOOTER_HEIGHT + insets.bottom;
const iconSize = 24;
const iconStyle = { width: iconSize, height: iconSize };
return (
<>
<Animated.View
style={[
styles.container,
{
paddingBottom: insets.bottom,
height: nonAgentFooterHeight,
},
keyboardAnimatedStyle,
]}
>
<View style={styles.nonAgentContent}>
{/* Realtime controls - only visible when active */}
{isRealtimeMode && (
<Animated.View
style={styles.realtimeSection}
entering={realtimeFadeIn}
exiting={realtimeFadeOut}
>
<RealtimeControls />
</Animated.View>
)}
{/* Action menu */}
<View style={styles.actionButtonContainer}>
<Pressable
onPress={() => router.push("/agents")}
style={({ pressed }) => [
styles.footerButton,
pressed && styles.buttonPressed,
]}
>
<View style={styles.footerIconWrapper}>
<Users
size={iconSize}
color={theme.colors.foreground}
style={iconStyle}
/>
</View>
<Text style={styles.footerButtonText}>Agents</Text>
</Pressable>
<Pressable
onPress={() => {
console.log("[HomeFooter] New Agent button pressed");
router.push("/agent" as any);
}}
style={({ pressed }) => [
styles.footerButton,
pressed && styles.buttonPressed,
]}
>
<View style={styles.footerIconWrapper}>
<Plus
size={iconSize}
color={theme.colors.foreground}
style={iconStyle}
/>
</View>
<Text style={styles.footerButtonText}>New agent</Text>
</Pressable>
<Pressable
onPress={handleStartRealtime}
disabled={realtimeEligibleHosts.length === 0}
style={({ pressed }) => [
styles.footerButton,
realtimeEligibleHosts.length === 0 && styles.buttonDisabled,
pressed && realtimeEligibleHosts.length === 0 && styles.buttonPressed,
]}
>
<View style={styles.footerIconWrapper}>
<AudioLines
size={iconSize}
color={theme.colors.foreground}
style={iconStyle}
/>
</View>
<Text style={styles.footerButtonText}>Realtime</Text>
</Pressable>
</View>
</View>
</Animated.View>
<Modal
visible={showRealtimeHostPicker}
transparent
animationType="fade"
onRequestClose={handleDismissHostPicker}
>
<View style={styles.hostPickerOverlay}>
<Pressable style={styles.hostPickerBackdrop} onPress={handleDismissHostPicker} />
<View style={styles.hostPickerContainer}>
<Text style={styles.hostPickerTitle}>Choose a host</Text>
{realtimeEligibleHosts.map((entry) => (
<Pressable
key={entry.daemon.id}
style={styles.hostPickerButton}
onPress={() => handleSelectRealtimeHost(entry.daemon.id)}
>
<Text style={styles.hostPickerButtonText}>{entry.daemon.label}</Text>
</Pressable>
))}
<Pressable style={styles.hostPickerCancel} onPress={handleDismissHostPicker}>
<Text style={styles.hostPickerCancelText}>Cancel</Text>
</Pressable>
</View>
</View>
</Modal>
</>
);
}
const styles = StyleSheet.create((theme) => ({
container: {
backgroundColor: theme.colors.surface0,
borderTopWidth: theme.borderWidth[1],
borderTopColor: theme.colors.border,
},
nonAgentContent: {
flexDirection: "column",
},
realtimeSection: {
height: FOOTER_HEIGHT,
borderBottomWidth: theme.borderWidth[1],
borderBottomColor: theme.colors.border,
},
actionButtonContainer: {
flexDirection: "row",
padding: theme.spacing[4],
gap: theme.spacing[3],
height: FOOTER_HEIGHT,
},
footerButton: {
flex: 1,
flexDirection: "column",
alignItems: "center",
justifyContent: "center",
paddingVertical: theme.spacing[3],
gap: theme.spacing[1],
},
footerIconWrapper: {
width: 28,
height: 28,
alignItems: "center",
justifyContent: "center",
},
footerButtonText: {
color: theme.colors.foreground,
fontSize: theme.fontSize.xs,
fontWeight: theme.fontWeight.normal,
},
buttonDisabled: {
opacity: 0.5,
},
buttonPressed: {
opacity: 0.5,
},
hostPickerOverlay: {
flex: 1,
backgroundColor: "rgba(0,0,0,0.5)",
justifyContent: "flex-end",
},
hostPickerBackdrop: {
flex: 1,
},
hostPickerContainer: {
backgroundColor: theme.colors.surface2,
padding: theme.spacing[4],
borderTopLeftRadius: theme.borderRadius.xl,
borderTopRightRadius: theme.borderRadius.xl,
gap: theme.spacing[3],
},
hostPickerTitle: {
fontSize: theme.fontSize.lg,
fontWeight: theme.fontWeight.semibold,
color: theme.colors.foreground,
textAlign: "center",
},
hostPickerButton: {
paddingVertical: theme.spacing[3],
borderRadius: theme.borderRadius.lg,
backgroundColor: theme.colors.surface2,
},
hostPickerButtonText: {
color: theme.colors.foreground,
fontSize: theme.fontSize.sm,
textAlign: "center",
},
hostPickerCancel: {
paddingVertical: theme.spacing[3],
},
hostPickerCancelText: {
color: theme.colors.foregroundMuted,
fontSize: theme.fontSize.sm,
textAlign: "center",
},
}));

7
packages/app/src/components/message-input.tsx
View File

@@ -28,6 +28,7 @@ import { useDictation } from "@/hooks/use-dictation";
import { DictationOverlay } from "./dictation-controls";
import type { DaemonClientV2 } from "@server/client/daemon-client-v2";
import { usePanelStore } from "@/stores/panel-store";
import { useVoiceOptional } from "@/contexts/voice-context";
export interface ImageAttachment {
uri: string;
@@ -118,6 +119,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
ref
) {
const { theme } = useUnistyles();
const voice = useVoiceOptional();
const toggleAgentList = usePanelStore((state) => state.toggleAgentList);
const [inputHeight, setInputHeight] = useState(MIN_INPUT_HEIGHT);
const textInputRef = useRef<
@@ -294,9 +296,12 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
if (isDictating) {
await cancelDictation();
} else {
if (voice?.isVoiceMode) {
await voice.stopVoice();
}
await startDictation();
}
}, [isDictating, cancelDictation, startDictation]);
}, [isDictating, cancelDictation, startDictation, voice]);
const handleCancelRecording = useCallback(async () => {
await cancelDictation();

110
packages/app/src/components/realtime-controls.tsx
View File

@@ -1,110 +0,0 @@
import { View, Text, Pressable } from "react-native";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { MicOff, Square } from "lucide-react-native";
import { VolumeMeter } from "./volume-meter";
import { useRealtime } from "@/contexts/realtime-context";
import { FOOTER_HEIGHT } from "@/constants/layout";
const CONTROL_BUTTON_SIZE = 48;
const VERTICAL_PADDING = (FOOTER_HEIGHT - CONTROL_BUTTON_SIZE) / 2;
export function RealtimeControls() {
const { theme } = useUnistyles();
const {
volume,
isMuted,
isDetecting,
isSpeaking,
segmentDuration,
stopRealtime,
toggleMute,
} = useRealtime();
function handleStop() {
stopRealtime();
}
return (
<View style={styles.container}>
<View style={styles.volumeContainer}>
<VolumeMeter
volume={volume}
isMuted={isMuted}
isDetecting={isDetecting}
isSpeaking={isSpeaking}
orientation="horizontal"
/>
</View>
<View style={styles.buttons}>
{/* Mute button */}
<Pressable
onPress={toggleMute}
style={[
styles.muteButton,
isMuted && styles.muteButtonActive,
]}
>
<MicOff
size={20}
color={
isMuted
? theme.colors.surface0
: theme.colors.foreground
}
/>
</Pressable>
{/* Stop button */}
<Pressable
onPress={handleStop}
style={styles.stopButton}
>
<Square size={18} color="white" fill="white" />
</Pressable>
</View>
</View>
);
}
const styles = StyleSheet.create((theme) => ({
container: {
flexDirection: "row",
alignItems: "center",
justifyContent: "space-between",
paddingHorizontal: theme.spacing[4],
paddingVertical: VERTICAL_PADDING,
height: FOOTER_HEIGHT,
},
volumeContainer: {
flex: 1,
justifyContent: "center",
alignItems: "flex-start",
},
buttons: {
flexDirection: "row",
alignItems: "center",
justifyContent: "center",
gap: theme.spacing[3],
},
muteButton: {
width: 48,
height: 48,
borderRadius: theme.borderRadius.full,
alignItems: "center",
justifyContent: "center",
backgroundColor: theme.colors.surface2,
borderWidth: theme.borderWidth[2],
borderColor: theme.colors.border,
},
muteButtonActive: {
backgroundColor: theme.colors.palette.red[500],
borderColor: theme.colors.palette.red[600],
},
stopButton: {
width: 48,
height: 48,
borderRadius: theme.borderRadius.full,
alignItems: "center",
justifyContent: "center",
backgroundColor: theme.colors.palette.red[600],
},
}));

209
packages/app/src/components/sliding-sidebar.tsx
View File

@@ -1,5 +1,5 @@
import { useCallback, useMemo, useState, useEffect } from "react";
import { View, Pressable, Text, Platform } from "react-native";
import { View, Pressable, Text, Platform, Modal, Alert } from "react-native";
import { useSafeAreaInsets } from "react-native-safe-area-context";
import Animated, {
useAnimatedStyle,
@@ -9,13 +9,16 @@ import Animated, {
} from "react-native-reanimated";
import { Gesture, GestureDetector } from "react-native-gesture-handler";
import { StyleSheet, UnistylesRuntime, useUnistyles } from "react-native-unistyles";
import { Plus, Settings, Users } from "lucide-react-native";
import { Plus, Settings, Users, AudioLines } from "lucide-react-native";
import { router } from "expo-router";
import { usePanelStore } from "@/stores/panel-store";
import { GroupedAgentList } from "./grouped-agent-list";
import { useAggregatedAgents } from "@/hooks/use-aggregated-agents";
import { useSidebarAnimation } from "@/contexts/sidebar-animation-context";
import { useTauriDragHandlers, useTrafficLightPadding } from "@/utils/tauri-window";
import { useVoice } from "@/contexts/voice-context";
import { useDaemonConnections } from "@/contexts/daemon-connections-context";
import { VoicePanel } from "./voice-panel";
const DESKTOP_SIDEBAR_WIDTH = 320;
const SIDEBAR_AGENT_LIMIT = 15;
@@ -48,6 +51,9 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
} = useSidebarAnimation();
const trafficLightPadding = useTrafficLightPadding();
const dragHandlers = useTauriDragHandlers();
const { connectionStates } = useDaemonConnections();
const { isVoiceMode, startVoice, stopVoice } = useVoice();
const [showVoiceHostPicker, setShowVoiceHostPicker] = useState(false);
// Track user-initiated refresh to avoid showing spinner on background revalidation
const [isManualRefresh, setIsManualRefresh] = useState(false);
@@ -124,6 +130,62 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
router.push("/agents");
}, [backdropOpacity, closeToAgent, isMobile, translateX, windowWidth]);
const voiceEligibleHosts = useMemo(() => {
return Array.from(connectionStates.values()).filter((entry) => entry.status === "online");
}, [connectionStates]);
const hasAnyConfiguredHosts = connectionStates.size > 0;
const handleToggleVoice = useCallback(() => {
if (isVoiceMode) {
void stopVoice().catch((error) => {
console.error("[SlidingSidebar] Failed to stop voice", error);
Alert.alert("Voice failed", "Unable to stop voice mode.");
});
return;
}
if (voiceEligibleHosts.length === 0) {
if (!hasAnyConfiguredHosts) {
Alert.alert(
"No hosts available",
"Add a host in Settings before starting Voice mode.",
[{ text: "Open Settings", onPress: () => router.push("/settings") }, { text: "OK" }]
);
return;
}
Alert.alert(
"Hosts reconnecting",
"Every host is offline right now. Paseo reconnects automatically—try Voice mode again once one comes online."
);
return;
}
if (voiceEligibleHosts.length === 1) {
void startVoice(voiceEligibleHosts[0].daemon.id).catch((error) => {
console.error("[SlidingSidebar] Failed to start voice", error);
Alert.alert("Voice failed", "Unable to start Voice mode for this host.");
});
return;
}
setShowVoiceHostPicker(true);
}, [hasAnyConfiguredHosts, isVoiceMode, startVoice, stopVoice, voiceEligibleHosts]);
const handleSelectVoiceHost = useCallback(
(daemonId: string) => {
setShowVoiceHostPicker(false);
void startVoice(daemonId).catch((error) => {
console.error("[SlidingSidebar] Failed to start voice", error);
Alert.alert("Voice failed", "Unable to start Voice mode for this host.");
});
},
[startVoice]
);
const handleDismissVoiceHostPicker = useCallback(() => {
setShowVoiceHostPicker(false);
}, []);
// Close gesture (swipe left to close when sidebar is open)
// Only activates on leftward swipe, fails on rightward or vertical movement
// This mirrors the explorer-sidebar pattern for the right sidebar
@@ -224,6 +286,8 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
parentGestureRef={closeGestureRef}
/>
{isVoiceMode ? <VoicePanel /> : null}
{/* Footer */}
<View style={styles.sidebarFooter}>
<Pressable
@@ -239,6 +303,25 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
</>
)}
</Pressable>
<View style={styles.footerIconRow}>
<Pressable
style={styles.footerIconButton}
testID="sidebar-voice"
onPress={handleToggleVoice}
>
{({ hovered }) => (
<AudioLines
size={20}
color={
isVoiceMode
? theme.colors.foreground
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
)}
</Pressable>
<Pressable
style={styles.footerIconButton}
onPress={handleSettingsMobile}
@@ -247,10 +330,37 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
<Settings size={20} color={hovered ? theme.colors.foreground : theme.colors.foregroundMuted} />
)}
</Pressable>
</View>
</View>
</View>
</Animated.View>
</GestureDetector>
<Modal
visible={showVoiceHostPicker}
transparent
animationType="fade"
onRequestClose={handleDismissVoiceHostPicker}
>
<View style={styles.hostPickerOverlay}>
<Pressable style={styles.hostPickerBackdrop} onPress={handleDismissVoiceHostPicker} />
<View style={styles.hostPickerContainer}>
<Text style={styles.hostPickerTitle}>Choose a host</Text>
{voiceEligibleHosts.map((entry) => (
<Pressable
key={entry.daemon.id}
style={styles.hostPickerButton}
onPress={() => handleSelectVoiceHost(entry.daemon.id)}
>
<Text style={styles.hostPickerButtonText}>{entry.daemon.label}</Text>
</Pressable>
))}
<Pressable style={styles.hostPickerCancel} onPress={handleDismissVoiceHostPicker}>
<Text style={styles.hostPickerCancelText}>Cancel</Text>
</Pressable>
</View>
</View>
</Modal>
</View>
);
}
@@ -289,6 +399,8 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
selectedAgentId={selectedAgentId}
/>
{isVoiceMode ? <VoicePanel /> : null}
{/* Footer */}
<View style={styles.sidebarFooter}>
<Pressable
@@ -304,14 +416,34 @@ export function SlidingSidebar({ selectedAgentId }: SlidingSidebarProps) {
</>
)}
</Pressable>
<Pressable
style={styles.footerIconButton}
onPress={handleSettingsDesktop}
>
{({ hovered }) => (
<Settings size={20} color={hovered ? theme.colors.foreground : theme.colors.foregroundMuted} />
)}
</Pressable>
<View style={styles.footerIconRow}>
<Pressable
style={styles.footerIconButton}
testID="sidebar-voice"
onPress={handleToggleVoice}
>
{({ hovered }) => (
<AudioLines
size={20}
color={
isVoiceMode
? theme.colors.foreground
: hovered
? theme.colors.foreground
: theme.colors.foregroundMuted
}
/>
)}
</Pressable>
<Pressable
style={styles.footerIconButton}
onPress={handleSettingsDesktop}
>
{({ hovered }) => (
<Settings size={20} color={hovered ? theme.colors.foreground : theme.colors.foregroundMuted} />
)}
</Pressable>
</View>
</View>
</View>
);
@@ -376,6 +508,11 @@ const styles = StyleSheet.create((theme) => ({
borderTopWidth: 1,
borderTopColor: theme.colors.border,
},
footerIconRow: {
flexDirection: "row",
alignItems: "center",
gap: theme.spacing[3],
},
footerButton: {
flexDirection: "row",
alignItems: "center",
@@ -395,4 +532,56 @@ const styles = StyleSheet.create((theme) => ({
footerButtonTextHovered: {
color: theme.colors.foreground,
},
hostPickerOverlay: {
flex: 1,
justifyContent: "center",
alignItems: "center",
backgroundColor: "rgba(0, 0, 0, 0.5)",
padding: theme.spacing[4],
},
hostPickerBackdrop: {
...StyleSheet.absoluteFillObject,
},
hostPickerContainer: {
width: "100%",
maxWidth: 360,
backgroundColor: theme.colors.surface0,
borderRadius: theme.borderRadius["2xl"],
padding: theme.spacing[4],
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
gap: theme.spacing[2],
},
hostPickerTitle: {
fontSize: theme.fontSize.base,
fontWeight: theme.fontWeight.semibold,
color: theme.colors.foreground,
marginBottom: theme.spacing[2],
},
hostPickerButton: {
paddingVertical: theme.spacing[3],
paddingHorizontal: theme.spacing[3],
borderRadius: theme.borderRadius.lg,
backgroundColor: theme.colors.surface2,
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
},
hostPickerButtonText: {
color: theme.colors.foreground,
fontSize: theme.fontSize.sm,
},
hostPickerCancel: {
marginTop: theme.spacing[2],
paddingVertical: theme.spacing[3],
paddingHorizontal: theme.spacing[3],
borderRadius: theme.borderRadius.lg,
backgroundColor: theme.colors.surface0,
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
alignItems: "center",
},
hostPickerCancelText: {
color: theme.colors.foregroundMuted,
fontSize: theme.fontSize.sm,
},
}));

139
packages/app/src/components/voice-panel.tsx Normal file
View File

@@ -0,0 +1,139 @@
import { View, Text, Pressable } from "react-native";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { MicOff, Square, AudioLines } from "lucide-react-native";
import { VolumeMeter } from "./volume-meter";
import { useVoice } from "@/contexts/voice-context";
import { useDaemonConnections } from "@/contexts/daemon-connections-context";
export function VoicePanel() {
const { theme } = useUnistyles();
const { connectionStates } = useDaemonConnections();
const {
volume,
isMuted,
isDetecting,
isSpeaking,
stopVoice,
toggleMute,
activeServerId,
} = useVoice();
const activeHost = activeServerId ? connectionStates.get(activeServerId) ?? null : null;
const hostLabel = activeHost?.daemon.label ?? null;
const hostStatus = activeHost?.status ?? null;
return (
<View style={styles.container}>
<View style={styles.headerRow}>
<View style={styles.titleRow}>
<AudioLines size={16} color={theme.colors.foregroundMuted} />
<Text style={styles.titleText}>Voice</Text>
</View>
<Text style={styles.hostText} numberOfLines={1}>
{hostLabel ? `Host: ${hostLabel}` : "Host: unknown"}
{hostStatus ? ` (${hostStatus})` : ""}
</Text>
</View>
<View style={styles.meterRow}>
<VolumeMeter
volume={volume}
isMuted={isMuted}
isDetecting={isDetecting}
isSpeaking={isSpeaking}
orientation="horizontal"
/>
</View>
<View style={styles.actionsRow}>
<Pressable
onPress={toggleMute}
accessibilityRole="button"
accessibilityLabel={isMuted ? "Unmute voice" : "Mute voice"}
style={[
styles.iconButton,
isMuted && styles.iconButtonMuted,
]}
>
<MicOff
size={18}
color={isMuted ? theme.colors.surface0 : theme.colors.foreground}
/>
</Pressable>
<Pressable
onPress={() => void stopVoice()}
accessibilityRole="button"
accessibilityLabel="Stop voice mode"
style={[styles.iconButton, styles.iconButtonStop]}
>
<Square size={16} color="white" fill="white" />
</Pressable>
</View>
</View>
);
}
const styles = StyleSheet.create((theme) => ({
container: {
marginHorizontal: theme.spacing[4],
marginBottom: theme.spacing[3],
borderRadius: theme.borderRadius["2xl"],
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
backgroundColor: theme.colors.surface2,
paddingVertical: theme.spacing[3],
paddingHorizontal: theme.spacing[3],
gap: theme.spacing[3],
},
headerRow: {
flexDirection: "row",
alignItems: "center",
justifyContent: "space-between",
gap: theme.spacing[3],
},
titleRow: {
flexDirection: "row",
alignItems: "center",
gap: theme.spacing[2],
flexShrink: 0,
},
titleText: {
color: theme.colors.foreground,
fontSize: theme.fontSize.sm,
fontWeight: theme.fontWeight.semibold,
},
hostText: {
flex: 1,
textAlign: "right",
color: theme.colors.foregroundMuted,
fontSize: theme.fontSize.xs,
},
meterRow: {
justifyContent: "center",
},
actionsRow: {
flexDirection: "row",
alignItems: "center",
justifyContent: "flex-end",
gap: theme.spacing[2],
},
iconButton: {
width: 40,
height: 40,
borderRadius: theme.borderRadius.full,
alignItems: "center",
justifyContent: "center",
backgroundColor: theme.colors.surface0,
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
},
iconButtonMuted: {
backgroundColor: theme.colors.palette.red[500],
borderColor: theme.colors.palette.red[600],
},
iconButtonStop: {
backgroundColor: theme.colors.palette.red[600],
borderColor: theme.colors.palette.red[800],
},
}));

80
packages/app/src/contexts/realtime-context.tsx → packages/app/src/contexts/voice-context.tsx
View File

@@ -7,34 +7,38 @@ import { randomUUID } from "expo-crypto";
const VOICE_CONVERSATION_ID_STORAGE_KEY = "@paseo:voice-conversation-id";
interface RealtimeContextValue {
isRealtimeMode: boolean;
interface VoiceContextValue {
isVoiceMode: boolean;
volume: number;
isMuted: boolean;
isDetecting: boolean;
isSpeaking: boolean;
segmentDuration: number;
startRealtime: (serverId: string) => Promise<void>;
stopRealtime: () => Promise<void>;
startVoice: (serverId: string) => Promise<void>;
stopVoice: () => Promise<void>;
toggleMute: () => void;
activeServerId: string | null;
}
const RealtimeContext = createContext<RealtimeContextValue | null>(null);
const VoiceContext = createContext<VoiceContextValue | null>(null);
export function useRealtime() {
const context = useContext(RealtimeContext);
export function useVoice() {
const context = useContext(VoiceContext);
if (!context) {
throw new Error("useRealtime must be used within RealtimeProvider");
throw new Error("useVoice must be used within VoiceProvider");
}
return context;
}
interface RealtimeProviderProps {
export function useVoiceOptional(): VoiceContextValue | null {
return useContext(VoiceContext);
}
interface VoiceProviderProps {
children: ReactNode;
}
export function RealtimeProvider({ children }: RealtimeProviderProps) {
export function VoiceProvider({ children }: VoiceProviderProps) {
const getSession = useSessionStore((state) => state.getSession);
const [activeServerId, setActiveServerId] = useState<string | null>(null);
const activeSession = useSessionStore(
@@ -49,12 +53,12 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
)
);
const realtimeSessionRef = useRef<SessionState | null>(null);
const [isRealtimeMode, setIsRealtimeMode] = useState(false);
const [isVoiceMode, setIsVoiceMode] = useState(false);
const bargeInPlaybackStopRef = useRef<number | null>(null);
const realtimeAudio = useSpeechmaticsAudio({
onSpeechStart: () => {
console.log("[Realtime] Speech detected");
console.log("[Voice] Speech detected");
// Stop audio playback if playing
const session = realtimeSessionRef.current;
const sessionAudioPlayer = session?.audioPlayer ?? null;
@@ -72,20 +76,20 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
try {
if (sessionClient) {
void sessionClient.abortRequest().catch((error) => {
console.error("[Realtime] Failed to send abort_request:", error);
console.error("[Voice] Failed to send abort_request:", error);
});
}
console.log("[Realtime] Sent abort_request before streaming audio");
console.log("[Voice] Sent abort_request before streaming audio");
} catch (error) {
console.error("[Realtime] Failed to send abort_request:", error);
console.error("[Voice] Failed to send abort_request:", error);
}
},
onSpeechEnd: () => {
console.log("[Realtime] Speech ended");
console.log("[Voice] Speech ended");
},
onAudioSegment: ({ audioData, isLast }) => {
console.log(
"[Realtime] Sending audio segment, length:",
"[Voice] Sending audio segment, length:",
audioData.length,
"isLast:",
isLast
@@ -96,25 +100,25 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
try {
if (session?.client) {
void session.client
.sendRealtimeAudioChunk(
.sendVoiceAudioChunk(
audioData,
"audio/pcm;rate=16000;bits=16",
isLast
)
.catch((error) => {
console.error("[Realtime] Failed to send audio segment:", error);
console.error("[Voice] Failed to send audio segment:", error);
});
}
} catch (error) {
console.error("[Realtime] Failed to send audio segment:", error);
console.error("[Voice] Failed to send audio segment:", error);
}
},
onError: (error) => {
console.error("[Realtime] Audio error:", error);
console.error("[Voice] Audio error:", error);
const session = realtimeSessionRef.current;
if (session?.client) {
// Send error through websocket instead of directly manipulating messages
console.error("[Realtime] Cannot handle error - setMessages not available from SessionState");
console.error("[Voice] Cannot handle error - setMessages not available from SessionState");
}
},
volumeThreshold: 0.3,
@@ -149,7 +153,7 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
}
}, [isPlayingAudio]);
const startRealtime = useCallback(
const startVoice = useCallback(
async (serverId: string) => {
const session = getSession(serverId) ?? null;
if (!session) {
@@ -160,8 +164,8 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
realtimeSessionRef.current = session;
setActiveServerId(serverId);
await realtimeAudio.start();
setIsRealtimeMode(true);
console.log("[Realtime] Mode enabled");
setIsVoiceMode(true);
console.log("[Voice] Mode enabled");
if (session?.client) {
let voiceConversationId =
@@ -175,10 +179,10 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
}
await session.client.setVoiceConversation(true, voiceConversationId);
} else {
console.warn("[Realtime] setRealtimeMode skipped: daemon unavailable");
console.warn("[Voice] setVoiceConversation skipped: daemon unavailable");
}
} catch (error: any) {
console.error("[Realtime] Failed to start:", error);
console.error("[Voice] Failed to start:", error);
setActiveServerId((current) => (current === serverId ? null : current));
throw error;
}
@@ -186,42 +190,42 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
[getSession, realtimeAudio]
);
const stopRealtime = useCallback(async () => {
const stopVoice = useCallback(async () => {
try {
const session = realtimeSessionRef.current;
session?.audioPlayer?.stop();
await realtimeAudio.stop();
setIsRealtimeMode(false);
setIsVoiceMode(false);
setActiveServerId(null);
console.log("[Realtime] Mode disabled");
console.log("[Voice] Mode disabled");
if (session?.client) {
await session.client.setVoiceConversation(false);
} else {
console.warn("[Realtime] setRealtimeMode skipped: daemon unavailable");
console.warn("[Voice] setVoiceConversation skipped: daemon unavailable");
}
} catch (error: any) {
console.error("[Realtime] Failed to stop:", error);
console.error("[Voice] Failed to stop:", error);
throw error;
}
}, [realtimeAudio]);
const value: RealtimeContextValue = {
isRealtimeMode,
const value: VoiceContextValue = {
isVoiceMode,
volume: realtimeAudio.volume,
isMuted: realtimeAudio.isMuted,
isDetecting: realtimeAudio.isDetecting,
isSpeaking: realtimeAudio.isSpeaking,
segmentDuration: realtimeAudio.segmentDuration,
startRealtime,
stopRealtime,
startVoice,
stopVoice,
toggleMute: realtimeAudio.toggleMute,
activeServerId,
};
return (
<RealtimeContext.Provider value={value}>
<VoiceContext.Provider value={value}>
{children}
</RealtimeContext.Provider>
</VoiceContext.Provider>
);
}

2
packages/app/src/hooks/use-audio-player.web.ts
View File

@@ -7,7 +7,7 @@ export interface AudioPlayerOptions {
/**
* Web shim for the native two-way audio player.
* We currently don't support realtime voice playback on web.
* We currently don't support Voice mode playback on web.
*/
export function useAudioPlayer(_options?: AudioPlayerOptions) {
return useMemo(

9
packages/server/agent-prompt.md
View File

@@ -6,6 +6,15 @@
You are a **voice-controlled** assistant. The user speaks to you via phone and hears your responses via TTS.
### Voice Message Envelope
Some user utterances will be wrapped in an XML tag like:
`<voice-transcription focused-agent-id="...">...</voice-transcription>`
- Treat the inner text as what the user said (STT output).
- `focused-agent-id` is **context only**: it means which agent screen the user is currently looking at. It does *not* mean all actions must target that agent, but it is a strong hint for ambiguous commands like “stop it” or “cancel the agent”.
**Critical constraints:**
- User typically codes from their **phone** using voice

4
packages/server/src/client/daemon-client-v2.ts
View File

@@ -1132,12 +1132,12 @@ export class DaemonClientV2 {
this.sendSessionMessage({ type: "set_voice_conversation", enabled, voiceConversationId });
}
async sendRealtimeAudioChunk(
async sendVoiceAudioChunk(
audio: string,
format: string,
isLast: boolean
): Promise<void> {
this.sendSessionMessage({ type: "realtime_audio_chunk", audio, format, isLast });
this.sendSessionMessage({ type: "voice_audio_chunk", audio, format, isLast });
}
startDictationStream(dictationId: string, format: string): Promise<void> {

4
packages/server/src/server/agent/tts-manager.ts
View File

@@ -32,7 +32,7 @@ export class TTSManager {
text: string,
emitMessage: (msg: SessionOutboundMessage) => void,
abortSignal: AbortSignal,
isRealtimeMode: boolean
isVoiceMode: boolean
): Promise<void> {
if (!this.tts) {
throw new Error("TTS not configured");
@@ -117,7 +117,7 @@ export class TTSManager {
isLastChunk: next.done,
audio: chunkBuffer.toString("base64"),
format,
isRealtimeMode,
isVoiceMode,
},
});

2
packages/server/src/server/agent/wait-for-agent-tracker.ts
View File

@@ -4,7 +4,7 @@ export type WaitForAgentCanceler = (agentId: string, reason?: string) => boolean
/**
* Tracks long-running wait_for_agent tool calls so they can be cancelled
* explicitly (e.g., when a realtime barge-in aborts the current turn).
* explicitly (e.g., when a voice barge-in aborts the current turn).
*/
export class WaitForAgentTracker {
private waiters = new Map<string, Set<(reason?: string) => void>>();

6
packages/server/src/server/daemon-client-v2.e2e.test.ts
View File

@@ -544,7 +544,7 @@ describe("daemon client v2 E2E", () => {
);
test(
"realtime mode buffers audio until isLast and emits transcription_result",
"voice mode buffers audio until isLast and emits transcription_result",
async () => {
requireEnv("OPENAI_API_KEY");
@@ -613,7 +613,7 @@ describe("daemon client v2 E2E", () => {
const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16
const firstChunk = pcm16.subarray(0, Math.min(chunkBytes, pcm16.length));
await ctx.client.sendRealtimeAudioChunk(firstChunk.toString("base64"), format, false);
await ctx.client.sendVoiceAudioChunk(firstChunk.toString("base64"), format, false);
await earlyTranscription
.then(() => {
throw new Error("Expected no transcription_result before isLast=true");
@@ -623,7 +623,7 @@ describe("daemon client v2 E2E", () => {
for (let offset = chunkBytes; offset < pcm16.length; offset += chunkBytes) {
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
const isLast = offset + chunkBytes >= pcm16.length;
await ctx.client.sendRealtimeAudioChunk(chunk.toString("base64"), format, isLast);
await ctx.client.sendVoiceAudioChunk(chunk.toString("base64"), format, isLast);
}
const outcome = await Promise.race([

77
packages/server/src/server/session.ts
View File

@@ -260,8 +260,8 @@ export class Session {
private processingPhase: ProcessingPhase = "idle";
private currentStreamPromise: Promise<void> | null = null;
// Realtime mode state
private isRealtimeMode = false;
// Voice mode state
private isVoiceMode = false;
private speechInProgress = false;
private voiceConversationId: string | null = null;
@@ -369,6 +369,28 @@ export class Session {
this.sessionLogger.info("Session created");
}
private escapeXmlText(value: string): string {
return value
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;");
}
private escapeXmlAttribute(value: string): string {
return this.escapeXmlText(value)
.replace(/"/g, "&quot;")
.replace(/'/g, "&apos;");
}
private formatVoiceTranscriptionForLLM(text: string): string {
const trimmed = text.trim();
const focusedAgentId = this.clientActivity?.focusedAgentId ?? null;
const focusedAttr = focusedAgentId
? ` focused-agent-id="${this.escapeXmlAttribute(focusedAgentId)}"`
: "";
return `<voice-transcription${focusedAttr}>${this.escapeXmlText(trimmed)}</voice-transcription>`;
}
/**
* Get the client's current activity state
*/
@@ -735,7 +757,7 @@ export class Session {
await this.handleUserText(msg.text);
break;
case "realtime_audio_chunk":
case "voice_audio_chunk":
await this.handleAudioChunk(msg);
break;
@@ -1214,7 +1236,7 @@ export class Session {
}
/**
* Handle realtime mode toggle
* Handle voice mode toggle
*/
private async handleSetVoiceConversation(
enabled: boolean,
@@ -1226,7 +1248,7 @@ export class Session {
return;
}
this.isRealtimeMode = true;
this.isVoiceMode = true;
this.voiceConversationId = voiceConversationId;
const loaded = await this.voiceConversationStore.load(
@@ -1242,7 +1264,7 @@ export class Session {
return;
}
this.isRealtimeMode = false;
this.isVoiceMode = false;
const idToPersist = this.voiceConversationId;
if (idToPersist) {
try {
@@ -3875,8 +3897,8 @@ export class Session {
// Add to conversation
this.messages.push({ role: "user", content: text });
// Process through LLM (TTS enabled in realtime mode for voice conversations)
this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode);
// Process through LLM (TTS enabled in voice mode for voice conversations)
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode);
await this.currentStreamPromise;
}
@@ -3884,9 +3906,9 @@ export class Session {
* Handle audio chunk for buffering and transcription
*/
private async handleAudioChunk(
msg: Extract<SessionInboundMessage, { type: "realtime_audio_chunk" }>
msg: Extract<SessionInboundMessage, { type: "voice_audio_chunk" }>
): Promise<void> {
await this.handleRealtimeSpeechStart();
await this.handleVoiceSpeechStart();
const chunkBuffer = Buffer.from(msg.audio, "base64");
const chunkFormat = msg.format || "audio/wav";
@@ -3932,19 +3954,19 @@ export class Session {
`Buffered audio chunk (${chunkBuffer.length} bytes, chunks: ${this.audioBuffer.chunks.length}${this.audioBuffer.isPCM ? `, PCM bytes: ${this.audioBuffer.totalPCMBytes}` : ""})`
);
// In realtime mode, only process audio when the user has finished speaking (isLast = true)
// In voice mode, only process audio when the user has finished speaking (isLast = true)
// This prevents partial transcriptions from being sent to the LLM
if (this.isRealtimeMode) {
if (this.isVoiceMode) {
if (!msg.isLast) {
this.sessionLogger.debug("Realtime mode: buffering audio, waiting for speech end");
this.sessionLogger.debug("Voice mode: buffering audio, waiting for speech end");
return;
}
this.sessionLogger.debug("Realtime mode: speech ended, processing complete audio");
this.sessionLogger.debug("Voice mode: speech ended, processing complete audio");
}
// In non-realtime mode, use streaming threshold to process chunks
// In non-voice mode, use streaming threshold to process chunks
const reachedStreamingThreshold =
!this.isRealtimeMode &&
!this.isVoiceMode &&
this.audioBuffer.isPCM &&
this.audioBuffer.totalPCMBytes >= MIN_STREAMING_SEGMENT_BYTES;
@@ -4073,7 +4095,7 @@ export class Session {
const requestId = uuidv4();
const result = await this.sttManager.transcribe(audio, format, {
requestId,
label: this.isRealtimeMode ? "realtime" : "buffered",
label: this.isVoiceMode ? "voice" : "buffered",
});
const transcriptText = result.text.trim();
@@ -4143,12 +4165,17 @@ export class Session {
});
// Add to conversation
this.messages.push({ role: "user", content: result.text });
this.messages.push({
role: "user",
content: this.isVoiceMode
? this.formatVoiceTranscriptionForLLM(result.text)
: result.text,
});
// Set phase to LLM and process (TTS enabled in realtime mode for voice conversations)
// Set phase to LLM and process (TTS enabled in voice mode for voice conversations)
this.clearSpeechInProgress("transcription complete");
this.setPhase("llm");
this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode);
this.currentStreamPromise = this.processWithLLM(this.isVoiceMode);
await this.currentStreamPromise;
this.setPhase("idle");
} catch (error: any) {
@@ -4180,7 +4207,7 @@ export class Session {
if (textBuffer.length > 0) {
// TTS handling (capture mode at generation time for drift protection)
if (enableTTS && !this.speechInProgress) {
const modeAtGeneration = this.isRealtimeMode;
const modeAtGeneration = this.isVoiceMode;
pendingTTS = this.ttsManager.generateAndWaitForPlayback(
textBuffer,
(msg) => this.emit(msg),
@@ -4572,7 +4599,7 @@ export class Session {
/**
* Mark speech detection start and abort any active playback/LLM
*/
private async handleRealtimeSpeechStart(): Promise<void> {
private async handleVoiceSpeechStart(): Promise<void> {
if (this.speechInProgress) {
return;
}
@@ -4582,13 +4609,13 @@ export class Session {
const hadActiveStream = Boolean(this.currentStreamPromise);
this.speechInProgress = true;
this.sessionLogger.debug("Realtime speech chunk detected – aborting playback and LLM");
this.ttsManager.cancelPendingPlaybacks("realtime speech detected");
this.sessionLogger.debug("Voice speech detected – aborting playback and LLM");
this.ttsManager.cancelPendingPlaybacks("voice speech detected");
if (this.pendingAudioSegments.length > 0) {
this.sessionLogger.debug(
{ segmentCount: this.pendingAudioSegments.length },
`Dropping ${this.pendingAudioSegments.length} buffered audio segment(s) due to realtime speech`
`Dropping ${this.pendingAudioSegments.length} buffered audio segment(s) due to voice speech`
);
this.pendingAudioSegments = [];
}

10
packages/server/src/shared/messages.ts
View File

@@ -288,8 +288,8 @@ export const UserTextMessageSchema = z.object({
text: z.string(),
});
export const RealtimeAudioChunkMessageSchema = z.object({
type: z.literal("realtime_audio_chunk"),
export const VoiceAudioChunkMessageSchema = z.object({
type: z.literal("voice_audio_chunk"),
audio: z.string(), // base64 encoded
format: z.string(),
isLast: z.boolean(),
@@ -804,7 +804,7 @@ export const KillTerminalRequestSchema = z.object({
export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
UserTextMessageSchema,
RealtimeAudioChunkMessageSchema,
VoiceAudioChunkMessageSchema,
AbortRequestMessageSchema,
AudioPlayedMessageSchema,
FetchAgentsRequestMessageSchema,
@@ -900,7 +900,7 @@ export const AudioOutputMessageSchema = z.object({
audio: z.string(), // base64 encoded
format: z.string(),
id: z.string(),
isRealtimeMode: z.boolean(), // Mode when audio was generated (for drift protection)
isVoiceMode: z.boolean(), // Mode when audio was generated (for drift protection)
groupId: z.string().optional(), // Logical utterance id
chunkIndex: z.number().int().nonnegative().optional(),
isLastChunk: z.boolean().optional(),
@@ -1643,7 +1643,7 @@ export type ActivityLogPayload = z.infer<typeof ActivityLogPayloadSchema>;
// Type exports for inbound message types
export type UserTextMessage = z.infer<typeof UserTextMessageSchema>;
export type RealtimeAudioChunkMessage = z.infer<typeof RealtimeAudioChunkMessageSchema>;
export type VoiceAudioChunkMessage = z.infer<typeof VoiceAudioChunkMessageSchema>;
export type FetchAgentsRequestMessage = z.infer<typeof FetchAgentsRequestMessageSchema>;
export type FetchAgentRequestMessage = z.infer<typeof FetchAgentRequestMessageSchema>;
export type SendAgentMessageRequest = z.infer<typeof SendAgentMessageRequestSchema>;