diff --git a/packages/app/src/app/index.tsx b/packages/app/src/app/index.tsx index 023cecef2..82f4e4e1f 100644 --- a/packages/app/src/app/index.tsx +++ b/packages/app/src/app/index.tsx @@ -53,6 +53,7 @@ import { ArrowUp, Square, AudioLines, + MicOff, } from "lucide-react-native"; import type { ActivityLogPayload, @@ -231,8 +232,14 @@ export default function VoiceAssistantScreen() { // Realtime mode state (defined early so we can use it in audioRecorder) const [isRealtimeMode, setIsRealtimeMode] = useState(false); + const isRealtimeModeRef = useRef(isRealtimeMode); const pulseAnim = useRef(new Animated.Value(1)).current; + // Keep ref in sync with state + useEffect(() => { + isRealtimeModeRef.current = isRealtimeMode; + }, [isRealtimeMode]); + const insets = useSafeAreaInsets(); const audioRecorder = useAudioRecorder(); const audioPlayer = useAudioPlayer(); @@ -293,7 +300,7 @@ export default function VoiceAssistantScreen() { ]); }, volumeThreshold: 0.3, - silenceDuration: 1000, + silenceDuration: 2000, speechConfirmationDuration: 300, detectionGracePeriod: 200, }); @@ -452,6 +459,11 @@ export default function VoiceAssistantScreen() { if (message.type !== "activity_log") return; const data = message.payload; + // Filter out transcription activity logs + if (data.type === "system" && data.content.includes("Transcribing")) { + return; + } + // Handle tool calls if (data.type === "tool_call" && data.metadata) { const { @@ -593,6 +605,42 @@ export default function VoiceAssistantScreen() { if (message.type !== "audio_output") return; const data = message.payload; + const currentIsRealtimeMode = isRealtimeModeRef.current; + + // Drift protection: Don't play audio generated in different mode + if (data.isRealtimeMode !== currentIsRealtimeMode) { + console.log( + `[App] Skipping audio playback due to mode drift (generated in ${data.isRealtimeMode ? "realtime" : "normal"} mode, currently in ${currentIsRealtimeMode ? "realtime" : "normal"} mode)` + ); + + // Still send confirmation to prevent server from waiting + const confirmMessage: WSInboundMessage = { + type: "session", + message: { + type: "audio_played", + id: data.id, + }, + }; + ws.send(confirmMessage); + return; + } + + // Additional check: Don't play if NOT in realtime mode (shouldn't happen with server-side fix, but defense in depth) + if (!currentIsRealtimeMode) { + console.log("[App] Skipping audio playback - not in realtime mode"); + + // Still send confirmation + const confirmMessage: WSInboundMessage = { + type: "session", + message: { + type: "audio_played", + id: data.id, + }, + }; + ws.send(confirmMessage); + return; + } + try { setIsPlayingAudio(true); @@ -669,18 +717,7 @@ export default function VoiceAssistantScreen() { // Conversation loaded handler const unsubConversationLoaded = ws.on("conversation_loaded", (message) => { if (message.type !== "conversation_loaded") return; - const { conversationId, messageCount } = message.payload; - - setMessages((prev) => [ - ...prev, - { - type: "activity", - id: generateMessageId(), - timestamp: Date.now(), - activityType: "success", - message: `Loaded conversation with ${messageCount} messages`, - }, - ]); + // Don't show message in UI }); // Artifact handler @@ -734,32 +771,6 @@ export default function VoiceAssistantScreen() { }; }, [ws, audioPlayer]); - // Connection status handler - useEffect(() => { - if (ws.isConnected) { - setMessages((prev) => [ - ...prev, - { - type: "activity", - id: generateMessageId(), - timestamp: Date.now(), - activityType: "success", - message: "WebSocket connected", - }, - ]); - } else if (messages.length > 0) { - setMessages((prev) => [ - ...prev, - { - type: "activity", - id: generateMessageId(), - timestamp: Date.now(), - activityType: "error", - message: "WebSocket disconnected", - }, - ]); - } - }, [ws.isConnected]); // Voice button handler async function handleVoicePress() { @@ -778,16 +789,6 @@ export default function VoiceAssistantScreen() { ); setIsProcessingAudio(true); - setMessages((prev) => [ - ...prev, - { - type: "activity", - id: generateMessageId(), - timestamp: Date.now(), - activityType: "info", - message: "Sending audio to server...", - }, - ]); // Convert to base64 const arrayBuffer = await audioBlob.arrayBuffer(); @@ -974,6 +975,16 @@ export default function VoiceAssistantScreen() { setIsRealtimeMode(true); console.log("[App] Realtime mode enabled"); + // Notify server of mode change + const modeMessage: WSInboundMessage = { + type: "session", + message: { + type: "set_realtime_mode", + enabled: true, + }, + }; + ws.send(modeMessage); + setMessages((prev) => [ ...prev, { @@ -1004,6 +1015,16 @@ export default function VoiceAssistantScreen() { setIsRealtimeMode(false); console.log("[App] Realtime mode disabled"); + // Notify server of mode change + const modeMessage: WSInboundMessage = { + type: "session", + message: { + type: "set_realtime_mode", + enabled: false, + }, + }; + ws.send(modeMessage); + setMessages((prev) => [ ...prev, { @@ -1204,13 +1225,49 @@ export default function VoiceAssistantScreen() { ]} > {isRealtimeMode ? ( - // Realtime mode - show volume meter - - - + // Realtime mode - show volume meter and mute button + + + + {/* Debug timer */} + {(realtimeAudio.isDetecting || realtimeAudio.isSpeaking) && ( + + {(realtimeAudio.segmentDuration / 1000).toFixed(1)}s + + )} + + + {/* Mute button */} + realtimeAudio.toggleMute()} + style={[ + styles.realtimeMuteButton, + realtimeAudio.isMuted && styles.realtimeMuteButtonActive, + ]} + > + + + {/* Close button */} + + + + + ) : ( // Normal mode - show text input and buttons @@ -1420,4 +1477,39 @@ const styles = StyleSheet.create((theme) => ({ buttonDisabled: { opacity: theme.opacity[50], }, + realtimeModeButtons: { + flexDirection: "row", + alignItems: "center", + justifyContent: "center", + gap: theme.spacing[3], + paddingTop: theme.spacing[4], + }, + realtimeMuteButton: { + width: 48, + height: 48, + borderRadius: theme.borderRadius.full, + alignItems: "center", + justifyContent: "center", + backgroundColor: theme.colors.muted, + borderWidth: theme.borderWidth[2], + borderColor: theme.colors.border, + }, + realtimeMuteButtonActive: { + backgroundColor: theme.colors.palette.red[500], + borderColor: theme.colors.palette.red[600], + }, + realtimeCloseButton: { + width: 48, + height: 48, + borderRadius: theme.borderRadius.full, + alignItems: "center", + justifyContent: "center", + backgroundColor: theme.colors.palette.red[600], + }, + debugTimer: { + marginTop: theme.spacing[2], + color: theme.colors.mutedForeground, + fontSize: theme.fontSize.sm, + fontFamily: "monospace", + }, })); diff --git a/packages/app/src/components/volume-meter.tsx b/packages/app/src/components/volume-meter.tsx index 19edae17f..b2481f580 100644 --- a/packages/app/src/components/volume-meter.tsx +++ b/packages/app/src/components/volume-meter.tsx @@ -13,15 +13,18 @@ import { StyleSheet, useUnistyles } from "react-native-unistyles"; interface VolumeMeterProps { volume: number; + isMuted?: boolean; + isDetecting?: boolean; + isSpeaking?: boolean; } -export function VolumeMeter({ volume }: VolumeMeterProps) { +export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSpeaking = false }: VolumeMeterProps) { const { theme } = useUnistyles(); // Base dimensions const LINE_SPACING = 8; - const MAX_HEIGHT = 80; - const MIN_HEIGHT = 4; + const MAX_HEIGHT = 50; + const MIN_HEIGHT = 20; // Shared values for each line's height const line1Height = useSharedValue(MIN_HEIGHT); @@ -35,6 +38,14 @@ export function VolumeMeter({ volume }: VolumeMeterProps) { // Start idle animations with different phases useEffect(() => { + if (isMuted) { + // When muted, set pulse to 1 (no animation) + line1Pulse.value = 1; + line2Pulse.value = 1; + line3Pulse.value = 1; + return; + } + // Line 1 - fastest pulse line1Pulse.value = withRepeat( withSequence( @@ -66,10 +77,18 @@ export function VolumeMeter({ volume }: VolumeMeterProps) { -1, false ); - }, []); + }, [isMuted]); // Update heights based on volume with different responsiveness per line useEffect(() => { + if (isMuted) { + // When muted, keep lines at minimum height without animation + line1Height.value = MIN_HEIGHT; + line2Height.value = MIN_HEIGHT; + line3Height.value = MIN_HEIGHT; + return; + } + if (volume > 0.01) { // Active volume - animate heights based on volume // Line 1 - most responsive, follows volume closely @@ -107,23 +126,41 @@ export function VolumeMeter({ volume }: VolumeMeterProps) { stiffness: 150, }); } - }, [volume]); + }, [volume, isMuted]); // Animated styles for each line - const line1Style = useAnimatedStyle(() => ({ - height: line1Height.value * (volume > 0.01 ? 1 : line1Pulse.value), - opacity: 0.8 + (volume * 0.2), - })); + const line1Style = useAnimatedStyle(() => { + const isActive = isDetecting || isSpeaking; + const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5; + const volumeBoost = isMuted ? 0 : volume * 0.3; + return { + height: line1Height.value * (isMuted || volume > 0.01 ? 1 : line1Pulse.value), + opacity: baseOpacity + volumeBoost, + }; + }); - const line2Style = useAnimatedStyle(() => ({ - height: line2Height.value * (volume > 0.01 ? 1 : line2Pulse.value), - opacity: 0.7 + (volume * 0.3), - })); + const line2Style = useAnimatedStyle(() => { + const isActive = isDetecting || isSpeaking; + const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5; + const volumeBoost = isMuted ? 0 : volume * 0.3; + return { + height: line2Height.value * (isMuted || volume > 0.01 ? 1 : line2Pulse.value), + opacity: baseOpacity + volumeBoost, + }; + }); - const line3Style = useAnimatedStyle(() => ({ - height: line3Height.value * (volume > 0.01 ? 1 : line3Pulse.value), - opacity: 0.6 + (volume * 0.4), - })); + const line3Style = useAnimatedStyle(() => { + const isActive = isDetecting || isSpeaking; + const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5; + const volumeBoost = isMuted ? 0 : volume * 0.3; + return { + height: line3Height.value * (isMuted || volume > 0.01 ? 1 : line3Pulse.value), + opacity: baseOpacity + volumeBoost, + }; + }); + + const lineColor = "#FFFFFF"; + const lineWidth = 8; return ( @@ -131,8 +168,8 @@ export function VolumeMeter({ volume }: VolumeMeterProps) { style={[ styles.line, { - width: 4, - backgroundColor: theme.colors.palette.blue[400], + width: lineWidth, + backgroundColor: lineColor, }, line1Style, ]} @@ -142,8 +179,8 @@ export function VolumeMeter({ volume }: VolumeMeterProps) { style={[ styles.line, { - width: 6, - backgroundColor: theme.colors.palette.blue[500], + width: lineWidth, + backgroundColor: lineColor, }, line2Style, ]} @@ -153,8 +190,8 @@ export function VolumeMeter({ volume }: VolumeMeterProps) { style={[ styles.line, { - width: 4, - backgroundColor: theme.colors.palette.blue[400], + width: lineWidth, + backgroundColor: lineColor, }, line3Style, ]} diff --git a/packages/app/src/hooks/use-speechmatics-audio.ts b/packages/app/src/hooks/use-speechmatics-audio.ts index f92b24e02..52037b919 100644 --- a/packages/app/src/hooks/use-speechmatics-audio.ts +++ b/packages/app/src/hooks/use-speechmatics-audio.ts @@ -22,10 +22,13 @@ export interface SpeechmaticsAudioConfig { export interface SpeechmaticsAudio { start: () => Promise; stop: () => Promise; + toggleMute: () => void; isActive: boolean; isSpeaking: boolean; isDetecting: boolean; + isMuted: boolean; volume: number; + segmentDuration: number; } /** @@ -131,6 +134,8 @@ export function useSpeechmaticsAudio( const [isDetecting, setIsDetecting] = useState(false); const [audioInitialized, setAudioInitialized] = useState(false); const [volume, setVolume] = useState(0); + const [isMuted, setIsMuted] = useState(false); + const [segmentDuration, setSegmentDuration] = useState(0); const audioBufferRef = useRef([]); const silenceStartRef = useRef(null); @@ -164,12 +169,28 @@ export function useSpeechmaticsAudio( }; }, []); + // Update segment duration timer + useEffect(() => { + if (!isDetecting && !isSpeaking) { + setSegmentDuration(0); + return; + } + + const startTime = speechDetectionStartRef.current || Date.now(); + const interval = setInterval(() => { + const elapsed = Date.now() - startTime; + setSegmentDuration(elapsed); + }, 100); + + return () => clearInterval(interval); + }, [isDetecting, isSpeaking]); + // Listen to microphone data useExpoTwoWayAudioEventListener( "onMicrophoneData", useCallback( (event) => { - if (!isActive) return; + if (!isActive || isMuted) return; const pcmData: Uint8Array = event.data; @@ -179,7 +200,7 @@ export function useSpeechmaticsAudio( audioBufferRef.current.push(pcmData); } }, - [isActive] + [isActive, isMuted] ) ); @@ -192,6 +213,9 @@ export function useSpeechmaticsAudio( const volumeLevel: number = event.data; setVolume(volumeLevel); + + if (isMuted) return; + const speechDetected = volumeLevel > VOLUME_THRESHOLD; // console.log('[SpeechmaticsAudio] Volume:', volumeLevel.toFixed(6), 'Threshold:', VOLUME_THRESHOLD); @@ -285,7 +309,7 @@ export function useSpeechmaticsAudio( } } }, - [isActive, VOLUME_THRESHOLD, SILENCE_DURATION_MS, SPEECH_CONFIRMATION_MS, DETECTION_GRACE_PERIOD_MS, config] + [isActive, isMuted, VOLUME_THRESHOLD, SILENCE_DURATION_MS, SPEECH_CONFIRMATION_MS, DETECTION_GRACE_PERIOD_MS, config] ) ); @@ -333,10 +357,32 @@ export function useSpeechmaticsAudio( setIsSpeaking(false); setIsDetecting(false); setVolume(0); + setIsMuted(false); console.log("[SpeechmaticsAudio] Audio capture stopped"); } + function toggleMute(): void { + setIsMuted((prev) => { + const newMuted = !prev; + console.log("[SpeechmaticsAudio] Mute toggled:", newMuted); + + if (newMuted) { + // Clear any ongoing speech detection/speaking state + audioBufferRef.current = []; + isSpeakingRef.current = false; + speechConfirmedRef.current = false; + speechDetectionStartRef.current = null; + detectionSilenceStartRef.current = null; + silenceStartRef.current = null; + setIsSpeaking(false); + setIsDetecting(false); + } + + return newMuted; + }); + } + // Cleanup on unmount useEffect(() => { return () => { @@ -349,9 +395,12 @@ export function useSpeechmaticsAudio( return { start, stop, + toggleMute, isActive, isSpeaking, isDetecting, + isMuted, volume, + segmentDuration, }; } diff --git a/packages/server/src/server/agent/tts-manager.ts b/packages/server/src/server/agent/tts-manager.ts index 5de37d25b..e186691e8 100644 --- a/packages/server/src/server/agent/tts-manager.ts +++ b/packages/server/src/server/agent/tts-manager.ts @@ -26,7 +26,8 @@ export class TTSManager { public async generateAndWaitForPlayback( text: string, emitMessage: (msg: SessionOutboundMessage) => void, - abortSignal: AbortSignal + abortSignal: AbortSignal, + isRealtimeMode: boolean ): Promise { if (abortSignal.aborted) { console.log( @@ -78,13 +79,14 @@ export class TTSManager { }); } - // Emit audio output message + // Emit audio output message (include mode for drift protection) emitMessage({ type: "audio_output", payload: { id: audioId, audio: audio.toString("base64"), format, + isRealtimeMode, }, }); diff --git a/packages/server/src/server/messages.ts b/packages/server/src/server/messages.ts index 45315db96..43b062a4b 100644 --- a/packages/server/src/server/messages.ts +++ b/packages/server/src/server/messages.ts @@ -7,6 +7,7 @@ import { z } from "zod"; export const UserTextMessageSchema = z.object({ type: z.literal("user_text"), text: z.string(), + disableTTS: z.boolean().optional(), }); export const AudioChunkMessageSchema = z.object({ @@ -39,6 +40,11 @@ export const DeleteConversationRequestMessageSchema = z.object({ conversationId: z.string(), }); +export const SetRealtimeModeMessageSchema = z.object({ + type: z.literal("set_realtime_mode"), + enabled: z.boolean(), +}); + export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ UserTextMessageSchema, AudioChunkMessageSchema, @@ -47,6 +53,7 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ LoadConversationRequestMessageSchema, ListConversationsRequestMessageSchema, DeleteConversationRequestMessageSchema, + SetRealtimeModeMessageSchema, ]); export type SessionInboundMessage = z.infer; @@ -88,6 +95,7 @@ export const AudioOutputMessageSchema = z.object({ audio: z.string(), // base64 encoded format: z.string(), id: z.string(), + isRealtimeMode: z.boolean(), // Mode when audio was generated (for drift protection) }), }); @@ -164,6 +172,12 @@ export const AgentStatusMessageSchema = z.object({ type: z.literal("claude"), sessionId: z.string().optional(), error: z.string().optional(), + currentModeId: z.string().optional(), + availableModes: z.array(z.object({ + id: z.string(), + name: z.string(), + description: z.string().nullable().optional(), + })).optional(), }), }), }); diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 881169fcf..e74b3796f 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -54,6 +54,9 @@ export class Session { private processingPhase: ProcessingPhase = "idle"; private currentStreamPromise: Promise | null = null; + // Realtime mode state + private isRealtimeMode = false; + // Audio buffering for interruption handling private pendingAudioSegments: Array<{ audio: Buffer; format: string }> = []; private bufferTimeout: NodeJS.Timeout | null = null; @@ -165,12 +168,12 @@ export class Session { info: { id: info.id, status: info.status, - createdAt: info.createdAt.toISOString(), + createdAt: info.createdAt, type: info.type, - sessionId: info.sessionId, - error: info.error, - currentModeId: info.currentModeId, - availableModes: info.availableModes, + sessionId: info.sessionId ?? undefined, + error: info.error ?? undefined, + currentModeId: info.currentModeId ?? undefined, + availableModes: info.availableModes ?? undefined, }, }, }); @@ -300,6 +303,10 @@ export class Session { case "delete_conversation_request": await this.handleDeleteConversation(msg.conversationId); break; + + case "set_realtime_mode": + this.handleSetRealtimeMode(msg.enabled); + break; } } catch (error: any) { console.error( @@ -400,6 +407,16 @@ export class Session { } } + /** + * Handle realtime mode toggle + */ + private handleSetRealtimeMode(enabled: boolean): void { + this.isRealtimeMode = enabled; + console.log( + `[Session ${this.clientId}] Realtime mode ${enabled ? "enabled" : "disabled"}` + ); + } + /** * Send current session state (live agents and commands) to client */ @@ -471,8 +488,8 @@ export class Session { // Add to conversation this.messages.push({ role: "user", content: text }); - // Process through LLM (TTS enabled for text input) - this.currentStreamPromise = this.processWithLLM(true); + // Process through LLM (TTS enabled in realtime mode for voice conversations) + this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode); await this.currentStreamPromise; } @@ -618,9 +635,9 @@ export class Session { // Add to conversation this.messages.push({ role: "user", content: result.text }); - // Set phase to LLM and process + // Set phase to LLM and process (TTS enabled in realtime mode for voice conversations) this.setPhase("llm"); - this.currentStreamPromise = this.processWithLLM(true); // Enable TTS for voice input + this.currentStreamPromise = this.processWithLLM(this.isRealtimeMode); await this.currentStreamPromise; this.setPhase("idle"); } catch (error: any) { @@ -648,12 +665,14 @@ export class Session { const flushTextBuffer = () => { if (textBuffer.length > 0) { - // TTS handling + // TTS handling (capture mode at generation time for drift protection) if (enableTTS) { + const modeAtGeneration = this.isRealtimeMode; pendingTTS = this.ttsManager.generateAndWaitForPlayback( textBuffer, (msg) => this.emit(msg), - this.abortController.signal + this.abortController.signal, + modeAtGeneration ); }