From 8202985bad4f2e14baafb8d222ba9596bb7cd6ad Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Sat, 15 Nov 2025 18:35:13 +0100 Subject: [PATCH] Interrupt focused agent before realtime turns --- packages/app/src/app/agent/[id].tsx | 13 ++++++++++++ .../app/src/contexts/realtime-context.tsx | 21 ++++++++++++++++++- packages/app/src/contexts/session-context.tsx | 5 +++++ plan.md | 3 ++- 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/packages/app/src/app/agent/[id].tsx b/packages/app/src/app/agent/[id].tsx index 97e97bdad..6f049f760 100644 --- a/packages/app/src/app/agent/[id].tsx +++ b/packages/app/src/app/agent/[id].tsx @@ -35,6 +35,7 @@ export default function AgentScreen() { respondToPermission, initializeAgent, refreshAgent, + setFocusedAgentId, } = useSession(); const { registerFooterControls, unregisterFooterControls } = useFooterControls(); const { width: windowWidth, height: windowHeight } = useWindowDimensions(); @@ -66,6 +67,18 @@ export default function AgentScreen() { Array.from(pendingPermissions.entries()).filter(([_, perm]) => perm.agentId === id) ); + useEffect(() => { + if (!id) { + setFocusedAgentId(null); + return; + } + + setFocusedAgentId(id); + return () => { + setFocusedAgentId(null); + }; + }, [id, setFocusedAgentId]); + const hasStreamState = id ? agentStreamState.has(id) : false; const initializationState = id ? initializingAgents.get(id) : undefined; const isInitializing = id diff --git a/packages/app/src/contexts/realtime-context.tsx b/packages/app/src/contexts/realtime-context.tsx index f1985ae86..90013e066 100644 --- a/packages/app/src/contexts/realtime-context.tsx +++ b/packages/app/src/contexts/realtime-context.tsx @@ -31,7 +31,15 @@ interface RealtimeProviderProps { } export function RealtimeProvider({ children }: RealtimeProviderProps) { - const { ws, audioPlayer, isPlayingAudio, setMessages, setVoiceDetectionFlags } = useSession(); + const { + ws, + audioPlayer, + isPlayingAudio, + setMessages, + setVoiceDetectionFlags, + cancelAgentRun, + focusedAgentId, + } = useSession(); const [isRealtimeMode, setIsRealtimeMode] = useState(false); const realtimeAudio = useSpeechmaticsAudio({ @@ -41,6 +49,17 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) { if (isPlayingAudio) { audioPlayer.stop(); } + + if (focusedAgentId) { + try { + cancelAgentRun(focusedAgentId); + console.log( + `[Realtime] Sent cancel_agent_request for agent ${focusedAgentId} before streaming audio` + ); + } catch (error) { + console.error("[Realtime] Failed to cancel focused agent:", error); + } + } // Abort any in-flight LLM turn before the new speech segment streams try { const abortMessage: WSInboundMessage = { diff --git a/packages/app/src/contexts/session-context.tsx b/packages/app/src/contexts/session-context.tsx index 8c8adcde6..9a2eda032 100644 --- a/packages/app/src/contexts/session-context.tsx +++ b/packages/app/src/contexts/session-context.tsx @@ -190,6 +190,8 @@ interface SessionContextValue { // Voice detection flags (updated by RealtimeContext) setVoiceDetectionFlags: (isDetecting: boolean, isSpeaking: boolean) => void; + focusedAgentId: string | null; + setFocusedAgentId: (agentId: string | null) => void; // Messages and stream state messages: MessageEntry[]; @@ -260,6 +262,7 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) { }); const [isPlayingAudio, setIsPlayingAudio] = useState(false); + const [focusedAgentId, setFocusedAgentId] = useState(null); const [messages, setMessages] = useState([]); const [currentAssistantMessage, setCurrentAssistantMessage] = useState(""); const [agentStreamState, setAgentStreamState] = useState>(new Map()); @@ -1073,6 +1076,8 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) { isPlayingAudio, setIsPlayingAudio, setVoiceDetectionFlags, + focusedAgentId, + setFocusedAgentId, messages, setMessages, currentAssistantMessage, diff --git a/plan.md b/plan.md index 1e948bbed..2530326f6 100644 --- a/plan.md +++ b/plan.md @@ -58,7 +58,8 @@ - Realtime speech detection now sends a `session/abort_request` as soon as VAD confirms speech, ensuring any in-flight LLM turn is interrupted before the buffered audio is uploaded (implemented in `packages/app/src/contexts/realtime-context.tsx` with logging/error handling). - [x] Abort server-side playback/LLM as soon as a realtime audio chunk arrives; set a speech-in-progress flag that pauses new TTS until the turn completes. - Added a `speechInProgress` guard in `packages/server/src/server/session.ts` that triggers on the first realtime chunk, cancels pending TTS playback via the new `TTSManager.cancelPendingPlaybacks`, and immediately routes through the abort flow before buffering audio; the flag is cleared when transcription finishes so the next assistant reply can synthesize speech. TTS generation now skips while the flag is set, and `npm run typecheck --workspace=@voice-dev/server` still fails in pre-existing cross-workspace `packages/app/src/types/stream.ts` imports (unchanged by this fix). -- [ ] Interrupt the currently focused agent whenever a realtime user turn begins, ensuring the next prompt starts fresh. +- [x] Interrupt the currently focused agent whenever a realtime user turn begins, ensuring the next prompt starts fresh. + - Session context now tracks a `focusedAgentId` that each agent screen sets/clears on mount, and the realtime provider looks at that value to send a `cancel_agent_request` before issuing voice aborts so the next spoken prompt doesn't pile onto a running turn. Attempted `npm run typecheck --workspace=@voice-dev/app` but it still fails in the pre-existing stream harness/tests complaining about `parsed*` fields. - [ ] Harden playback stop/queue clearing so speech detection purges suppressed audio and the server stops emitting TTS while the user is talking. - [ ] Prototype chunked audio input: emit smaller PCM frames with `isLast=false`, update `handleAudioChunk` to buffer and trigger STT once the minimum duration is reached. - [ ] Investigate streaming STT/server-side VAD (Whisper or equivalent) and document requirements for production use.