diff --git a/packages/app/src/contexts/realtime-context.tsx b/packages/app/src/contexts/realtime-context.tsx index c05f8ad84..f1985ae86 100644 --- a/packages/app/src/contexts/realtime-context.tsx +++ b/packages/app/src/contexts/realtime-context.tsx @@ -41,6 +41,19 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) { if (isPlayingAudio) { audioPlayer.stop(); } + // Abort any in-flight LLM turn before the new speech segment streams + try { + const abortMessage: WSInboundMessage = { + type: "session", + message: { + type: "abort_request", + }, + }; + ws.send(abortMessage); + console.log("[Realtime] Sent abort_request before streaming audio"); + } catch (error) { + console.error("[Realtime] Failed to send abort_request:", error); + } }, onSpeechEnd: () => { console.log("[Realtime] Speech ended"); diff --git a/plan.md b/plan.md index 2fc4043a1..1b3b8dac4 100644 --- a/plan.md +++ b/plan.md @@ -54,7 +54,8 @@ ## Voice Interruptions & Streaming -- [ ] Wire interrupt signal on speech start so VAD immediately issues `abort_request` to the server before any audio segment is sent. +- [x] Wire interrupt signal on speech start so VAD immediately issues `abort_request` to the server before any audio segment is sent. + - Realtime speech detection now sends a `session/abort_request` as soon as VAD confirms speech, ensuring any in-flight LLM turn is interrupted before the buffered audio is uploaded (implemented in `packages/app/src/contexts/realtime-context.tsx` with logging/error handling). - [ ] Abort server-side playback/LLM as soon as a realtime audio chunk arrives; set a speech-in-progress flag that pauses new TTS until the turn completes. - [ ] Interrupt the currently focused agent whenever a realtime user turn begins, ensuring the next prompt starts fresh. - [ ] Harden playback stop/queue clearing so speech detection purges suppressed audio and the server stops emitting TTS while the user is talking.