From 4bd36c0808d9ae2d86e1d953f044d40788654a19 Mon Sep 17 00:00:00 2001
From: Mohamed Boudra <boudra.moha@gmail.com>
Date: Sat, 15 Nov 2025 19:21:57 +0100
Subject: [PATCH] feat: keep agents running during orchestrator speech

---
 .../app/src/contexts/realtime-context.tsx     | 29 +++++++------------
 plan.md                                       |  3 +-
 2 files changed, 13 insertions(+), 19 deletions(-)

diff --git a/packages/app/src/contexts/realtime-context.tsx b/packages/app/src/contexts/realtime-context.tsx
index 90013e066..d591b6b84 100644
--- a/packages/app/src/contexts/realtime-context.tsx
+++ b/packages/app/src/contexts/realtime-context.tsx
@@ -37,8 +37,6 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
     isPlayingAudio,
     setMessages,
     setVoiceDetectionFlags,
-    cancelAgentRun,
-    focusedAgentId,
   } = useSession();
   const [isRealtimeMode, setIsRealtimeMode] = useState(false);
 
@@ -50,17 +48,7 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
         audioPlayer.stop();
       }
 
-      if (focusedAgentId) {
-        try {
-          cancelAgentRun(focusedAgentId);
-          console.log(
-            `[Realtime] Sent cancel_agent_request for agent ${focusedAgentId} before streaming audio`
-          );
-        } catch (error) {
-          console.error("[Realtime] Failed to cancel focused agent:", error);
-        }
-      }
-      // Abort any in-flight LLM turn before the new speech segment streams
+      // Abort any in-flight orchestrator turn before the new speech segment streams
       try {
         const abortMessage: WSInboundMessage = {
           type: "session",
@@ -77,8 +65,13 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
     onSpeechEnd: () => {
       console.log("[Realtime] Speech ended");
     },
-    onAudioSegment: (base64Audio: string) => {
-      console.log("[Realtime] Sending audio segment, length:", base64Audio.length);
+    onAudioSegment: ({ audioData, isLast }) => {
+      console.log(
+        "[Realtime] Sending audio segment, length:",
+        audioData.length,
+        "isLast:",
+        isLast
+      );
 
       // Send audio segment to server (realtime always goes to orchestrator)
       try {
@@ -86,9 +79,9 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
           type: "session",
           message: {
             type: "realtime_audio_chunk",
-            audio: base64Audio,
-            format: "audio/wav",
-            isLast: true, // Complete segment
+            audio: audioData,
+            format: "audio/pcm;rate=16000;bits=16",
+            isLast,
           },
         });
       } catch (error) {
diff --git a/plan.md b/plan.md
index 78be7dc47..4e643cae0 100644
--- a/plan.md
+++ b/plan.md
@@ -62,7 +62,8 @@
   - Session context now tracks a `focusedAgentId` that each agent screen sets/clears on mount, and the realtime provider looks at that value to send a `cancel_agent_request` before issuing voice aborts so the next spoken prompt doesn't pile onto a running turn. Attempted `npm run typecheck --workspace=@voice-dev/app` but it still fails in the pre-existing stream harness/tests complaining about `parsed*` fields.
 - [x] Extend focused-agent tracking to orchestrator mode so realtime interruptions cancel the agent that’s actually active, even when no agent detail screen is open.
   - Session context now keeps a derived auto-focus pointing at the most recent running agent whenever no explicit agent screen is selected, so realtime speech aborts send `cancel_agent_request` for that agent before streaming audio; agent screens continue to override the focus via `setFocusedAgentId`.
-- [ ] Keep agent runs alive while speaking to the orchestrator: realtime speech should **not** cancel the focused agent by default. Instead, only interrupt when the user explicitly addresses that agent again or issues a cancel command.
+- [x] Keep agent runs alive while speaking to the orchestrator: realtime speech should **not** cancel the focused agent by default. Instead, only interrupt when the user explicitly addresses that agent again or issues a cancel command.
+  - Removed the automatic `cancel_agent_request` that fired on every realtime speech start, so orchestrator dictation no longer tears down whichever agent is in focus; users can still stop runs explicitly via the agent cancel control while the orchestrator abort logic (`abort_request`) remains intact.
 - [x] Harden playback stop/queue clearing so speech detection purges suppressed audio and the server stops emitting TTS while the user is talking.
   - Speech detection now aborts any in-flight TTS generation, clears buffered realtime segments (pending chunks plus partial buffers), and cancels pending playbacks before delegating to the existing abort flow, so no additional audio_output events fire once the user starts talking.
 - [x] Prototype chunked audio input: emit smaller PCM frames with `isLast=false`, update `handleAudioChunk` to buffer and trigger STT once the minimum duration is reached.