From 24a245bd1707f8d496e916d07ea74d29f30d941f Mon Sep 17 00:00:00 2001
From: Mohamed Boudra <boudra.moha@gmail.com>
Date: Sat, 15 Nov 2025 19:18:39 +0100
Subject: [PATCH] auto-focus orchestrator agent during realtime

---
 packages/app/src/contexts/session-context.tsx | 36 ++++++++++++++++++-
 plan.md                                       |  3 ++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/packages/app/src/contexts/session-context.tsx b/packages/app/src/contexts/session-context.tsx
index 2871601d5..1b7b4496b 100644
--- a/packages/app/src/contexts/session-context.tsx
+++ b/packages/app/src/contexts/session-context.tsx
@@ -262,7 +262,8 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
   });
 
   const [isPlayingAudio, setIsPlayingAudio] = useState(false);
-  const [focusedAgentId, setFocusedAgentId] = useState<string | null>(null);
+  const [focusedAgentOverride, setFocusedAgentOverride] = useState<string | null>(null);
+  const [orchestratorFocusedAgentId, setOrchestratorFocusedAgentId] = useState<string | null>(null);
   const [messages, setMessages] = useState<MessageEntry[]>([]);
   const [currentAssistantMessage, setCurrentAssistantMessage] = useState("");
   const [agentStreamState, setAgentStreamState] = useState<Map<string, StreamItem[]>>(new Map());
@@ -275,6 +276,39 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
   const [fileExplorer, setFileExplorer] = useState<Map<string, AgentFileExplorerState>>(new Map());
   const activeAudioGroupsRef = useRef<Set<string>>(new Set());
 
+  const focusedAgentId = focusedAgentOverride ?? orchestratorFocusedAgentId;
+  const setFocusedAgentId = useCallback((agentId: string | null) => {
+    setFocusedAgentOverride(agentId);
+  }, []);
+
+  useEffect(() => {
+    if (focusedAgentOverride) {
+      if (orchestratorFocusedAgentId !== null) {
+        setOrchestratorFocusedAgentId(null);
+      }
+      return;
+    }
+
+    let latestRunningAgentId: string | null = null;
+    let latestActivityTimestamp = -Infinity;
+
+    for (const agent of agents.values()) {
+      if (agent.status !== "running") {
+        continue;
+      }
+
+      const activityTimestamp = agent.lastActivityAt?.getTime() ?? agent.updatedAt.getTime();
+      if (activityTimestamp > latestActivityTimestamp) {
+        latestActivityTimestamp = activityTimestamp;
+        latestRunningAgentId = agent.id;
+      }
+    }
+
+    if (latestRunningAgentId !== orchestratorFocusedAgentId) {
+      setOrchestratorFocusedAgentId(latestRunningAgentId);
+    }
+  }, [agents, focusedAgentOverride, orchestratorFocusedAgentId]);
+
   const updateExplorerState = useCallback(
     (agentId: string, updater: (state: AgentFileExplorerState) => AgentFileExplorerState) => {
       setFileExplorer((prev) => {
diff --git a/plan.md b/plan.md
index d8db5ef3b..78be7dc47 100644
--- a/plan.md
+++ b/plan.md
@@ -60,6 +60,9 @@
   - Added a `speechInProgress` guard in `packages/server/src/server/session.ts` that triggers on the first realtime chunk, cancels pending TTS playback via the new `TTSManager.cancelPendingPlaybacks`, and immediately routes through the abort flow before buffering audio; the flag is cleared when transcription finishes so the next assistant reply can synthesize speech. TTS generation now skips while the flag is set, and `npm run typecheck --workspace=@voice-dev/server` still fails in pre-existing cross-workspace `packages/app/src/types/stream.ts` imports (unchanged by this fix).
 - [x] Interrupt the currently focused agent whenever a realtime user turn begins, ensuring the next prompt starts fresh.
   - Session context now tracks a `focusedAgentId` that each agent screen sets/clears on mount, and the realtime provider looks at that value to send a `cancel_agent_request` before issuing voice aborts so the next spoken prompt doesn't pile onto a running turn. Attempted `npm run typecheck --workspace=@voice-dev/app` but it still fails in the pre-existing stream harness/tests complaining about `parsed*` fields.
+- [x] Extend focused-agent tracking to orchestrator mode so realtime interruptions cancel the agent that’s actually active, even when no agent detail screen is open.
+  - Session context now keeps a derived auto-focus pointing at the most recent running agent whenever no explicit agent screen is selected, so realtime speech aborts send `cancel_agent_request` for that agent before streaming audio; agent screens continue to override the focus via `setFocusedAgentId`.
+- [ ] Keep agent runs alive while speaking to the orchestrator: realtime speech should **not** cancel the focused agent by default. Instead, only interrupt when the user explicitly addresses that agent again or issues a cancel command.
 - [x] Harden playback stop/queue clearing so speech detection purges suppressed audio and the server stops emitting TTS while the user is talking.
   - Speech detection now aborts any in-flight TTS generation, clears buffered realtime segments (pending chunks plus partial buffers), and cancels pending playbacks before delegating to the existing abort flow, so no additional audio_output events fire once the user starts talking.
 - [x] Prototype chunked audio input: emit smaller PCM frames with `isLast=false`, update `handleAudioChunk` to buffer and trigger STT once the minimum duration is reached.