From f72aba59ea53ef2bfb575f40e4eb90130d7262b8 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Sun, 15 Nov 2026 12:13:08 +0100 Subject: [PATCH] fix(server): wake queued delivery at turn and permission checkpoints A pending follow-up must not hide the permission needed to finish the current turn. Wake ready batches on recipient state changes, let completion waits report permission checkpoints, and preserve admission order and session-mode boundaries when busy dispatches retry. --- docs/agent-lifecycle.md | 5 +- .../src/server/agent/agent-manager.test.ts | 9 +- .../server/src/server/agent/agent-manager.ts | 4 + .../agent/agent-messages/delivery.test.ts | 148 ++++++++++++++++++ .../server/agent/agent-messages/delivery.ts | 54 ++++++- .../finish-subscriptions.test.ts | 23 +++ .../agent/finish-subscriptions/index.ts | 4 +- .../src/server/agent/mcp-server.test.ts | 13 +- .../peer-message-permissions.test.ts | 2 +- .../src/server/test-utils/agent-messages.ts | 8 + .../server/src/server/websocket-server.ts | 8 + 11 files changed, 261 insertions(+), 17 deletions(-) diff --git a/docs/agent-lifecycle.md b/docs/agent-lifecycle.md index 6ea0270d2f..5ccce59d9b 100644 --- a/docs/agent-lifecycle.md +++ b/docs/agent-lifecycle.md @@ -150,12 +150,13 @@ Users can also detach an existing subagent from the subagents track. Detach is d `notifyOnFinish` defaults to `true` for agent-scoped creation and prompt follow-ups because most delegated work needs to report back to the creating agent. Set it to `false` for explicit reporting or no completion callback. -Agent-originated messages and automatic notifications share a recipient-daemon delivery module. A queued receipt acknowledges admission before provider dispatch. Idle recipients retain the two-second quiet window. While a recipient has a running or starting turn (including compaction), messages stay queued across quiet windows and are delivered together after that turn settles, including failure or cancellation. Delivery checks again every two seconds. Agent callers using `steer`, including the default, wait for the turn to end; only explicit `interrupt` replaces a running turn. Human prompts dispatch directly with their existing behavior. Conflicting session-mode choices retain separate ordered dispatches. +Agent-originated messages and automatic notifications share a recipient-daemon delivery module. A queued receipt acknowledges admission before provider dispatch. Idle recipients retain the two-second quiet window. While a recipient has a running or starting turn (including compaction), messages stay queued across quiet windows and are delivered together after that turn settles, including failure or cancellation. A ready batch is released when the daemon observes the recipient settle; the two-second timer also retries it. Agent callers using `steer`, including the default, wait for the turn to end; only explicit `interrupt` replaces a running turn. Human prompts dispatch directly with their existing behavior. Conflicting session-mode choices retain separate ordered dispatches. The pending message window is in memory. Clean shutdown flushes it before closing agents; an abrupt crash can lose queued prompts. A durable send receipt remains pending until actual dispatch acknowledgment, so a lost window never becomes a completed receipt. Completion waits cover queued -admission within their original timeout and cancellation scope. +admission within their original timeout and cancellation scope, and return permission checkpoints +while messages are waiting. Finish notification intent has a separate daemon-owned lifetime. The finish-subscriptions module persists caller, child, host, and ownership policy through an injectable store; its JSON adapter uses diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index a492c3f770..f8f052cd2b 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -14486,6 +14486,13 @@ test.each([ let tick = () => {}; const delivery = createAgentMessageDelivery({ isRunning: (id) => manager.hasInFlightRun(id), + subscribe: (notify) => + manager.subscribe( + (event) => { + if (event.type === "agent_state") notify(event.agent.id); + }, + { replayState: false }, + ), schedule: (_delay, callback) => { tick = callback; return () => {}; @@ -14529,8 +14536,6 @@ test.each([ turnId, ...(ending === "turn_failed" ? { error: "test failure" } : {}), } as AgentStreamEvent); - await vi.waitFor(() => expect(manager.hasInFlightRun(agent.id)).toBe(false)); - tick(); await receipt.dispatched; await manager.waitForAgentRunStart(agent.id); expect(session.startCount).toBe(2); diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 0ccb209e0f..3db43f0119 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -1117,6 +1117,10 @@ export class AgentManager { return new Date(nextMs).toISOString(); } + hasPendingPermissions(agentId: string): boolean { + return (this.agents.get(agentId)?.pendingPermissions.size ?? 0) > 0; + } + hasInFlightRun(agentId: string): boolean { const agent = this.agents.get(agentId); if (!agent) { diff --git a/packages/server/src/server/agent/agent-messages/delivery.test.ts b/packages/server/src/server/agent/agent-messages/delivery.test.ts index fb10e46489..bc68340f92 100644 --- a/packages/server/src/server/agent/agent-messages/delivery.test.ts +++ b/packages/server/src/server/agent/agent-messages/delivery.test.ts @@ -454,3 +454,151 @@ test("shutdown flushes a dispatch that discovered a concurrent turn during prepa await expect(receipt.dispatched).resolves.toBe("turn_started"); expect(modes).toEqual([true, false]); }); + +test("busy retries retain admission order across conflicting session modes", async () => { + const clock = createClock(); + let busy = true; + const sent: Array<{ ids: readonly string[]; mode?: string }> = []; + const delivery = createAgentMessageDelivery({ + schedule: clock.schedule, + dispatch: async (request) => { + if (busy) throw new AgentTurnBusyError("concurrent turn"); + sent.push({ ids: request.messageIds, mode: request.sessionMode }); + return "turn_started"; + }, + onError: (error) => { + throw error; + }, + }); + const message = { + recipientId: "parent", + source: { kind: "agent-message" as const, agentId: "child" }, + prompt: "message", + }; + const a = delivery.enqueue({ ...message, messageId: "A", sessionMode: "plan" }); + const b = delivery.enqueue({ ...message, messageId: "B", sessionMode: "default" }); + clock.advance(2000); + await new Promise((resolve) => setImmediate(resolve)); + busy = false; + clock.advance(2000); + await Promise.all([a.dispatched, b.dispatched]); + expect(sent).toEqual([ + { ids: ["A"], mode: "plan" }, + { ids: ["B"], mode: "default" }, + ]); + await delivery.close(); +}); + +test("interrupts retain separate mode choices while the recipient is busy", async () => { + const clock = createClock(); + const sent: Array<{ ids: readonly string[]; mode?: string }> = []; + const delivery = createAgentMessageDelivery({ + schedule: clock.schedule, + isRunning: () => true, + dispatch: async (request) => { + sent.push({ ids: request.messageIds, mode: request.sessionMode }); + return "turn_started"; + }, + onError: (error) => { + throw error; + }, + }); + const message = { + recipientId: "parent", + source: { kind: "agent-message" as const, agentId: "child" }, + prompt: "message", + activeTurnBehavior: "interrupt" as const, + }; + const a = delivery.enqueue({ ...message, messageId: "A", sessionMode: "plan" }); + const b = delivery.enqueue({ ...message, messageId: "B", sessionMode: "default" }); + clock.advance(2000); + await Promise.all([a.dispatched, b.dispatched]); + expect(sent).toEqual([ + { ids: ["A"], mode: "plan" }, + { ids: ["B"], mode: "default" }, + ]); + await delivery.close(); +}); + +test("a settled-turn event releases a ready batch without waiting for the next timer sample", async () => { + const clock = createClock(); + let running = true; + let settled: (id: string) => void = () => {}; + let subscribed = true; + const sent: string[] = []; + const delivery = createAgentMessageDelivery({ + schedule: clock.schedule, + isRunning: () => running, + subscribe: (listener) => { + settled = listener; + return () => { + subscribed = false; + }; + }, + dispatch: async (request) => { + sent.push(request.recipientId); + return "turn_started"; + }, + onError: (error) => { + throw error; + }, + }); + delivery.enqueue({ + recipientId: "parent", + prompt: "report", + source: { kind: "agent-message", agentId: "child" }, + }); + clock.advance(2000); + expect(sent).toEqual([]); + running = false; + settled("parent"); + await new Promise((resolve) => setImmediate(resolve)); + expect(sent).toEqual(["parent"]); + running = true; + await delivery.close(); + expect(subscribed).toBe(false); +}); + +test.each([false, true])( + "completion waits report permission checkpoints without draining messages (initial: %s)", + async (initial) => { + const clock = createClock(); + let permission = initial; + let changed: (id: string) => void = () => {}; + let reported = false; + let sent = 0; + const delivery = createAgentMessageDelivery({ + schedule: clock.schedule, + isRunning: () => true, + hasPendingPermission: () => permission, + subscribe: (listener) => { + changed = listener; + return () => {}; + }, + dispatch: async () => { + sent++; + return "turn_started"; + }, + onError: (error) => { + throw error; + }, + }); + delivery.enqueue({ + recipientId: "child", + prompt: "follow up", + source: { kind: "agent-message", agentId: "parent" }, + }); + const wait = delivery.waitForDispatch("child", new AbortController().signal).then(() => { + reported = true; + return undefined; + }); + permission = true; + changed("child"); + await new Promise((resolve) => setImmediate(resolve)); + expect(reported).toBe(true); + expect(sent).toBe(0); + expect(delivery.hasPending("child")).toBe(true); + await wait; + await delivery.close(); + }, +); diff --git a/packages/server/src/server/agent/agent-messages/delivery.ts b/packages/server/src/server/agent/agent-messages/delivery.ts index 74befe8090..ece01ec3c7 100644 --- a/packages/server/src/server/agent/agent-messages/delivery.ts +++ b/packages/server/src/server/agent/agent-messages/delivery.ts @@ -21,6 +21,7 @@ interface Receipt { dispatched: Promise; } interface Admitted extends Message { + sequence: number; receipt: Receipt; resolve(disposition: PromptDispatchDisposition): void; reject(error: unknown): void; @@ -38,6 +39,9 @@ interface DispatchRequest { interface Dependencies { /** Includes admitted starts and provider turns, including compaction. */ isRunning?: (recipientId: string) => boolean; + hasPendingPermission?: (recipientId: string) => boolean; + /** Notify when recipient state changes; admission still checks whether it is idle. */ + subscribe?: (listener: (recipientId: string) => void) => () => void; dispatch(request: DispatchRequest): Promise; onError(error: unknown, request: DispatchRequest): void; /** Return a cancellation function. Production uses the daemon's event loop. */ @@ -49,7 +53,7 @@ export interface AgentMessageDelivery { hasPending(recipientId: string): boolean; /** Stop admission, flush pending messages, and await provider dispatch acknowledgments. */ close(): Promise; - /** Existing completion waits include agent messages admitted before the wait. */ + /** Completion waits include admitted messages, but return at permission checkpoints. */ waitForDispatch(recipientId: string, signal: AbortSignal): Promise; } @@ -61,9 +65,10 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes const timer = setTimeout(callback, delayMs); return () => clearTimeout(timer); }); - const pending = new Map(); + const pending = new Map(); const dispatching = new Map>(); const receipts = new Map>(); + let sequence = 0; let closed = false; let closing: Promise | undefined; @@ -112,9 +117,14 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes return undefined; } catch (error) { if (error instanceof AgentTurnBusyError && !closed) { - const entry = pending.get(recipientId) ?? { messages: [], cancel: () => {} }; + const entry = pending.get(recipientId) ?? { + messages: [], + ready: false, + cancel: () => {}, + }; entry.cancel(); - entry.messages.unshift(...messages); + entry.messages.push(...messages); + entry.messages.sort((a, b) => a.sequence - b.sequence); entry.cancel = schedule(2000, () => flush(recipientId)); pending.set(recipientId, entry); return; @@ -135,6 +145,7 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes function flush(recipientId: string): void { const entry = pending.get(recipientId); if (!entry) return; + entry.ready = true; entry.cancel(); if (!closed && dependencies.isRunning?.(recipientId)) { const interrupts = entry.messages.filter( @@ -143,7 +154,7 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes entry.messages = entry.messages.filter( (message) => message.activeTurnBehavior !== "interrupt", ); - if (interrupts.length) dispatch(recipientId, interrupts); + if (interrupts.length) dispatchGroups(recipientId, interrupts); if (entry.messages.length) { entry.cancel = schedule(2000, () => flush(recipientId)); } else { @@ -152,9 +163,13 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes return; } pending.delete(recipientId); + dispatchGroups(recipientId, entry.messages); + } + + function dispatchGroups(recipientId: string, messages: Admitted[]): void { let group: Admitted[] = []; let mode: string | undefined; - for (const message of entry.messages) { + for (const message of messages) { // Explicitly conflicting send options require separate, ordered dispatches. if ( group.length && @@ -171,6 +186,11 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes if (group.length) dispatch(recipientId, group); } + const unsubscribe = dependencies.subscribe?.((recipientId) => { + if (pending.get(recipientId)?.ready && !dependencies.isRunning?.(recipientId)) + flush(recipientId); + }); + return { hasPending: (recipientId) => pending.has(recipientId) || dispatching.has(recipientId), enqueue(message) { @@ -196,9 +216,14 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes if (!owned.size) receipts.delete(message.recipientId); }) .catch(() => undefined); - const entry = pending.get(message.recipientId) ?? { messages: [], cancel: () => {} }; + const entry = pending.get(message.recipientId) ?? { + messages: [], + ready: false, + cancel: () => {}, + }; entry.cancel(); - entry.messages.push({ ...message, receipt, resolve, reject }); + entry.ready = false; + entry.messages.push({ ...message, receipt, resolve, reject, sequence: sequence++ }); entry.cancel = schedule(2000, () => flush(message.recipientId)); pending.set(message.recipientId, entry); return receipt; @@ -206,6 +231,16 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes async waitForDispatch(recipientId, signal) { signal.throwIfAborted(); const admitted = Array.from(receipts.get(recipientId) ?? [], (receipt) => receipt.dispatched); + let unsubscribePermission: (() => void) | undefined; + const permission = new Promise((resolve) => { + const check = () => { + if (dependencies.hasPendingPermission?.(recipientId)) resolve(); + }; + unsubscribePermission = dependencies.subscribe?.((agentId) => { + if (agentId === recipientId) check(); + }); + check(); + }); let aborted!: () => void; const cancellation = new Promise((_resolve, reject) => { aborted = () => reject(new DOMException("Wait aborted", "AbortError")); @@ -215,14 +250,17 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes await Promise.race([ Promise.all([dispatching.get(recipientId), ...admitted]), cancellation, + permission, ]); } finally { signal.removeEventListener("abort", aborted); + unsubscribePermission?.(); } }, close() { if (closing) return closing; closed = true; + unsubscribe?.(); for (const recipientId of pending.keys()) flush(recipientId); // Each failure is reported and rejects its receipts; shutdown still drains other recipients. closing = Promise.allSettled(dispatching.values()).then(() => undefined); diff --git a/packages/server/src/server/agent/finish-subscriptions/finish-subscriptions.test.ts b/packages/server/src/server/agent/finish-subscriptions/finish-subscriptions.test.ts index 9e18f87948..acd44998b5 100644 --- a/packages/server/src/server/agent/finish-subscriptions/finish-subscriptions.test.ts +++ b/packages/server/src/server/agent/finish-subscriptions/finish-subscriptions.test.ts @@ -498,3 +498,26 @@ test("finish recovery protects callers before durable intent has loaded", async expect(recovered.hasPendingForCaller("parent")).toBe(true); expect(recovered.hasPendingForCaller("unrelated")).toBe(false); }); + +test("an undispatched follow-up does not block the running child's permission notification", async () => { + const state = await scenario(); + await state.service.register(intent); + await until(() => state.observation.waits.length === 1); + await state.service.register(intent, true); + const result = outcome("running"); + result.status = "permission"; + result.final!.pendingPermissions = [ + { + id: "approval", + provider: "codex", + name: "Bash", + kind: "tool", + input: { command: "run tests" }, + } as NonNullable["pendingPermissions"][number], + ]; + state.observation.waits[0]!.result.resolve(result); + await until(() => state.delivery.prompts.length === 1); + expect(state.delivery.prompts[0]).toContain("needs permission"); + expect(state.delivery.prompts[0]).toContain("approval"); + expect(await state.store.list()).toHaveLength(2); +}); diff --git a/packages/server/src/server/agent/finish-subscriptions/index.ts b/packages/server/src/server/agent/finish-subscriptions/index.ts index 22324f0ecc..226ed5af51 100644 --- a/packages/server/src/server/agent/finish-subscriptions/index.ts +++ b/packages/server/src/server/agent/finish-subscriptions/index.ts @@ -267,7 +267,9 @@ export function createFinishSubscriptions(deps: Dependencies): FinishSubscriptio continue; } if (signal.aborted) return; - await decisions(group); + // A queued follow-up cannot start until the current permission is answered. + // Its pending registration must not hold the running turn's checkpoint. + if (pending.status !== "permission" || watch.pending) await decisions(group); if (signal.aborted) return; if (await suppressed(record, pending)) { await remove(watch); diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index add4f74177..199e779031 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -275,6 +275,7 @@ function buildAgentManagerSpies() { effectiveOptionsFor: (_provider: string, own?: Record) => own ?? {}, reloadStalePluginSession: async () => {}, hasInFlightRun: vi.fn().mockReturnValue(false), + hasPendingPermissions: () => false, steerOrReplaceActiveTurn: async () => ({ status: "inactive" as const }), tryRunOutOfBand: vi.fn().mockReturnValue(false), subscribe: vi.fn().mockReturnValue(() => {}), @@ -624,6 +625,12 @@ class FakeBrowserToolsBroker { } class BoundaryAgentManagerFake { + subscribe() { + return () => {}; + } + hasPendingPermissions() { + return false; + } public getPluginTools(): [] { return []; } @@ -3969,7 +3976,7 @@ describe("send_agent_prompt MCP tool", () => { const response = await tool.handler(parsed.data as Record); expect(response.structuredContent).toMatchObject({ deliveryStatus: "queued" }); - expect(spies.agentManager.subscribe).not.toHaveBeenCalled(); + expect(serverSubscriptionRecords.get(server)!.size).toBe(1); await serverDeliveries .get(server)! .waitForDispatch("child-agent", new AbortController().signal); @@ -4010,7 +4017,7 @@ describe("send_agent_prompt MCP tool", () => { await tool.handler(parsed.data as Record); - expect(spies.agentManager.subscribe).not.toHaveBeenCalled(); + expect(serverSubscriptionRecords.get(server)!.size).toBe(0); expect(spies.agentManager.waitForAgentEvent).not.toHaveBeenCalled(); }); @@ -4053,7 +4060,7 @@ describe("send_agent_prompt MCP tool", () => { notifyOnFinish: false, }); - expect(spies.agentManager.subscribe).not.toHaveBeenCalled(); + expect(serverSubscriptionRecords.get(server)!.size).toBe(0); expect(spies.agentManager.waitForAgentEvent).not.toHaveBeenCalled(); }); diff --git a/packages/server/src/server/daemon-e2e/peer-message-permissions.test.ts b/packages/server/src/server/daemon-e2e/peer-message-permissions.test.ts index bc2803a65e..a8cc2c2379 100644 --- a/packages/server/src/server/daemon-e2e/peer-message-permissions.test.ts +++ b/packages/server/src/server/daemon-e2e/peer-message-permissions.test.ts @@ -95,7 +95,7 @@ test("an agent-sourced steer leaves the question pending and answerable", async await client.sendAgentMessage(recipient.id, "Peer status check", { sourceAgentId: sender.id, }); - // The public wait includes admitted message delivery before reporting permission status. + // Permission checkpoints are reported while the peer message waits for turn end. const waiting = await client.waitForFinish(recipient.id); expect(waiting.status).toBe("permission"); expect(waiting.final?.pendingPermissions).toMatchObject([ diff --git a/packages/server/src/server/test-utils/agent-messages.ts b/packages/server/src/server/test-utils/agent-messages.ts index cace3f0660..e102ce97b3 100644 --- a/packages/server/src/server/test-utils/agent-messages.ts +++ b/packages/server/src/server/test-utils/agent-messages.ts @@ -15,6 +15,14 @@ export function createTestAgentMessages(params: { }) { return createAgentMessageDelivery({ isRunning: (agentId) => params.agentManager.hasInFlightRun(agentId), + hasPendingPermission: (agentId) => params.agentManager.hasPendingPermissions(agentId), + subscribe: (notify) => + params.agentManager.subscribe( + (event) => { + if (event.type === "agent_state") notify(event.agent.id); + }, + { replayState: false }, + ), schedule: (_delay, callback) => { const timer = setTimeout(callback, 0); return () => clearTimeout(timer); diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 000e4dffa1..8685bb83b0 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -729,6 +729,14 @@ export class VoiceAssistantWebSocketServer { this.agentStorage = agentStorage; this.agentMessages = createAgentMessageDelivery({ isRunning: (agentId) => agentManager.hasInFlightRun(agentId), + hasPendingPermission: (agentId) => agentManager.hasPendingPermissions(agentId), + subscribe: (notify) => + agentManager.subscribe( + (event) => { + if (event.type === "agent_state") notify(event.agent.id); + }, + { replayState: false }, + ), dispatch: async (batch) => { const { disposition } = await sendPreparedPromptToAgent({ agentManager,