fix(server): wake queued delivery at turn and permission checkpoints

A pending follow-up must not hide the permission needed to finish the current turn. Wake ready batches on recipient state changes, let completion waits report permission checkpoints, and preserve admission order and session-mode boundaries when busy dispatches retry.
This commit is contained in:
Mohamed Boudra
2026-11-15 12:13:08 +01:00
parent 28428ebacf
commit f72aba59ea
11 changed files with 261 additions and 17 deletions
+3 -2
View File
@@ -150,12 +150,13 @@ Users can also detach an existing subagent from the subagents track. Detach is d
`notifyOnFinish` defaults to `true` for agent-scoped creation and prompt follow-ups because most delegated work needs to report back to the creating agent. Set it to `false` for explicit reporting or no completion callback.
Agent-originated messages and automatic notifications share a recipient-daemon delivery module. A queued receipt acknowledges admission before provider dispatch. Idle recipients retain the two-second quiet window. While a recipient has a running or starting turn (including compaction), messages stay queued across quiet windows and are delivered together after that turn settles, including failure or cancellation. Delivery checks again every two seconds. Agent callers using `steer`, including the default, wait for the turn to end; only explicit `interrupt` replaces a running turn. Human prompts dispatch directly with their existing behavior. Conflicting session-mode choices retain separate ordered dispatches.
Agent-originated messages and automatic notifications share a recipient-daemon delivery module. A queued receipt acknowledges admission before provider dispatch. Idle recipients retain the two-second quiet window. While a recipient has a running or starting turn (including compaction), messages stay queued across quiet windows and are delivered together after that turn settles, including failure or cancellation. A ready batch is released when the daemon observes the recipient settle; the two-second timer also retries it. Agent callers using `steer`, including the default, wait for the turn to end; only explicit `interrupt` replaces a running turn. Human prompts dispatch directly with their existing behavior. Conflicting session-mode choices retain separate ordered dispatches.
The pending message window is in memory. Clean shutdown flushes it before closing agents; an abrupt
crash can lose queued prompts. A durable send receipt remains pending until actual dispatch
acknowledgment, so a lost window never becomes a completed receipt. Completion waits cover queued
admission within their original timeout and cancellation scope.
admission within their original timeout and cancellation scope, and return permission checkpoints
while messages are waiting.
Finish notification intent has a separate daemon-owned lifetime. The finish-subscriptions module
persists caller, child, host, and ownership policy through an injectable store; its JSON adapter uses
@@ -14486,6 +14486,13 @@ test.each([
let tick = () => {};
const delivery = createAgentMessageDelivery({
isRunning: (id) => manager.hasInFlightRun(id),
subscribe: (notify) =>
manager.subscribe(
(event) => {
if (event.type === "agent_state") notify(event.agent.id);
},
{ replayState: false },
),
schedule: (_delay, callback) => {
tick = callback;
return () => {};
@@ -14529,8 +14536,6 @@ test.each([
turnId,
...(ending === "turn_failed" ? { error: "test failure" } : {}),
} as AgentStreamEvent);
await vi.waitFor(() => expect(manager.hasInFlightRun(agent.id)).toBe(false));
tick();
await receipt.dispatched;
await manager.waitForAgentRunStart(agent.id);
expect(session.startCount).toBe(2);
@@ -1117,6 +1117,10 @@ export class AgentManager {
return new Date(nextMs).toISOString();
}
hasPendingPermissions(agentId: string): boolean {
return (this.agents.get(agentId)?.pendingPermissions.size ?? 0) > 0;
}
hasInFlightRun(agentId: string): boolean {
const agent = this.agents.get(agentId);
if (!agent) {
@@ -454,3 +454,151 @@ test("shutdown flushes a dispatch that discovered a concurrent turn during prepa
await expect(receipt.dispatched).resolves.toBe("turn_started");
expect(modes).toEqual([true, false]);
});
test("busy retries retain admission order across conflicting session modes", async () => {
const clock = createClock();
let busy = true;
const sent: Array<{ ids: readonly string[]; mode?: string }> = [];
const delivery = createAgentMessageDelivery({
schedule: clock.schedule,
dispatch: async (request) => {
if (busy) throw new AgentTurnBusyError("concurrent turn");
sent.push({ ids: request.messageIds, mode: request.sessionMode });
return "turn_started";
},
onError: (error) => {
throw error;
},
});
const message = {
recipientId: "parent",
source: { kind: "agent-message" as const, agentId: "child" },
prompt: "message",
};
const a = delivery.enqueue({ ...message, messageId: "A", sessionMode: "plan" });
const b = delivery.enqueue({ ...message, messageId: "B", sessionMode: "default" });
clock.advance(2000);
await new Promise<void>((resolve) => setImmediate(resolve));
busy = false;
clock.advance(2000);
await Promise.all([a.dispatched, b.dispatched]);
expect(sent).toEqual([
{ ids: ["A"], mode: "plan" },
{ ids: ["B"], mode: "default" },
]);
await delivery.close();
});
test("interrupts retain separate mode choices while the recipient is busy", async () => {
const clock = createClock();
const sent: Array<{ ids: readonly string[]; mode?: string }> = [];
const delivery = createAgentMessageDelivery({
schedule: clock.schedule,
isRunning: () => true,
dispatch: async (request) => {
sent.push({ ids: request.messageIds, mode: request.sessionMode });
return "turn_started";
},
onError: (error) => {
throw error;
},
});
const message = {
recipientId: "parent",
source: { kind: "agent-message" as const, agentId: "child" },
prompt: "message",
activeTurnBehavior: "interrupt" as const,
};
const a = delivery.enqueue({ ...message, messageId: "A", sessionMode: "plan" });
const b = delivery.enqueue({ ...message, messageId: "B", sessionMode: "default" });
clock.advance(2000);
await Promise.all([a.dispatched, b.dispatched]);
expect(sent).toEqual([
{ ids: ["A"], mode: "plan" },
{ ids: ["B"], mode: "default" },
]);
await delivery.close();
});
test("a settled-turn event releases a ready batch without waiting for the next timer sample", async () => {
const clock = createClock();
let running = true;
let settled: (id: string) => void = () => {};
let subscribed = true;
const sent: string[] = [];
const delivery = createAgentMessageDelivery({
schedule: clock.schedule,
isRunning: () => running,
subscribe: (listener) => {
settled = listener;
return () => {
subscribed = false;
};
},
dispatch: async (request) => {
sent.push(request.recipientId);
return "turn_started";
},
onError: (error) => {
throw error;
},
});
delivery.enqueue({
recipientId: "parent",
prompt: "report",
source: { kind: "agent-message", agentId: "child" },
});
clock.advance(2000);
expect(sent).toEqual([]);
running = false;
settled("parent");
await new Promise<void>((resolve) => setImmediate(resolve));
expect(sent).toEqual(["parent"]);
running = true;
await delivery.close();
expect(subscribed).toBe(false);
});
test.each([false, true])(
"completion waits report permission checkpoints without draining messages (initial: %s)",
async (initial) => {
const clock = createClock();
let permission = initial;
let changed: (id: string) => void = () => {};
let reported = false;
let sent = 0;
const delivery = createAgentMessageDelivery({
schedule: clock.schedule,
isRunning: () => true,
hasPendingPermission: () => permission,
subscribe: (listener) => {
changed = listener;
return () => {};
},
dispatch: async () => {
sent++;
return "turn_started";
},
onError: (error) => {
throw error;
},
});
delivery.enqueue({
recipientId: "child",
prompt: "follow up",
source: { kind: "agent-message", agentId: "parent" },
});
const wait = delivery.waitForDispatch("child", new AbortController().signal).then(() => {
reported = true;
return undefined;
});
permission = true;
changed("child");
await new Promise<void>((resolve) => setImmediate(resolve));
expect(reported).toBe(true);
expect(sent).toBe(0);
expect(delivery.hasPending("child")).toBe(true);
await wait;
await delivery.close();
},
);
@@ -21,6 +21,7 @@ interface Receipt {
dispatched: Promise<PromptDispatchDisposition>;
}
interface Admitted extends Message {
sequence: number;
receipt: Receipt;
resolve(disposition: PromptDispatchDisposition): void;
reject(error: unknown): void;
@@ -38,6 +39,9 @@ interface DispatchRequest {
interface Dependencies {
/** Includes admitted starts and provider turns, including compaction. */
isRunning?: (recipientId: string) => boolean;
hasPendingPermission?: (recipientId: string) => boolean;
/** Notify when recipient state changes; admission still checks whether it is idle. */
subscribe?: (listener: (recipientId: string) => void) => () => void;
dispatch(request: DispatchRequest): Promise<PromptDispatchDisposition>;
onError(error: unknown, request: DispatchRequest): void;
/** Return a cancellation function. Production uses the daemon's event loop. */
@@ -49,7 +53,7 @@ export interface AgentMessageDelivery {
hasPending(recipientId: string): boolean;
/** Stop admission, flush pending messages, and await provider dispatch acknowledgments. */
close(): Promise<void>;
/** Existing completion waits include agent messages admitted before the wait. */
/** Completion waits include admitted messages, but return at permission checkpoints. */
waitForDispatch(recipientId: string, signal: AbortSignal): Promise<void>;
}
@@ -61,9 +65,10 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes
const timer = setTimeout(callback, delayMs);
return () => clearTimeout(timer);
});
const pending = new Map<string, { messages: Admitted[]; cancel(): void }>();
const pending = new Map<string, { messages: Admitted[]; ready: boolean; cancel(): void }>();
const dispatching = new Map<string, Promise<void>>();
const receipts = new Map<string, Set<Receipt>>();
let sequence = 0;
let closed = false;
let closing: Promise<void> | undefined;
@@ -112,9 +117,14 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes
return undefined;
} catch (error) {
if (error instanceof AgentTurnBusyError && !closed) {
const entry = pending.get(recipientId) ?? { messages: [], cancel: () => {} };
const entry = pending.get(recipientId) ?? {
messages: [],
ready: false,
cancel: () => {},
};
entry.cancel();
entry.messages.unshift(...messages);
entry.messages.push(...messages);
entry.messages.sort((a, b) => a.sequence - b.sequence);
entry.cancel = schedule(2000, () => flush(recipientId));
pending.set(recipientId, entry);
return;
@@ -135,6 +145,7 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes
function flush(recipientId: string): void {
const entry = pending.get(recipientId);
if (!entry) return;
entry.ready = true;
entry.cancel();
if (!closed && dependencies.isRunning?.(recipientId)) {
const interrupts = entry.messages.filter(
@@ -143,7 +154,7 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes
entry.messages = entry.messages.filter(
(message) => message.activeTurnBehavior !== "interrupt",
);
if (interrupts.length) dispatch(recipientId, interrupts);
if (interrupts.length) dispatchGroups(recipientId, interrupts);
if (entry.messages.length) {
entry.cancel = schedule(2000, () => flush(recipientId));
} else {
@@ -152,9 +163,13 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes
return;
}
pending.delete(recipientId);
dispatchGroups(recipientId, entry.messages);
}
function dispatchGroups(recipientId: string, messages: Admitted[]): void {
let group: Admitted[] = [];
let mode: string | undefined;
for (const message of entry.messages) {
for (const message of messages) {
// Explicitly conflicting send options require separate, ordered dispatches.
if (
group.length &&
@@ -171,6 +186,11 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes
if (group.length) dispatch(recipientId, group);
}
const unsubscribe = dependencies.subscribe?.((recipientId) => {
if (pending.get(recipientId)?.ready && !dependencies.isRunning?.(recipientId))
flush(recipientId);
});
return {
hasPending: (recipientId) => pending.has(recipientId) || dispatching.has(recipientId),
enqueue(message) {
@@ -196,9 +216,14 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes
if (!owned.size) receipts.delete(message.recipientId);
})
.catch(() => undefined);
const entry = pending.get(message.recipientId) ?? { messages: [], cancel: () => {} };
const entry = pending.get(message.recipientId) ?? {
messages: [],
ready: false,
cancel: () => {},
};
entry.cancel();
entry.messages.push({ ...message, receipt, resolve, reject });
entry.ready = false;
entry.messages.push({ ...message, receipt, resolve, reject, sequence: sequence++ });
entry.cancel = schedule(2000, () => flush(message.recipientId));
pending.set(message.recipientId, entry);
return receipt;
@@ -206,6 +231,16 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes
async waitForDispatch(recipientId, signal) {
signal.throwIfAborted();
const admitted = Array.from(receipts.get(recipientId) ?? [], (receipt) => receipt.dispatched);
let unsubscribePermission: (() => void) | undefined;
const permission = new Promise<void>((resolve) => {
const check = () => {
if (dependencies.hasPendingPermission?.(recipientId)) resolve();
};
unsubscribePermission = dependencies.subscribe?.((agentId) => {
if (agentId === recipientId) check();
});
check();
});
let aborted!: () => void;
const cancellation = new Promise<never>((_resolve, reject) => {
aborted = () => reject(new DOMException("Wait aborted", "AbortError"));
@@ -215,14 +250,17 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes
await Promise.race([
Promise.all([dispatching.get(recipientId), ...admitted]),
cancellation,
permission,
]);
} finally {
signal.removeEventListener("abort", aborted);
unsubscribePermission?.();
}
},
close() {
if (closing) return closing;
closed = true;
unsubscribe?.();
for (const recipientId of pending.keys()) flush(recipientId);
// Each failure is reported and rejects its receipts; shutdown still drains other recipients.
closing = Promise.allSettled(dispatching.values()).then(() => undefined);
@@ -498,3 +498,26 @@ test("finish recovery protects callers before durable intent has loaded", async
expect(recovered.hasPendingForCaller("parent")).toBe(true);
expect(recovered.hasPendingForCaller("unrelated")).toBe(false);
});
test("an undispatched follow-up does not block the running child's permission notification", async () => {
const state = await scenario();
await state.service.register(intent);
await until(() => state.observation.waits.length === 1);
await state.service.register(intent, true);
const result = outcome("running");
result.status = "permission";
result.final!.pendingPermissions = [
{
id: "approval",
provider: "codex",
name: "Bash",
kind: "tool",
input: { command: "run tests" },
} as NonNullable<typeof result.final>["pendingPermissions"][number],
];
state.observation.waits[0]!.result.resolve(result);
await until(() => state.delivery.prompts.length === 1);
expect(state.delivery.prompts[0]).toContain("needs permission");
expect(state.delivery.prompts[0]).toContain("approval");
expect(await state.store.list()).toHaveLength(2);
});
@@ -267,7 +267,9 @@ export function createFinishSubscriptions(deps: Dependencies): FinishSubscriptio
continue;
}
if (signal.aborted) return;
await decisions(group);
// A queued follow-up cannot start until the current permission is answered.
// Its pending registration must not hold the running turn's checkpoint.
if (pending.status !== "permission" || watch.pending) await decisions(group);
if (signal.aborted) return;
if (await suppressed(record, pending)) {
await remove(watch);
@@ -275,6 +275,7 @@ function buildAgentManagerSpies() {
effectiveOptionsFor: (_provider: string, own?: Record<string, unknown>) => own ?? {},
reloadStalePluginSession: async () => {},
hasInFlightRun: vi.fn().mockReturnValue(false),
hasPendingPermissions: () => false,
steerOrReplaceActiveTurn: async () => ({ status: "inactive" as const }),
tryRunOutOfBand: vi.fn().mockReturnValue(false),
subscribe: vi.fn().mockReturnValue(() => {}),
@@ -624,6 +625,12 @@ class FakeBrowserToolsBroker {
}
class BoundaryAgentManagerFake {
subscribe() {
return () => {};
}
hasPendingPermissions() {
return false;
}
public getPluginTools(): [] {
return [];
}
@@ -3969,7 +3976,7 @@ describe("send_agent_prompt MCP tool", () => {
const response = await tool.handler(parsed.data as Record<string, unknown>);
expect(response.structuredContent).toMatchObject({ deliveryStatus: "queued" });
expect(spies.agentManager.subscribe).not.toHaveBeenCalled();
expect(serverSubscriptionRecords.get(server)!.size).toBe(1);
await serverDeliveries
.get(server)!
.waitForDispatch("child-agent", new AbortController().signal);
@@ -4010,7 +4017,7 @@ describe("send_agent_prompt MCP tool", () => {
await tool.handler(parsed.data as Record<string, unknown>);
expect(spies.agentManager.subscribe).not.toHaveBeenCalled();
expect(serverSubscriptionRecords.get(server)!.size).toBe(0);
expect(spies.agentManager.waitForAgentEvent).not.toHaveBeenCalled();
});
@@ -4053,7 +4060,7 @@ describe("send_agent_prompt MCP tool", () => {
notifyOnFinish: false,
});
expect(spies.agentManager.subscribe).not.toHaveBeenCalled();
expect(serverSubscriptionRecords.get(server)!.size).toBe(0);
expect(spies.agentManager.waitForAgentEvent).not.toHaveBeenCalled();
});
@@ -95,7 +95,7 @@ test("an agent-sourced steer leaves the question pending and answerable", async
await client.sendAgentMessage(recipient.id, "Peer status check", {
sourceAgentId: sender.id,
});
// The public wait includes admitted message delivery before reporting permission status.
// Permission checkpoints are reported while the peer message waits for turn end.
const waiting = await client.waitForFinish(recipient.id);
expect(waiting.status).toBe("permission");
expect(waiting.final?.pendingPermissions).toMatchObject([
@@ -15,6 +15,14 @@ export function createTestAgentMessages(params: {
}) {
return createAgentMessageDelivery({
isRunning: (agentId) => params.agentManager.hasInFlightRun(agentId),
hasPendingPermission: (agentId) => params.agentManager.hasPendingPermissions(agentId),
subscribe: (notify) =>
params.agentManager.subscribe(
(event) => {
if (event.type === "agent_state") notify(event.agent.id);
},
{ replayState: false },
),
schedule: (_delay, callback) => {
const timer = setTimeout(callback, 0);
return () => clearTimeout(timer);
@@ -729,6 +729,14 @@ export class VoiceAssistantWebSocketServer {
this.agentStorage = agentStorage;
this.agentMessages = createAgentMessageDelivery({
isRunning: (agentId) => agentManager.hasInFlightRun(agentId),
hasPendingPermission: (agentId) => agentManager.hasPendingPermissions(agentId),
subscribe: (notify) =>
agentManager.subscribe(
(event) => {
if (event.type === "agent_state") notify(event.agent.id);
},
{ replayState: false },
),
dispatch: async (batch) => {
const { disposition } = await sendPreparedPromptToAgent({
agentManager,