From 82edec3bccd2b4d60fa0c84da7a4aafe9b430eee Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Sun, 15 Nov 2026 13:12:40 +0100 Subject: [PATCH] fix(server): keep later mode groups queued until turn end Dispatching every mode group at once prepared later modes against the preceding turn. Keep later groups pending and reject busy sends before changing mode, leaving mode validation outside run startup. --- .../src/server/agent/agent-manager.test.ts | 148 +++++++++--------- .../server/src/server/agent/agent-manager.ts | 6 +- .../agent/agent-messages/delivery.test.ts | 24 ++- .../server/agent/agent-messages/delivery.ts | 38 +++-- .../server/src/server/agent/agent-prompt.ts | 12 +- .../src/server/agent/agent-sdk-types.ts | 2 - 6 files changed, 124 insertions(+), 106 deletions(-) diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index f080feab44..a056153484 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -27,7 +27,7 @@ import type { ProviderOptions } from "@getpaseo/protocol/agent-types"; import { projectTimelineRows } from "./timeline-projection.js"; import { getOpenAgentTabLabel, PARENT_AGENT_ID_LABEL } from "@getpaseo/protocol/agent-labels"; import { createAgentMessageDelivery } from "./agent-messages/index.js"; -import { startAgentRun, sendPreparedPromptToAgent, AgentTurnBusyError } from "./agent-prompt.js"; +import { AgentTurnBusyError, sendPreparedPromptToAgent, startAgentRun } from "./agent-prompt.js"; import { StaleProviderSessionError } from "./stale-provider-session-error.js"; import { ensureAgentLoaded, ensureUnarchivedAgentLoaded } from "./agent-loading.js"; import type { StoredAgentRecord } from "./agent-storage.js"; @@ -14617,73 +14617,54 @@ test.each(["turn_completed", "turn_failed", "turn_canceled"] as const)( }, ); -test.each([false, true])( - "an idle-only prompt cannot change another turn's mode (starts during preparation: %s)", - async (concurrent) => { - const workdir = mkdtempSync(join(tmpdir(), "agent-queued-mode-")); - class ModeSession extends SteeringTestSession { - mode = "plan"; - applied: string[] = []; - startedModes: string[] = []; - override async setMode(mode?: string) { - if (!mode) throw new Error("Mode required"); - this.mode = mode; - this.applied.push(mode); - } - override async getCurrentMode() { - return this.mode; - } - override async startTurn(prompt: AgentPromptInput) { - this.startedModes.push(this.mode); - return super.startTurn(prompt); - } +test("an idle-only prompt rejects a busy recipient before changing its mode", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-queued-mode-")); + class ModeSession extends SteeringTestSession { + mode = "plan"; + applied: string[] = []; + startedModes: string[] = []; + override async setMode(mode?: string) { + if (!mode) throw new Error("Mode required"); + this.mode = mode; + this.applied.push(mode); } - const session = new ModeSession({ provider: "codex", cwd: workdir }); - const client = new (class extends TestAgentClient { - override async createSession(): Promise { - return session; + override async getCurrentMode() { + return this.mode; + } + override async startTurn(prompt: AgentPromptInput) { + this.startedModes.push(this.mode); + return super.startTurn(prompt); + } + } + const session = new ModeSession({ provider: "codex", cwd: workdir }); + const client = new (class extends TestAgentClient { + override async createSession(): Promise { + return session; + } + })(); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const manager = new AgentManager({ clients: { codex: client }, registry: storage, logger }); + const agent = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + await manager.setAgentMode(agent.id, "plan"); + session.applied.length = 0; + const providerStarted = new Promise((resolve) => { + const unsubscribe = session.subscribe((event) => { + if (event.type === "turn_started") { + unsubscribe(); + resolve(); } - })(); - const storage = new AgentStorage(join(workdir, "agents"), logger); - const manager = new AgentManager({ clients: { codex: client }, registry: storage, logger }); - const agent = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, { - workspaceId: undefined, }); - await manager.setAgentMode(agent.id, "plan"); - session.applied.length = 0; - const providerStarted = new Promise((resolve) => { - const unsubscribe = session.subscribe((event) => { - if (event.type === "turn_started") { - unsubscribe(); - resolve(); - } - }); - }); - const startOther = async () => { - await startAgentRun(manager, agent.id, "other turn", logger); - await manager.waitForAgentRunStart(agent.id); - }; - try { - if (!concurrent) await startOther(); - await expect( - sendPreparedPromptToAgent({ - agentManager: manager, - agentStorage: storage, - agentId: agent.id, - prompt: "queued follow-up", - sessionMode: "full-access", - idleOnly: true, - logger, - beforeDispatch: concurrent ? startOther : undefined, - }), - ).rejects.toBeInstanceOf(AgentTurnBusyError); - expect(session.mode).toBe("plan"); - expect(session.applied).toEqual([]); - expect(session.startedModes).toEqual(["plan"]); - await providerStarted; - session.pushEvent({ type: "turn_completed", provider: "codex", turnId: "active-turn-1" }); - await vi.waitFor(() => expect(manager.hasInFlightRun(agent.id)).toBe(false)); - await sendPreparedPromptToAgent({ + }); + const startOther = async () => { + await startAgentRun(manager, agent.id, "other turn", logger); + await manager.waitForAgentRunStart(agent.id); + }; + try { + await startOther(); + await expect( + sendPreparedPromptToAgent({ agentManager: manager, agentStorage: storage, agentId: agent.id, @@ -14691,14 +14672,29 @@ test.each([false, true])( sessionMode: "full-access", idleOnly: true, logger, - }); - await manager.waitForAgentRunStart(agent.id); - expect(session.mode).toBe("full-access"); - expect(session.applied).toEqual(["full-access"]); - expect(session.startedModes).toEqual(["plan", "full-access"]); - } finally { - await manager.closeAgent(agent.id); - rmSync(workdir, { recursive: true, force: true }); - } - }, -); + }), + ).rejects.toBeInstanceOf(AgentTurnBusyError); + expect(session.mode).toBe("plan"); + expect(session.applied).toEqual([]); + expect(session.startedModes).toEqual(["plan"]); + await providerStarted; + session.pushEvent({ type: "turn_completed", provider: "codex", turnId: "active-turn-1" }); + await vi.waitFor(() => expect(manager.hasInFlightRun(agent.id)).toBe(false)); + await sendPreparedPromptToAgent({ + agentManager: manager, + agentStorage: storage, + agentId: agent.id, + prompt: "queued follow-up", + sessionMode: "full-access", + idleOnly: true, + logger, + }); + await manager.waitForAgentRunStart(agent.id); + expect(session.mode).toBe("full-access"); + expect(session.applied).toEqual(["full-access"]); + expect(session.startedModes).toEqual(["plan", "full-access"]); + } finally { + await manager.closeAgent(agent.id); + rmSync(workdir, { recursive: true, force: true }); + } +}); diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 5c23afa667..59c2ff73e6 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -3128,11 +3128,7 @@ export class AgentManager { }): Promise { const { agent, agentId, pendingRun, prompt, options } = params; try { - const { sessionMode, ...providerOptions } = options ?? {}; - if (sessionMode) await this.setAgentMode(agentId, sessionMode); - if (pendingRun.settled) - throw new Error(`Agent ${agentId} run was canceled before its turn started`); - const result = await agent.session.startTurn(prompt, sessionMode ? providerOptions : options); + const result = await agent.session.startTurn(prompt, options); if (pendingRun.settled) { throw new Error(`Agent ${agentId} run was canceled before its turn started`); } diff --git a/packages/server/src/server/agent/agent-messages/delivery.test.ts b/packages/server/src/server/agent/agent-messages/delivery.test.ts index 37fdc94b4a..4f6e94f6f3 100644 --- a/packages/server/src/server/agent/agent-messages/delivery.test.ts +++ b/packages/server/src/server/agent/agent-messages/delivery.test.ts @@ -455,15 +455,25 @@ test("shutdown flushes a dispatch that discovered a concurrent turn during prepa expect(modes).toEqual([true, false]); }); -test("busy retries retain admission order across conflicting session modes", async () => { +test("conflicting modes dispatch on separate turn ends without changing the active turn mode", async () => { const clock = createClock(); - let busy = true; + let busy = false; + let mode = "initial"; + let wake!: (recipientId: string) => void; const sent: Array<{ ids: readonly string[]; mode?: string }> = []; const delivery = createAgentMessageDelivery({ schedule: clock.schedule, + isRunning: () => busy, + subscribe: (listener) => { + wake = listener; + return () => {}; + }, dispatch: async (request) => { + // Mode preparation precedes run admission in the real dispatcher. + mode = request.sessionMode ?? mode; if (busy) throw new AgentTurnBusyError("concurrent turn"); - sent.push({ ids: request.messageIds, mode: request.sessionMode }); + sent.push({ ids: request.messageIds, mode }); + busy = true; return "turn_started"; }, onError: (error) => { @@ -478,10 +488,14 @@ test("busy retries retain admission order across conflicting session modes", asy const a = delivery.enqueue({ ...message, messageId: "A", sessionMode: "plan" }); const b = delivery.enqueue({ ...message, messageId: "B", sessionMode: "default" }); clock.advance(2000); + await a.dispatched; await new Promise((resolve) => setImmediate(resolve)); + expect(mode).toBe("plan"); + expect(sent).toEqual([{ ids: ["A"], mode: "plan" }]); busy = false; - clock.advance(2000); - await Promise.all([a.dispatched, b.dispatched]); + wake("parent"); + await b.dispatched; + expect(mode).toBe("default"); expect(sent).toEqual([ { ids: ["A"], mode: "plan" }, { ids: ["B"], mode: "default" }, diff --git a/packages/server/src/server/agent/agent-messages/delivery.ts b/packages/server/src/server/agent/agent-messages/delivery.ts index f4ac993176..24d18baf11 100644 --- a/packages/server/src/server/agent/agent-messages/delivery.ts +++ b/packages/server/src/server/agent/agent-messages/delivery.ts @@ -117,16 +117,7 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes return undefined; } catch (error) { if (error instanceof AgentTurnBusyError && !closed) { - const entry = pending.get(recipientId) ?? { - messages: [], - ready: true, - cancel: () => {}, - }; - entry.cancel(); - entry.messages.push(...messages); - entry.messages.sort((a, b) => a.sequence - b.sequence); - entry.cancel = schedule(2000, () => flush(recipientId)); - pending.set(recipientId, entry); + requeue(recipientId, messages); return; } for (const message of messages) message.reject(error); @@ -142,6 +133,19 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes .catch(() => undefined); } + function requeue(recipientId: string, messages: Admitted[]): void { + const entry = pending.get(recipientId) ?? { + messages: [], + ready: true, + cancel: () => {}, + }; + entry.cancel(); + entry.messages.push(...messages); + entry.messages.sort((a, b) => a.sequence - b.sequence); + entry.cancel = schedule(2000, () => flush(recipientId)); + pending.set(recipientId, entry); + } + function flush(recipientId: string): void { const entry = pending.get(recipientId); if (!entry) return; @@ -167,6 +171,7 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes } function dispatchGroups(recipientId: string, messages: Admitted[]): void { + const groups: Admitted[][] = []; let group: Admitted[] = []; let mode: string | undefined; for (const message of messages) { @@ -176,14 +181,23 @@ export function createAgentMessageDelivery(dependencies: Dependencies): AgentMes ((message.activeTurnBehavior ?? "steer") !== (group[0]!.activeTurnBehavior ?? "steer") || (mode && message.sessionMode && mode !== message.sessionMode)) ) { - dispatch(recipientId, group); + groups.push(group); group = []; mode = undefined; } group.push(message); mode ??= message.sessionMode; } - if (group.length) dispatch(recipientId, group); + if (group.length) groups.push(group); + let started = false; + for (const batch of groups) { + if (!closed && started && batch[0]!.activeTurnBehavior !== "interrupt") { + requeue(recipientId, batch); + } else { + dispatch(recipientId, batch); + started = true; + } + } } const unsubscribe = dependencies.subscribe?.((recipientId) => { diff --git a/packages/server/src/server/agent/agent-prompt.ts b/packages/server/src/server/agent/agent-prompt.ts index 004ec459eb..f23a549491 100644 --- a/packages/server/src/server/agent/agent-prompt.ts +++ b/packages/server/src/server/agent/agent-prompt.ts @@ -219,7 +219,7 @@ export async function unarchiveAgentState( export interface SendPromptToAgentParams { idleOnly?: boolean; - /** Awaited after loading and before run admission; direct sends apply their mode first. */ + /** Awaited after the agent is loaded and its mode is set, before the prompt is handed to a run. */ beforeDispatch?: () => Promise; onDispatch?: StartAgentRunOptions["onDispatch"]; agentManager: AgentManager; @@ -364,7 +364,10 @@ export async function sendPreparedPromptToAgent( throw error; } - if (params.sessionMode && !params.idleOnly) { + if (params.idleOnly && params.agentManager.hasInFlightRun(params.agentId)) { + throw new AgentTurnBusyError(`Agent ${params.agentId} is running`); + } + if (params.sessionMode) { await params.agentManager.setAgentMode(params.agentId, params.sessionMode); } @@ -375,10 +378,7 @@ export async function sendPreparedPromptToAgent( onDispatch: params.onDispatch, activeTurnBehavior: params.activeTurnBehavior, clearPendingPermissions: params.clearPendingPermissions, - runOptions: - params.idleOnly && params.sessionMode - ? { ...params.runOptions, sessionMode: params.sessionMode } - : params.runOptions, + runOptions: params.runOptions, }); } diff --git a/packages/server/src/server/agent/agent-sdk-types.ts b/packages/server/src/server/agent/agent-sdk-types.ts index aebe822554..0267876d0d 100644 --- a/packages/server/src/server/agent/agent-sdk-types.ts +++ b/packages/server/src/server/agent/agent-sdk-types.ts @@ -197,8 +197,6 @@ export type AgentPromptContentBlock = export type AgentPromptInput = string | AgentPromptContentBlock[]; export interface AgentRunOptions { - /** Apply the requested mode after reserving this run, before provider startup. */ - sessionMode?: string; outputSchema?: unknown; resumeFrom?: AgentPersistenceHandle; maxThinkingTokens?: number;