diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.ts index 62b8ffc564..66f9dd198a 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.ts @@ -64,6 +64,7 @@ import { mapCodexToolCallEnvelope, mapCodexToolCallFromThreadItem, normalizeCommandExecutionCommand, + normalizeCollabAgentChildStatus, splitCodexMcpToolResultImages, } from "./codex/tool-call-mapper.js"; import { normalizeCodexFilePath } from "./codex/tool-call-detail-parser.js"; @@ -103,6 +104,7 @@ import { codexAppServerStartup } from "./codex/app-server-startup.js"; import { type CodexUserMessageTurnIndex, revertCodexConversation } from "./codex/rewind.js"; import { type CodexHistoryRow, + type CodexHistoryStatus, CodexThreadHistoryTooLargeError, readCodexThreadHistory, } from "./codex/thread-history.js"; @@ -557,12 +559,14 @@ interface CodexThreadHistory { interface PersistedSubAgentRoute { childThreadId: string; - toolCall: ToolCallTimelineItem; + entry: PersistedTimelineEntry; + legacyCollaboration: boolean; } interface CodexThreadHistoryProjection { timeline: PersistedTimelineEntry[]; subAgentRoutes: PersistedSubAgentRoute[]; + status: CodexHistoryStatus | null; } function mergeCodexConfiguredDefaults( @@ -1976,6 +1980,94 @@ function readCodexSubAgentActivity(item: unknown): CodexSubAgentActivity | null }; } +interface CodexCollabAgentUpdate { + threadId: string; + status: ToolCallTimelineItem["status"]; + preservesTerminal: boolean; +} + +type CodexCollabAgentControl = + | { kind: "submission"; childThreadIds: string[] } + | { kind: "outcome"; childThreadIds: string[]; updates: CodexCollabAgentUpdate[] }; + +interface HistoricalCodexSubAgent { + timelineIndex: number; + pendingFollowUp: boolean; + legacyCollaboration: boolean; +} + +function readCodexCollabAgentControl(item: unknown): CodexCollabAgentControl | null { + const record = toObjectRecord(item); + if ( + !record || + normalizeCodexThreadItemType(typeof record.type === "string" ? record.type : undefined) !== + "collabAgentToolCall" || + record.tool === "spawnAgent" + ) { + return null; + } + const tool = typeof record.tool === "string" ? record.tool : undefined; + const receivers = Array.isArray(record.receiverThreadIds) + ? record.receiverThreadIds.filter((id): id is string => typeof id === "string") + : []; + if (tool === "sendInput") { + return record.status === "completed" && record.error == null + ? { kind: "submission", childThreadIds: receivers } + : null; + } + if (tool !== "wait" && tool !== "closeAgent") return null; + const outcomeStates = toObjectRecord(record.agentsStates); + const updates = Object.entries(outcomeStates ?? {}).flatMap( + ([threadId, value]): CodexCollabAgentUpdate[] => { + const state = toObjectRecord(value); + if (typeof state?.status !== "string") return []; + // Unlike a spawn snapshot, a control result can report the child's final error. + const status = normalizeCollabAgentChildStatus(state.status, tool); + return [ + { + threadId, + status, + preservesTerminal: state.status === "shutdown" || record.tool === "closeAgent", + }, + ]; + }, + ); + return { + kind: "outcome", + childThreadIds: [...new Set([...receivers, ...updates.map((update) => update.threadId)])], + updates, + }; +} + +function resolveCodexCollabAgentUpdate( + previous: ToolCallTimelineItem["status"], + update: CodexCollabAgentUpdate, +): ToolCallTimelineItem["status"] { + if ( + isTerminalSubAgentStatus(previous) && + (update.preservesTerminal || update.status === "running") + ) { + return previous; + } + return update.status; +} + +function restoreCollabAgentCallFromChildHistory( + state: CodexSubAgentCallState, + recorded: ToolCallTimelineItem, + status: CodexHistoryStatus | null, +): void { + // A live event received during the read is newer than the history snapshot. + if (state.toolCall !== recorded || status === null) return; + // A stored child turn can predate a terminal parent outcome, such as a close. + // A currently active child is direct evidence that it is working again. + if (status !== "running" && isTerminalSubAgentStatus(recorded.status)) return; + state.toolCall = + status === "failed" + ? { ...recorded, status, error: { message: "Sub-agent failed" } } + : { ...recorded, status, error: null }; +} + function shouldIgnoreMirroredLifecycleItem(source: "item" | "codex_event", item: unknown): boolean { return source === "codex_event" && !readCodexSubAgentActivity(item); } @@ -2016,7 +2108,7 @@ function updateHistoricalSubAgentActivity( }; } -function readCodexHistoricalSubAgentThreadIds(item: unknown): string[] { +function readCodexSpawnedSubAgentThreadIds(item: unknown): string[] { const activity = readCodexSubAgentActivity(item); if (activity) { return activity.kind === "started" ? [activity.agentThreadId] : []; @@ -2179,46 +2271,95 @@ async function threadItemToTimelineEntries( return [timelineItem, ...(await mcpToolResultImagesToTimeline(item))]; } +function updateHistoricalCollabAgentCalls( + timeline: PersistedTimelineEntry[], + subAgentsByThreadId: Map, + item: unknown, +): boolean { + const control = readCodexCollabAgentControl(item); + if (!control) return false; + if (control.kind === "submission") { + for (const threadId of control.childThreadIds) { + const child = subAgentsByThreadId.get(threadId); + if (child) child.pendingFollowUp = true; + } + return false; + } + for (const update of control.updates) { + const child = subAgentsByThreadId.get(update.threadId); + const existing = child === undefined ? undefined : timeline[child.timelineIndex]; + if (!child || existing?.item.type !== "tool_call") continue; + const status = child.pendingFollowUp + ? update.status + : resolveCodexCollabAgentUpdate(existing.item.status, update); + child.pendingFollowUp = false; + const toolCall: ToolCallTimelineItem = + status === "failed" + ? { ...existing.item, status, error: { message: "Sub-agent failed" } } + : { ...existing.item, status, error: null }; + timeline[child.timelineIndex] = { ...existing, item: toolCall }; + } + return ( + control.childThreadIds.length > 0 && + control.childThreadIds.every((id) => subAgentsByThreadId.has(id)) + ); +} + async function loadCodexThreadHistoryTimeline(params: { client: CodexAppServerClientLike; threadId: string; cwd: string | null; + readLatestTurn?: boolean; }): Promise { const timeline: PersistedTimelineEntry[] = []; - const subAgentTimelineIndexByThreadId = new Map(); + const subAgentsByThreadId = new Map(); + let status: CodexHistoryStatus | null = null; try { - await projectCodexThreadHistory(params, { timeline, subAgentTimelineIndexByThreadId }); + const history = await readCodexThreadHistory(params.client, params.threadId, { + readLatestTurn: params.readLatestTurn, + }); + status = history.status; + await projectCodexThreadHistory( + { cwd: params.cwd, rows: history.rows }, + { + timeline, + subAgentsByThreadId, + }, + ); } catch (error) { if (!(error instanceof CodexThreadHistoryTooLargeError)) throw error; // The session stays usable; the row says why the history stops here. timeline.push({ item: { type: "notification", level: "warning", message: error.message } }); } - const subAgentRoutes = Array.from(subAgentTimelineIndexByThreadId.entries()).flatMap( - ([childThreadId, timelineIndex]): PersistedSubAgentRoute[] => { - const item = timeline[timelineIndex]?.item; - return item?.type === "tool_call" && item.detail.type === "sub_agent" - ? [{ childThreadId, toolCall: item }] + const subAgentRoutes = Array.from(subAgentsByThreadId.entries()).flatMap( + ([childThreadId, child]): PersistedSubAgentRoute[] => { + const entry = timeline[child.timelineIndex]; + return entry?.item.type === "tool_call" && entry.item.detail.type === "sub_agent" + ? [{ childThreadId, entry, legacyCollaboration: child.legacyCollaboration }] : []; }, ); - return { timeline, subAgentRoutes }; + return { timeline, subAgentRoutes, status }; } async function projectCodexThreadHistory( - params: { client: CodexAppServerClientLike; threadId: string; cwd: string | null }, + params: { rows: Iterable | AsyncIterable; cwd: string | null }, projection: { timeline: PersistedTimelineEntry[]; - subAgentTimelineIndexByThreadId: Map; + subAgentsByThreadId: Map; }, ): Promise { - const { timeline, subAgentTimelineIndexByThreadId } = projection; - for await (const row of readCodexThreadHistory(params.client, params.threadId)) { + const { timeline, subAgentsByThreadId } = projection; + for await (const row of params.rows) { const { item } = row; + if (updateHistoricalCollabAgentCalls(timeline, subAgentsByThreadId, item)) { + continue; + } const historicalSubAgentActivity = readCodexSubAgentActivity(item); if (historicalSubAgentActivity) { - const existingIndex = subAgentTimelineIndexByThreadId.get( + const existingIndex = subAgentsByThreadId.get( historicalSubAgentActivity.agentThreadId, - ); + )?.timelineIndex; if (existingIndex !== undefined) { const activityTimelineItem = await threadItemToTimeline(item, { cwd: params.cwd }); updateHistoricalSubAgentActivity( @@ -2248,8 +2389,12 @@ async function projectCodexThreadHistory( ? { providerTurnId: row.turnId } : {}), }); - for (const childThreadId of readCodexHistoricalSubAgentThreadIds(item)) { - subAgentTimelineIndexByThreadId.set(childThreadId, timeline.length - 1); + for (const childThreadId of readCodexSpawnedSubAgentThreadIds(item)) { + subAgentsByThreadId.set(childThreadId, { + timelineIndex: timeline.length - 1, + pendingFollowUp: false, + legacyCollaboration: historicalSubAgentActivity === null, + }); } } } @@ -4299,31 +4444,46 @@ export class CodexAppServerAgentSession implements AgentSession { parentSubagentId: null as string | null, })); const visitedThreadIds = new Set(this.currentThreadId ? [this.currentThreadId] : []); + const replayed: { + route: PersistedSubAgentRoute; + state: CodexSubAgentCallState; + timeline: readonly PersistedTimelineEntry[]; + }[] = []; while (queue.length > 0 && visitedThreadIds.size < 100) { const next = queue.shift(); if (!next || visitedThreadIds.has(next.route.childThreadId)) { continue; } visitedThreadIds.add(next.route.childThreadId); + const toolCall = next.route.entry.item; + if (toolCall.type !== "tool_call") continue; this.registerSubAgentToolCall({ - timelineItem: next.route.toolCall, + timelineItem: toolCall, rawItem: { agentThreadId: next.route.childThreadId }, + childThreadIds: [next.route.childThreadId], parentCallId: next.parentCallId, parentSubagentId: next.parentSubagentId, + deferUpsert: true, }); + const state = this.subAgentCallsByCallId.get(toolCall.callId); + if (!state?.childThreadIds.has(next.route.childThreadId)) continue; + let timeline: readonly PersistedTimelineEntry[] = []; try { const childHistory = await loadCodexThreadHistoryTimeline({ client, threadId: next.route.childThreadId, cwd: this.config.cwd ?? null, + readLatestTurn: + next.route.legacyCollaboration && !isTerminalSubAgentStatus(toolCall.status), }); - for (const entry of childHistory.timeline) { - this.emitProviderSubagentTimeline(next.route.childThreadId, entry.item, entry.timestamp); + if (next.route.legacyCollaboration) { + restoreCollabAgentCallFromChildHistory(state, toolCall, childHistory.status); } + timeline = childHistory.timeline; for (const route of childHistory.subAgentRoutes) { queue.push({ route, - parentCallId: next.route.toolCall.callId, + parentCallId: toolCall.callId, parentSubagentId: next.route.childThreadId, }); } @@ -4333,6 +4493,17 @@ export class CodexAppServerAgentSession implements AgentSession { "Failed to load persisted Codex child history", ); } + replayed.push({ route: next.route, state, timeline }); + } + // Settle nested cards before emitting the parent timelines that contain them. + for (const { route, state } of replayed) { + if (route.legacyCollaboration) route.entry.item = state.toolCall; + } + for (const { route, state, timeline } of replayed) { + this.emitProviderSubagentUpsert(route.childThreadId, state, state.toolCall.status); + for (const entry of timeline) { + this.emitProviderSubagentTimeline(route.childThreadId, entry.item, entry.timestamp); + } } } @@ -6235,8 +6406,10 @@ export class CodexAppServerAgentSession implements AgentSession { private registerSubAgentToolCall(params: { timelineItem: ToolCallTimelineItem; rawItem: { [key: string]: unknown }; + childThreadIds: readonly string[]; parentCallId: string | null; parentSubagentId: string | null; + deferUpsert?: boolean; }): string[] { const { timelineItem, rawItem, parentCallId, parentSubagentId } = params; if (timelineItem.detail.type !== "sub_agent") { @@ -6266,16 +6439,9 @@ export class CodexAppServerAgentSession implements AgentSession { } this.subAgentCallsByCallId.set(timelineItem.callId, state); - const receiverThreadIds = Array.isArray(rawItem.receiverThreadIds) - ? rawItem.receiverThreadIds.filter((value): value is string => typeof value === "string") - : []; - const agentThreadId = - typeof rawItem.agentThreadId === "string" && rawItem.agentThreadId.length > 0 - ? rawItem.agentThreadId - : null; - const childThreadIds = Array.from( - new Set(agentThreadId ? [...receiverThreadIds, agentThreadId] : receiverThreadIds), - ).filter((threadId) => threadId !== this.currentThreadId); + const childThreadIds = Array.from(new Set(params.childThreadIds)).filter( + (threadId) => threadId !== this.currentThreadId, + ); for (const receiverThreadId of childThreadIds) { const owningCallId = this.subAgentCallIdByChildThreadId.get(receiverThreadId); if (owningCallId && owningCallId !== timelineItem.callId) { @@ -6283,12 +6449,43 @@ export class CodexAppServerAgentSession implements AgentSession { } this.subAgentCallIdByChildThreadId.set(receiverThreadId, timelineItem.callId); state.childThreadIds.add(receiverThreadId); - this.emitProviderSubagentUpsert(receiverThreadId, state, timelineItem.status); + if (!params.deferUpsert) { + this.emitProviderSubagentUpsert(receiverThreadId, state, timelineItem.status); + } } return childThreadIds; } + private handleRegisteredCollabAgentControl( + rawItem: { [key: string]: unknown }, + control: CodexCollabAgentControl, + ): boolean { + if (control.kind === "submission") return false; + for (const update of control.updates) { + const callId = this.subAgentCallIdByChildThreadId.get(update.threadId); + if (!callId) continue; + const state = this.subAgentCallsByCallId.get(callId); + if (!state) continue; + const status = resolveCodexCollabAgentUpdate(state.toolCall.status, update); + if (status !== state.toolCall.status) { + this.emitSubAgentActivityUpdate(callId, status); + } + } + // A control rendered at start must also render its completion, even if + // Codex's final result lists fewer receivers than its started record. + const hasOwnCard = typeof rawItem.id === "string" && this.subAgentCallsByCallId.has(rawItem.id); + return ( + !hasOwnCard && + control.childThreadIds.length > 0 && + control.childThreadIds.every((id) => this.subAgentCallIdByChildThreadId.has(id)) + ); + } + private handleRegisteredSubAgentActivity(rawItem: { [key: string]: unknown }): boolean { + const control = readCodexCollabAgentControl(rawItem); + if (control) { + return this.handleRegisteredCollabAgentControl(rawItem, control); + } const activity = readCodexSubAgentActivity(rawItem); if (!activity) { return false; @@ -7228,6 +7425,7 @@ export class CodexAppServerAgentSession implements AgentSession { ? this.registerSubAgentToolCall({ timelineItem, rawItem: parsed.item, + childThreadIds: readCodexSpawnedSubAgentThreadIds(parsed.item), parentCallId: childSubAgentCallId, parentSubagentId: resolveCodexParentSubagentId(childSubAgentCallId, parsed.threadId), }) @@ -7401,6 +7599,7 @@ export class CodexAppServerAgentSession implements AgentSession { const registeredChildThreadIds = this.registerSubAgentToolCall({ timelineItem, rawItem: parsed.item, + childThreadIds: readCodexSpawnedSubAgentThreadIds(parsed.item), parentCallId: childSubAgentCallId, parentSubagentId: resolveCodexParentSubagentId(childSubAgentCallId, parsed.threadId), }); diff --git a/packages/server/src/server/agent/providers/codex/collab-replay.test.ts b/packages/server/src/server/agent/providers/codex/collab-replay.test.ts new file mode 100644 index 0000000000..737a27476c --- /dev/null +++ b/packages/server/src/server/agent/providers/codex/collab-replay.test.ts @@ -0,0 +1,692 @@ +import { describe, expect, test, vi } from "vitest"; +import { CodexAppServerAgentSession } from "../codex-app-server-agent.js"; +import { createTestLogger } from "../../../../test-utils/test-logger.js"; +import { createFakeCodexAppServer } from "./test-utils/fake-app-server.js"; + +function collab(tool: string, states: Record, id = tool) { + return { + type: "collabAgentToolCall", + id, + tool, + status: "completed", + prompt: tool === "spawnAgent" ? "Reply with a short result." : null, + receiverThreadIds: Object.keys(states), + agentsStates: Object.fromEntries( + Object.entries(states).map(([threadId, status]) => [threadId, { status }]), + ), + }; +} + +interface ChildHistory { + status?: { type: string }; + turns: { status: string; items?: unknown[] }[]; + historyMode?: "paginated"; + turnProbe?: "supported" | "unsupported"; +} + +async function replay(items: unknown[], children: Record = {}) { + const appServer = createFakeCodexAppServer({ + "thread/read": (params) => { + const { threadId, includeTurns } = params as { threadId: string; includeTurns: boolean }; + const child = children[threadId]; + const turns = threadId === "parent" ? [{ items }] : (child?.turns ?? []); + return { + thread: { + status: child?.status, + historyMode: child?.historyMode, + turns: includeTurns ? turns : [], + }, + }; + }, + "thread/items/list": () => ({ data: [], nextCursor: null }), + "thread/turns/list": (params) => { + const child = children[(params as { threadId: string }).threadId]; + if (child?.turnProbe !== "supported") throw new Error("method not supported"); + return { data: child.turns.slice(-1) }; + }, + }); + const session = new CodexAppServerAgentSession( + { provider: "codex", cwd: process.cwd(), modeId: "full-access" }, + { sessionId: "parent" }, + createTestLogger(), + async () => appServer.child, + {}, + false, + false, + false, + undefined, + "history", + ); + try { + await session.connect(); + const history = await Array.fromAsync(session.streamHistory()); + for (const [threadId, child] of Object.entries(children)) { + // Status is recovered from the history we already read for this child. + expect( + appServer + .requests() + .flatMap(({ method, params }) => + method === "thread/read" && (params as { threadId: string }).threadId === threadId + ? [(params as { includeTurns: boolean }).includeTurns] + : [], + ), + ).toEqual(child.historyMode === "paginated" ? [false] : [false, true]); + expect( + appServer + .requests() + .filter( + ({ method, params }) => + method === "thread/turns/list" && + (params as { threadId: string }).threadId === threadId, + ), + ).toMatchObject( + child.turnProbe + ? [{ params: { threadId, limit: 1, itemsView: "notLoaded", sortDirection: "desc" } }] + : [], + ); + } + return history; + } finally { + await session.close(); + appServer.assertNoErrors(); + } +} + +describe("Codex collaboration history", () => { + test.each([ + { turn: "completed", expected: "completed" }, + { turn: "failed", expected: "failed" }, + { turn: "interrupted", expected: "canceled" }, + ])( + "recovers a $turn child even when the parent did not wait or close it", + async ({ turn, expected }) => { + const history = await replay( + [collab("spawnAgent", { first: "pendingInit" }, "spawn-first")], + { + first: { + status: { type: "notLoaded" }, + turns: [{ status: "completed" }, { status: turn }], + }, + }, + ); + expect( + history.flatMap((event) => + event.type === "provider_subagent" && event.event.type === "upsert" + ? [event.event.status] + : [], + ), + ).toEqual([expected]); + expect( + history.flatMap((event) => + event.type === "timeline" && event.item.type === "tool_call" ? [event.item.status] : [], + ), + ).toEqual([expected]); + }, + ); + + test("keeps a live child running when the rewound parent only has its earlier wait", async () => { + const history = await replay( + [ + collab("spawnAgent", { first: "pendingInit" }, "spawn-first"), + collab("wait", { first: "completed" }), + ], + { first: { status: { type: "active" }, turns: [{ status: "completed" }] } }, + ); + expect( + history.flatMap((event) => + event.type === "provider_subagent" && event.event.type === "upsert" + ? [event.event.status] + : [], + ), + ).toEqual(["running"]); + expect( + history.flatMap((event) => + event.type === "timeline" && event.item.type === "tool_call" ? [event.item.status] : [], + ), + ).toEqual(["running"]); + }); + + test("keeps a live completion received while the child history is loading", async () => { + const reading = Promise.withResolvers(); + const childReply = Promise.withResolvers(); + const appServer = createFakeCodexAppServer({ + "thread/read": (params) => { + const { threadId, includeTurns } = params as { threadId: string; includeTurns: boolean }; + if (!includeTurns) return { thread: {} }; + if (threadId === "parent") { + return { + thread: { + turns: [{ items: [collab("spawnAgent", { first: "pendingInit" }, "spawn-first")] }], + }, + }; + } + reading.resolve(); + return childReply.promise; + }, + }); + const session = new CodexAppServerAgentSession( + { provider: "codex", cwd: process.cwd(), modeId: "full-access" }, + { sessionId: "parent" }, + createTestLogger(), + async () => appServer.child, + ); + const completed = Promise.withResolvers(); + session.subscribe((event) => { + if ( + event.type === "timeline" && + event.item.type === "tool_call" && + event.item.callId === "spawn-first" && + event.item.status === "completed" + ) { + completed.resolve(); + } + }); + try { + await session.connect(); + const restoring = Array.fromAsync(session.streamHistory()); + await reading.promise; + appServer.completeTurn({ threadId: "first" }); + await completed.promise; + childReply.resolve({ thread: { status: { type: "active" }, turns: [] } }); + const history = await restoring; + expect( + history + .flatMap((event) => + event.type === "provider_subagent" && event.event.type === "upsert" + ? [event.event.status] + : [], + ) + .at(-1), + ).toBe("completed"); + expect(history.filter((event) => event.type === "timeline")).toMatchObject([ + { item: { callId: "spawn-first", status: "completed" } }, + ]); + } finally { + await session.close(); + appServer.assertNoErrors(); + } + }); + + test("does not let an older child turn override an explicit close", async () => { + const history = await replay( + [ + collab("spawnAgent", { first: "pendingInit" }, "spawn-first"), + collab("closeAgent", { first: "running" }), + ], + { first: { status: { type: "notLoaded" }, turns: [{ status: "completed" }] } }, + ); + expect( + history.flatMap((event) => + event.type === "provider_subagent" && event.event.type === "upsert" + ? [event.event.status] + : [], + ), + ).toEqual(["canceled"]); + }); + + test("reads one turn without items for an unresolved paginated child", async () => { + const history = await replay( + [ + collab("spawnAgent", { migrated: "pendingInit" }, "spawn-migrated"), + collab("spawnAgent", { settled: "pendingInit" }, "spawn-settled"), + collab("wait", { settled: "completed" }), + collab("spawnAgent", { unsupported: "pendingInit" }, "spawn-unsupported"), + ], + { + migrated: { + historyMode: "paginated", + turns: [{ status: "failed" }], + turnProbe: "supported", + }, + settled: { historyMode: "paginated", turns: [{ status: "completed" }] }, + unsupported: { historyMode: "paginated", turns: [], turnProbe: "unsupported" }, + }, + ); + expect( + history.flatMap((event) => + event.type === "provider_subagent" && event.event.type === "upsert" + ? [[event.event.id, event.event.status]] + : [], + ), + ).toEqual([ + ["migrated", "failed"], + ["settled", "completed"], + ["unsupported", "running"], + ]); + }); + + test("settles a nested child's own card and descriptor", async () => { + const history = await replay([collab("spawnAgent", { first: "pendingInit" }, "spawn-first")], { + first: { + turns: [ + { + status: "completed", + items: [collab("spawnAgent", { nested: "pendingInit" }, "spawn-nested")], + }, + ], + }, + nested: { turns: [{ status: "interrupted" }] }, + }); + expect(history.filter((event) => event.type === "provider_subagent")).toMatchObject([ + { event: { type: "upsert", id: "first", status: "completed" } }, + { + event: { + type: "timeline", + id: "first", + item: { callId: "spawn-nested", status: "canceled" }, + }, + }, + { event: { type: "upsert", id: "nested", parentSubagentId: "first", status: "canceled" } }, + ]); + }); + + test("restores completed children from nullable wait and close results", async () => { + const history = await replay([ + collab("spawnAgent", { first: "pendingInit" }, "spawn-first"), + collab("spawnAgent", { second: "pendingInit" }, "spawn-second"), + collab("wait", { first: "completed" }, "wait-first"), + collab("wait", { second: "completed" }, "wait-second"), + collab("closeAgent", { first: "completed" }, "close-first"), + collab("closeAgent", { second: "completed" }, "close-second"), + ]); + expect( + history.flatMap((event) => + event.type === "provider_subagent" && event.event.type === "upsert" + ? [{ id: event.event.id, status: event.event.status }] + : [], + ), + ).toEqual([ + { id: "first", status: "completed" }, + { id: "second", status: "completed" }, + ]); + }); + + test.each(["completed", "interrupted", "running"])( + "keeps follow-up instructions in replay without applying the %s submission snapshot", + async (submissionStatus) => { + const history = await replay([ + collab("spawnAgent", { first: "pendingInit" }, "spawn-first"), + collab("wait", { first: "completed" }), + { ...collab("sendInput", { first: submissionStatus }), prompt: "Now add tests" }, + ]); + expect( + history.flatMap((event) => + event.type === "timeline" && event.item.type === "tool_call" + ? [{ id: event.item.callId, status: event.item.status, detail: event.item.detail }] + : [], + ), + ).toEqual([ + { id: "spawn-first", status: "completed", detail: expect.anything() }, + { + id: "sendInput", + status: "completed", + detail: expect.objectContaining({ description: "Now add tests" }), + }, + ]); + expect( + history.flatMap((event) => + event.type === "provider_subagent" && event.event.type === "upsert" + ? [{ id: event.event.id, status: event.event.status }] + : [], + ), + ).toEqual([{ id: "first", status: "completed" }]); + }, + ); + + test.each([ + ["completed", "closeAgent", "errored", "failed"], + ["errored", "closeAgent", "completed", "completed"], + ["completed", "wait", "running", "running"], + ["completed", "closeAgent", "running", "canceled"], + ])( + "replaces the earlier %s outcome after a follow-up and %s result %s", + async (initialStatus, tool, resultStatus, expected) => { + const history = await replay([ + collab("spawnAgent", { first: "pendingInit" }, "spawn-first"), + collab("spawnAgent", { second: "pendingInit" }, "spawn-second"), + { + ...collab("wait", { first: initialStatus, second: "completed" }), + status: initialStatus === "errored" ? "failed" : "completed", + }, + { ...collab("sendInput", { first: "running" }), prompt: "Do more work" }, + { + ...collab(tool, { first: resultStatus }), + status: resultStatus === "errored" ? "failed" : "completed", + }, + ]); + expect( + history.flatMap((event) => + event.type === "provider_subagent" && event.event.type === "upsert" + ? [{ id: event.event.id, status: event.event.status }] + : [], + ), + ).toEqual([ + { id: "first", status: expected }, + { id: "second", status: "completed" }, + ]); + expect( + history.flatMap((event) => + event.type === "timeline" && + event.item.type === "tool_call" && + event.item.callId === "spawn-first" + ? [event.item.status] + : [], + ), + ).toEqual([expected]); + }, + ); + + test("keeps the prior outcome when a follow-up submission fails", async () => { + const history = await replay([ + collab("spawnAgent", { first: "pendingInit" }, "spawn-first"), + collab("wait", { first: "completed" }), + { ...collab("sendInput", { first: "running" }), status: "failed", prompt: "Do more work" }, + collab("wait", { first: "running" }, "stale-wait"), + ]); + expect( + history.flatMap((event) => + event.type === "provider_subagent" && event.event.type === "upsert" + ? [event.event.status] + : [], + ), + ).toEqual(["completed"]); + }); + + test("settles a close card without a registered spawn", async () => { + const history = await replay([collab("closeAgent", { external: "running" })]); + expect( + history.flatMap((event) => + event.type === "timeline" && event.item.type === "tool_call" + ? [{ id: event.item.callId, status: event.item.status }] + : [], + ), + ).toEqual([{ id: "closeAgent", status: "canceled" }]); + }); + + test.each(["notFound", "errored"])( + "preserves completion and shows a failed resume reporting %s", + async (status) => { + const history = await replay([ + collab("spawnAgent", { first: "pendingInit" }, "spawn-first"), + collab("wait", { first: "completed" }), + collab("closeAgent", { first: "completed" }), + { ...collab("resumeAgent", { first: status }), status: "failed" }, + ]); + expect( + history.flatMap((event) => + event.type === "provider_subagent" && event.event.type === "upsert" + ? [event.event.status] + : [], + ), + ).toEqual(["completed"]); + expect( + history.flatMap((event) => + event.type === "timeline" && event.item.type === "tool_call" + ? [{ id: event.item.callId, status: event.item.status }] + : [], + ), + ).toEqual([ + { id: "spawn-first", status: "completed" }, + { id: "resumeAgent", status: "failed" }, + ]); + }, + ); + + test("stops counting a child closed while running", async () => { + const history = await replay([ + collab("spawnAgent", { first: "pendingInit" }, "spawn-first"), + collab("closeAgent", { first: "running" }), + ]); + expect( + history.flatMap((event) => + event.type === "provider_subagent" && event.event.type === "upsert" + ? [{ id: event.event.id, status: event.event.status }] + : [], + ), + ).toEqual([{ id: "first", status: "canceled" }]); + }); + + test.each(["wait", "closeAgent"])( + "restores an errored child as failed after %s", + async (tool) => { + const history = await replay([ + collab("spawnAgent", { first: "pendingInit" }, "spawn-first"), + { ...collab(tool, { first: "errored" }), status: "failed" }, + ]); + expect( + history.flatMap((event) => + event.type === "provider_subagent" && event.event.type === "upsert" + ? [{ id: event.event.id, status: event.event.status }] + : [], + ), + ).toEqual([{ id: "first", status: "failed" }]); + expect( + history.flatMap((event) => + event.type === "timeline" && event.item.type === "tool_call" + ? [{ id: event.item.callId, status: event.item.status, error: event.item.error }] + : [], + ), + ).toEqual([{ id: "spawn-first", status: "failed", error: { message: "Sub-agent failed" } }]); + }, + ); + + test.each(["wait", "closeAgent"])( + "settles a live errored child as failed after %s", + async (tool) => { + const appServer = createFakeCodexAppServer(); + const session = new CodexAppServerAgentSession( + { provider: "codex", cwd: process.cwd(), modeId: "full-access" }, + null, + createTestLogger(), + async () => appServer.child, + ); + const statuses = new Map(); + const cards = new Map(); + session.subscribe((event) => { + if (event.type === "provider_subagent" && event.event.type === "upsert") { + statuses.set(event.event.id, event.event.status); + } + if (event.type === "timeline" && event.item.type === "tool_call") { + cards.set(event.item.callId, { status: event.item.status, error: event.item.error }); + } + }); + function notify(method: string, item: ReturnType) { + appServer.child.stdout.write( + `${JSON.stringify({ method, params: { threadId: "thread-1", item } })}\n`, + ); + } + try { + const result = session.run("Delegate the work."); + await appServer.waitForTurnStart(); + notify("item/completed", collab("spawnAgent", { first: "pendingInit" }, "spawn-first")); + notify("item/started", { + ...collab(tool, { first: "errored" }), + status: "inProgress", + agentsStates: {}, + }); + notify("item/completed", { ...collab(tool, { first: "errored" }), status: "failed" }); + appServer.completeTurn(); + await result; + expect([...statuses]).toEqual([["first", "failed"]]); + expect([...cards]).toEqual([ + ["spawn-first", { status: "failed", error: { message: "Sub-agent failed" } }], + ]); + } finally { + await session.close(); + appServer.assertNoErrors(); + } + }, + ); + + test("preserves completion when a close carries an older running snapshot", async () => { + const history = await replay([ + collab("spawnAgent", { first: "pendingInit" }, "spawn-first"), + collab("wait", { first: "completed" }), + collab("closeAgent", { first: "running" }), + ]); + expect( + history.flatMap((event) => + event.type === "provider_subagent" && event.event.type === "upsert" + ? [{ id: event.event.id, status: event.event.status }] + : [], + ), + ).toEqual([{ id: "first", status: "completed" }]); + }); + + test.each([ + ["shutdown", "canceled"], + ["notFound", "failed"], + ])("restores a %s child as %s and preserves its running sibling", async (status, expected) => { + const history = await replay([ + collab("spawnAgent", { first: "pendingInit" }, "spawn-first"), + collab("spawnAgent", { second: "pendingInit" }, "spawn-second"), + collab("closeAgent", { first: status }), + ]); + expect( + history.flatMap((event) => + event.type === "provider_subagent" && event.event.type === "upsert" + ? [{ id: event.event.id, status: event.event.status }] + : [], + ), + ).toEqual([ + { id: "first", status: expected }, + { id: "second", status: "running" }, + ]); + }); + + test("settles a rendered live wait when its completion lists only the finished receiver", async () => { + const appServer = createFakeCodexAppServer(); + const session = new CodexAppServerAgentSession( + { provider: "codex", cwd: process.cwd(), modeId: "full-access" }, + null, + createTestLogger(), + async () => appServer.child, + ); + const cards = new Map(); + const statuses = new Map(); + session.subscribe((event) => { + if (event.type === "timeline" && event.item.type === "tool_call") { + cards.set(event.item.callId, event.item.status); + } + if (event.type === "provider_subagent" && event.event.type === "upsert") { + statuses.set(event.event.id, event.event.status); + } + }); + function notify(method: string, item: ReturnType) { + appServer.child.stdout.write( + `${JSON.stringify({ method, params: { threadId: "thread-1", item } })}\n`, + ); + } + try { + const result = session.run("Wait for the children."); + await appServer.waitForTurnStart(); + notify("item/completed", collab("spawnAgent", { first: "pendingInit" }, "spawn-first")); + notify("item/started", { + ...collab("wait", { first: "running", second: "running" }, "mixed-wait"), + status: "inProgress", + agentsStates: {}, + }); + await vi.waitFor(() => expect(cards.get("mixed-wait")).toBe("running")); + notify("item/completed", collab("wait", { first: "completed" }, "mixed-wait")); + notify("item/completed", collab("wait", { second: "completed" }, "wait-second")); + appServer.completeTurn(); + await result; + expect([...statuses]).toEqual([["first", "completed"]]); + expect([...cards]).toEqual([ + ["spawn-first", "completed"], + ["mixed-wait", "completed"], + ["wait-second", "completed"], + ]); + } finally { + await session.close(); + appServer.assertNoErrors(); + } + }); + + test.each(["completed", "interrupted", "running"])( + "keeps a live follow-up running despite a late %s sendInput snapshot", + async (submissionStatus) => { + const appServer = createFakeCodexAppServer(); + const session = new CodexAppServerAgentSession( + { provider: "codex", cwd: process.cwd(), modeId: "full-access" }, + null, + createTestLogger(), + async () => appServer.child, + ); + const statuses = new Map(); + const timelineCards = new Map(); + let parentMessage = ""; + session.subscribe((event) => { + if (event.type === "provider_subagent" && event.event.type === "upsert") { + statuses.set(event.event.id, event.event.status); + } + if (event.type === "timeline" && event.item.type === "tool_call") { + timelineCards.set(event.item.callId, { + status: event.item.status, + detail: event.item.detail, + }); + } + if (event.type === "timeline" && event.item.type === "assistant_message") { + parentMessage = event.item.text; + } + }); + function complete(tool: string, states: Record, id = tool) { + const item = { + ...collab(tool, states, id), + ...(tool === "sendInput" ? { prompt: "Now add tests" } : {}), + }; + if (tool !== "spawnAgent") { + appServer.child.stdout.write( + `${JSON.stringify({ + method: "item/started", + params: { + threadId: "thread-1", + item: { ...item, status: "inProgress", agentsStates: {} }, + }, + })}\n`, + ); + } + appServer.child.stdout.write( + `${JSON.stringify({ + method: "item/completed", + params: { threadId: "thread-1", item }, + })}\n`, + ); + } + try { + const result = session.run("Delegate the work."); + await appServer.waitForTurnStart(); + complete("spawnAgent", { first: "pendingInit" }, "spawn-first"); + complete("spawnAgent", { second: "pendingInit" }, "spawn-second"); + complete("wait", { first: "completed", second: "running" }); + await vi.waitFor(() => { + expect(statuses.get("first")).toBe("completed"); + expect(statuses.get("second")).toBe("running"); + }); + complete("closeAgent", { first: "shutdown" }); + complete("wait", { first: "running" }, "stale-wait"); + appServer.says({ threadId: "thread-1", text: "Still waiting." }); + appServer.completeTurn(); + await result; + expect(statuses.get("first")).toBe("completed"); + appServer.startsTurn({ threadId: "first", turnId: "follow-up" }); + await vi.waitFor(() => expect(statuses.get("first")).toBe("running")); + complete("sendInput", { first: submissionStatus }); + appServer.says({ threadId: "thread-1", text: "Follow-up submitted." }); + await vi.waitFor(() => expect(parentMessage).toBe("Follow-up submitted.")); + expect(statuses.get("first")).toBe("running"); + expect([...timelineCards.keys()]).toEqual(["spawn-first", "spawn-second", "sendInput"]); + expect(timelineCards.get("sendInput")).toEqual({ + status: "completed", + detail: expect.objectContaining({ description: "Now add tests" }), + }); + // Only the child's completion settles the follow-up. + appServer.completeTurn({ threadId: "first" }); + await vi.waitFor(() => expect(statuses.get("first")).toBe("completed")); + } finally { + await session.close(); + appServer.assertNoErrors(); + } + }, + ); +}); diff --git a/packages/server/src/server/agent/providers/codex/thread-history.ts b/packages/server/src/server/agent/providers/codex/thread-history.ts index 557d92150b..1dda8eae71 100644 --- a/packages/server/src/server/agent/providers/codex/thread-history.ts +++ b/packages/server/src/server/agent/providers/codex/thread-history.ts @@ -10,6 +10,7 @@ export interface CodexThreadHistoryClient { } export type CodexThreadHistoryMode = "legacy" | "paginated"; +export type CodexHistoryStatus = "running" | "completed" | "failed" | "canceled"; export interface CodexHistoryRow { item: unknown; @@ -19,6 +20,11 @@ export interface CodexHistoryRow { completedAt: unknown; } +interface CodexThreadHistoryRead { + rows: Iterable | AsyncIterable; + status: CodexHistoryStatus | null; +} + /** A history reply over the line cap; the rows before it were already read. */ export class CodexThreadHistoryTooLargeError extends Error { constructor(message: string) { @@ -29,7 +35,22 @@ export class CodexThreadHistoryTooLargeError extends Error { const CodexThreadMetadataSchema = z .object({ - thread: z.object({ historyMode: z.enum(["legacy", "paginated"]).optional() }).passthrough(), + thread: z + .object({ + historyMode: z.enum(["legacy", "paginated"]).optional(), + status: z.unknown().optional(), + }) + .passthrough(), + }) + .passthrough(); + +const CodexLegacyTurnSchema = z + .object({ + id: z.string().optional(), + startedAt: z.unknown().optional(), + completedAt: z.unknown().optional(), + status: z.unknown().optional(), + items: z.array(z.unknown()).default([]), }) .passthrough(); @@ -37,18 +58,8 @@ const CodexLegacyThreadReadSchema = z .object({ thread: z .object({ - turns: z - .array( - z - .object({ - id: z.string().optional(), - startedAt: z.unknown().optional(), - completedAt: z.unknown().optional(), - items: z.array(z.unknown()).default([]), - }) - .passthrough(), - ) - .default([]), + status: z.unknown().optional(), + turns: z.array(CodexLegacyTurnSchema).default([]), }) .passthrough() .default({ turns: [] }), @@ -71,15 +82,48 @@ const CodexItemsPageSchema = z }) .passthrough(); +const CodexLatestTurnPageSchema = z.object({ + data: z.array(z.object({ status: z.unknown().optional() })), +}); + /** Codex builds before 0.143 report no history mode; their threads are legacy. */ export async function readCodexThreadHistoryMode( client: CodexThreadHistoryClient, threadId: string, ): Promise { - const response = CodexThreadMetadataSchema.parse( + return (await readThreadMetadata(client, threadId)).historyMode ?? "legacy"; +} + +async function readThreadMetadata(client: CodexThreadHistoryClient, threadId: string) { + return CodexThreadMetadataSchema.parse( await client.request("thread/read", { threadId, includeTurns: false }), - ); - return response.thread.historyMode ?? "legacy"; + ).thread; +} + +function readHistoryStatus( + threadStatus: unknown, + latestTurnStatus?: unknown, +): CodexHistoryStatus | null { + if ( + typeof threadStatus === "object" && + threadStatus !== null && + "type" in threadStatus && + threadStatus.type === "active" + ) { + return "running"; + } + // An unloaded thread can contain an unfinished rollout. Only the runtime + // status proves that it is still working. + switch (latestTurnStatus) { + case "completed": + return "completed"; + case "failed": + return "failed"; + case "interrupted": + return "canceled"; + default: + return null; + } } /** @@ -87,21 +131,20 @@ export async function readCodexThreadHistoryMode( * pages; a legacy thread arrives in one `thread/read` reply, as Codex offers * no pages for it. */ -export async function* readCodexThreadHistory( +export async function readCodexThreadHistory( client: CodexThreadHistoryClient, threadId: string, -): AsyncGenerator { - if ((await readCodexThreadHistoryMode(client, threadId)) === "paginated") { - yield* readPaginatedHistory(client, threadId); - } else { - yield* readLegacyHistory(client, threadId); + options?: { readLatestTurn?: boolean }, +): Promise { + const metadata = await readThreadMetadata(client, threadId); + if (metadata.historyMode === "paginated") { + const active = readHistoryStatus(metadata.status); + return { + rows: readPaginatedHistory(client, threadId), + status: + active ?? (options?.readLatestTurn ? await readLatestTurnStatus(client, threadId) : null), + }; } -} - -async function* readLegacyHistory( - client: CodexThreadHistoryClient, - threadId: string, -): AsyncGenerator { const response = await requestHistory( client, "thread/read", @@ -111,7 +154,37 @@ async function* readLegacyHistory( `run \`codex migrate-rollouts --apply --thread ${threadId}\` (Codex 0.148 or newer), ` + `then unarchive the agent.`, ); - for (const turn of CodexLegacyThreadReadSchema.parse(response).thread.turns) { + const { thread } = CodexLegacyThreadReadSchema.parse(response); + return { + rows: readLegacyRows(thread.turns), + status: readHistoryStatus(thread.status, thread.turns.at(-1)?.status), + }; +} + +async function readLatestTurnStatus( + client: CodexThreadHistoryClient, + threadId: string, +): Promise { + try { + const response = await client.request("thread/turns/list", { + threadId, + limit: 1, + sortDirection: "desc", + itemsView: "notLoaded", + }); + const page = CodexLatestTurnPageSchema.parse(response); + return readHistoryStatus(undefined, page.data[0]?.status); + } catch { + // Older Codex builds and stores can reject this optional probe. The + // recorded parent outcome and the child's timeline still load normally. + return null; + } +} + +function* readLegacyRows( + turns: z.infer[], +): Generator { + for (const turn of turns) { for (const item of turn.items) { yield { item, turnId: turn.id, startedAt: turn.startedAt, completedAt: turn.completedAt }; } diff --git a/packages/server/src/server/agent/providers/codex/tool-call-mapper.test.ts b/packages/server/src/server/agent/providers/codex/tool-call-mapper.test.ts index 423aa08c76..01f8f24afa 100644 --- a/packages/server/src/server/agent/providers/codex/tool-call-mapper.test.ts +++ b/packages/server/src/server/agent/providers/codex/tool-call-mapper.test.ts @@ -213,6 +213,20 @@ describe("codex tool-call mapper", () => { }); }); + it("accepts a nullable prompt in completed collaboration control items", () => { + expect( + mapCodexToolCallFromThreadItem({ + type: "collabAgentToolCall", + id: "wait-child", + tool: "wait", + prompt: null, + status: "completed", + receiverThreadIds: ["child"], + agentsStates: { child: { status: "completed" } }, + }), + ).toMatchObject({ callId: "wait-child", status: "completed" }); + }); + it("maps collabAgentToolCall into canonical sub-agent detail", () => { const item = mapCodexToolCallFromThreadItem({ type: "collabAgentToolCall", diff --git a/packages/server/src/server/agent/providers/codex/tool-call-mapper.ts b/packages/server/src/server/agent/providers/codex/tool-call-mapper.ts index 8c0f8b9558..8e335de76b 100644 --- a/packages/server/src/server/agent/providers/codex/tool-call-mapper.ts +++ b/packages/server/src/server/agent/providers/codex/tool-call-mapper.ts @@ -187,7 +187,7 @@ const CodexCollabAgentToolCallItemSchema = z id: z.string().min(1), status: z.string().optional(), error: z.unknown().optional(), - prompt: z.string().optional(), + prompt: z.string().nullish(), tool: z.string().optional(), receiverThreadIds: z.array(z.string()).optional(), agentsStates: z.record(z.string(), z.unknown()).optional(), @@ -546,12 +546,18 @@ function readStatus(value: unknown): string | undefined { return typeof value.status === "string" ? value.status : undefined; } -function normalizeCollabAgentChildStatus(status: string): ToolCallTimelineItem["status"] { +export function normalizeCollabAgentChildStatus( + status: string, + tool?: string, +): ToolCallTimelineItem["status"] { const normalized = status.trim().toLowerCase(); switch (normalized) { case "error": case "errored": - return "running"; + return tool === "wait" || tool === "closeAgent" ? "failed" : "running"; + case "running": + case "pendinginit": + return tool === "closeAgent" ? "canceled" : "running"; case "shutdown": return "canceled"; case "notfound": @@ -573,10 +579,15 @@ function resolveCollabAgentStatus( return "failed"; } + // Sending input completes the submission, not the child's new turn. + if (item.tool === "sendInput") { + return parentStatus; + } + const childStatuses = Object.values(item.agentsStates ?? {}) .map(readStatus) .filter((status): status is string => typeof status === "string" && status.trim().length > 0) - .map(normalizeCollabAgentChildStatus); + .map((status) => normalizeCollabAgentChildStatus(status, item.tool)); if (childStatuses.some((status) => status === "failed")) { return "failed";