From 4178497d3dd5fbe9196557194bda84449bc40e60 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Tue, 10 Nov 2026 08:11:00 +0100 Subject: [PATCH] refactor(server): give agent timeline history a single awaitable owner Agent timeline history had several writers that appended and published independently, so a persistent backend could not be swapped in without reordering or losing items. AgentTimeline now owns each agent's history: one FIFO queue per agent (child agents share the parent's queue) commits each item and then publishes it, so subscribers never see an item before it is recorded. Backends may be async, which prepares the SQLite swap. Terminal turn events wait for queued writes so a finished turn never races its own last chunks. A subscriber that throws after commit leaves the item in history. A failed commit surfaces as an agent error ("Conversation history could not be saved") via the existing lastError/attention path; it survives the turn's terminal transition and clears on the next run. --- .../src/server/agent/agent-loading.test.ts | 6 +- .../agent-manager-stream-coalescing.test.ts | 88 ++- .../src/server/agent/agent-manager.test.ts | 424 ++++++----- .../server/src/server/agent/agent-manager.ts | 715 +++++++++--------- .../agent/agent-timeline-store-types.ts | 19 - .../server/agent/agent-timeline-store.test.ts | 443 ----------- .../src/server/agent/agent-timeline-store.ts | 185 ----- .../src/server/agent/import-sessions.test.ts | 2 +- .../src/server/agent/import-sessions.ts | 4 +- .../src/server/agent/mcp-parity.e2e.test.ts | 12 +- .../server/src/server/agent/mcp-shared.ts | 2 +- .../agent/projected-timeline-contract.test.ts | 39 +- .../agent/provider-subagents/store.test.ts | 33 +- .../server/agent/provider-subagents/store.ts | 53 +- .../claude/subagents/observation.test.ts | 19 +- .../claude/subagents/replay-source.test.ts | 25 +- .../subagents/workflow-replay-source.test.ts | 9 +- .../providers/codex-app-server-agent.test.ts | 18 +- .../providers/mock-load-test-agent.test.ts | 2 +- .../edxeth-pi-subagents/index.test.ts | 7 +- .../gotgenes-pi-subagents/index.test.ts | 11 +- .../src/server/agent/rewind/rewind.test.ts | 10 +- .../agent/test-utils/plugin-stream-memory.ts | 10 +- .../test-utils/timeline-responsiveness.ts | 22 +- .../src/server/agent/timeline-projection.ts | 21 +- .../agent/timeline/agent-timeline.test.ts | 659 ++++++++++++++++ .../server/src/server/agent/timeline/index.ts | 10 + .../agent/timeline/internal/agent-timeline.ts | 242 ++++++ .../server/agent/timeline/internal/backend.ts | 88 +++ .../agent/timeline/internal/memory-backend.ts | 177 +++++ .../src/server/agent/timeline/testing.ts | 90 +++ .../src/server/agent/tools/paseo-tools.ts | 2 +- .../daemon-e2e/checkout-ship.e2e.test.ts | 11 +- .../daemon-e2e/timeline-window.e2e.test.ts | 30 +- .../hub/provider-policy.real.e2e.test.ts | 5 +- .../hub/test-utils/relationship-harness.ts | 6 +- packages/server/src/server/session.ts | 28 +- .../server/src/server/wire-compat.test.ts | 20 +- 38 files changed, 2153 insertions(+), 1394 deletions(-) delete mode 100644 packages/server/src/server/agent/agent-timeline-store.test.ts delete mode 100644 packages/server/src/server/agent/agent-timeline-store.ts create mode 100644 packages/server/src/server/agent/timeline/agent-timeline.test.ts create mode 100644 packages/server/src/server/agent/timeline/index.ts create mode 100644 packages/server/src/server/agent/timeline/internal/agent-timeline.ts create mode 100644 packages/server/src/server/agent/timeline/internal/backend.ts create mode 100644 packages/server/src/server/agent/timeline/internal/memory-backend.ts create mode 100644 packages/server/src/server/agent/timeline/testing.ts diff --git a/packages/server/src/server/agent/agent-loading.test.ts b/packages/server/src/server/agent/agent-loading.test.ts index d75fa8739a..dbae7adb46 100644 --- a/packages/server/src/server/agent/agent-loading.test.ts +++ b/packages/server/src/server/agent/agent-loading.test.ts @@ -185,9 +185,9 @@ test("loads an archived agent's history after its working directory is removed", expect(loaded.id).toBe(agentId); // The transcript is replayed from the provider's persisted history, so the reply the // agent gave before the worktree went away is still readable. - const replies = manager - .getTimeline(agentId) - .filter((item) => item.type === "assistant_message"); + const replies = (await manager.getTimeline(agentId)).filter( + (item) => item.type === "assistant_message", + ); expect(replies.length).toBeGreaterThan(0); expect(replies.every((item) => item.type === "assistant_message" && item.text.length > 0)).toBe( true, diff --git a/packages/server/src/server/agent/agent-manager-stream-coalescing.test.ts b/packages/server/src/server/agent/agent-manager-stream-coalescing.test.ts index c423264869..5c57915ad5 100644 --- a/packages/server/src/server/agent/agent-manager-stream-coalescing.test.ts +++ b/packages/server/src/server/agent/agent-manager-stream-coalescing.test.ts @@ -3,12 +3,16 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, test, vi } from "vitest"; +import pino, { type Logger } from "pino"; + import { createTestLogger } from "../../test-utils/test-logger.js"; import { AgentManager, type AgentManagerEvent } from "./agent-manager.js"; import { AGENT_STREAM_COALESCE_DEFAULT_WINDOW_MS } from "./agent-stream-coalescer.js"; import { limitAgentTimelineItemContent } from "./agent-timeline-content.js"; import type { AgentTimelineRow } from "./agent-timeline-store-types.js"; import { projectTimelineRows } from "./timeline-projection.js"; +import { AgentTimeline } from "./timeline/index.js"; +import { DeferredTimelineBackend } from "./timeline/testing.js"; import type { AgentCapabilityFlags, AgentClient, @@ -243,13 +247,18 @@ interface Harness { cleanup: () => void; } -function createHarness(options?: { provider?: AgentProvider }): Harness { +function createHarness(options?: { + provider?: AgentProvider; + timeline?: AgentTimeline; + logger?: Logger; +}): Harness { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-stream-coalescing-")); const client = new TestAgentClient(options?.provider ?? "codex"); const manager = new AgentManager({ clients: { [client.provider]: client }, idFactory: createIdFactory(), - logger: createTestLogger(), + logger: options?.logger ?? createTestLogger(), + timeline: options?.timeline, }); const events: AgentManagerEvent[] = []; manager.subscribe((event) => events.push(event), { replayState: false }); @@ -1037,6 +1046,81 @@ describe("target coalesced behavior", () => { } }); + test.each(["turn_completed", "turn_failed", "turn_canceled"] as const)( + "%s waits for the final chunk an asynchronous timeline is still committing", + async (terminalType) => { + const harness = createHarness({ timeline: new AgentTimeline(new DeferredTimelineBackend()) }); + try { + const { agentId, session } = await createManagedSession(harness); + + session.pushEvent(assistant("final chunk", "codex", "turn")); + session.pushEvent(terminalEvent(terminalType, "turn")); + + const streamTypes = () => + getStreamEvents(harness.events, agentId).map((event) => + event.type === "agent_stream" ? event.event.type : event.type, + ); + await expect.poll(streamTypes).toContain(terminalType); + const streamEvents = getStreamEvents(harness.events, agentId); + expect(streamEvents[0]).toMatchObject({ + event: { type: "timeline", item: { type: "assistant_message", text: "final chunk" } }, + seq: 1, + }); + expect(streamTypes().indexOf(terminalType)).toBeGreaterThan(0); + } finally { + await harness.manager.flush(); + harness.cleanup(); + } + }, + ); + + test("a final chunk the timeline fails to commit surfaces a history error after the turn", async () => { + const backend = new DeferredTimelineBackend(); + const logs: Array> = []; + const harness = createHarness({ + timeline: new AgentTimeline(backend), + logger: pino({ level: "error" }, { write: (line: string) => logs.push(JSON.parse(line)) }), + }); + try { + const { agentId, session } = await createManagedSession(harness); + session.pushEvent(assistant("earlier chunk", "codex", "earlier")); + session.pushEvent(terminalEvent("turn_completed", "earlier")); + await expect.poll(() => getTimelineStreamEvents(harness.events, agentId)).toHaveLength(1); + await harness.manager.flush(); + session.pushEvent({ type: "turn_started", provider: "codex", turnId: "turn" }); + await expect.poll(() => harness.manager.getAgent(agentId)?.lifecycle).toBe("running"); + backend.failNextAppend = true; + session.pushEvent(assistant("final chunk", "codex", "turn")); + session.pushEvent(terminalEvent("turn_completed", "turn")); + + const completions = () => + getStreamEvents(harness.events, agentId).filter( + (event) => event.type === "agent_stream" && event.event.type === "turn_completed", + ); + await expect.poll(completions).toHaveLength(2); + await harness.manager.flush(); + expect(harness.manager.getAgent(agentId)).toMatchObject({ + lifecycle: "error", + lastError: "Conversation history could not be saved: disk full", + attention: { requiresAttention: true, attentionReason: "error" }, + }); + const earlier = { type: "assistant_message", text: "earlier chunk" }; + expect(getTimelineStreamEvents(harness.events, agentId)).toEqual([ + expect.objectContaining({ event: expect.objectContaining({ item: earlier }) }), + ]); + expect(getTimelineItems(await harness.manager.getTimelineRows(agentId))).toEqual([earlier]); + expect(logs).toContainEqual( + expect.objectContaining({ + msg: "Failed to record timeline item", + agentId, + err: expect.objectContaining({ message: "disk full" }), + }), + ); + } finally { + harness.cleanup(); + } + }); + test("lifecycle paths are timer-safe", async () => { vi.useFakeTimers(); diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 4b3fb5ea46..c85c21a619 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -11,6 +11,7 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; +import pino from "pino"; import { createTestLogger } from "../../test-utils/test-logger.js"; import { AgentManager, @@ -20,7 +21,8 @@ import { type ManagedAgent, } from "./agent-manager.js"; import { AgentStorage } from "./agent-storage.js"; -import { InMemoryAgentTimelineStore } from "./agent-timeline-store.js"; +import { AgentTimeline } from "./timeline/index.js"; +import { DeferredTimelineBackend } from "./timeline/testing.js"; import { limitAgentTimelineItemContent } from "./agent-timeline-content.js"; import { toAgentPayload } from "./agent-projections.js"; import { projectTimelineRows } from "./timeline-projection.js"; @@ -29,12 +31,6 @@ import { startAgentRun } from "./agent-prompt.js"; import { StaleProviderSessionError } from "./stale-provider-session-error.js"; import { ensureAgentLoaded, ensureUnarchivedAgentLoaded } from "./agent-loading.js"; import type { StoredAgentRecord } from "./agent-storage.js"; -import type { - AgentTimelineFetchOptions, - AgentTimelineFetchResult, - AgentTimelineRow, - AgentTimelineStore, -} from "./agent-timeline-store-types.js"; import type { AgentClient, AgentCreateSessionOptions, @@ -115,73 +111,6 @@ const TEST_CAPABILITIES = { supportsToolInvocations: false, } as const; -class RecordingTimelineStore implements AgentTimelineStore { - readonly writes: AgentTimelineRow[][] = []; - private readonly memory = new InMemoryAgentTimelineStore(); - - private ensure(agentId: string): void { - if (!this.memory.has(agentId)) this.memory.initialize(agentId); - } - - async appendCommitted( - agentId: string, - item: AgentTimelineItem, - options?: { timestamp?: string; turnId?: string }, - ): Promise { - this.ensure(agentId); - return this.memory.append(agentId, item, options); - } - - async fetchCommitted( - agentId: string, - options?: AgentTimelineFetchOptions, - ): Promise { - this.ensure(agentId); - return this.memory.fetch(agentId, options); - } - - async getLatestCommittedSeq(agentId: string): Promise { - return this.memory.has(agentId) ? (this.memory.getRows(agentId).at(-1)?.seq ?? 0) : 0; - } - - async getCommittedRows(agentId: string): Promise { - return this.memory.has(agentId) ? this.memory.getRows(agentId) : []; - } - - async getLastItem(agentId: string): Promise { - return this.memory.has(agentId) ? this.memory.getLastItem(agentId) : null; - } - - async getLastAssistantMessage(agentId: string): Promise { - return this.memory.has(agentId) ? this.memory.getLastAssistantMessage(agentId) : null; - } - - async deleteAgent(agentId: string): Promise { - this.memory.delete(agentId); - } - - async bulkInsert(agentId: string, rows: readonly AgentTimelineRow[]): Promise { - this.writes.push(rows.map((row) => ({ ...row }))); - this.ensure(agentId); - for (const row of rows) { - this.memory.append(agentId, row.item, { - timestamp: row.timestamp, - turnId: row.turnId, - }); - } - } - - async updateCommittedRow(agentId: string, row: AgentTimelineRow): Promise { - this.ensure(agentId); - const rows = this.memory.getRows(agentId); - const index = rows.findIndex((candidate) => candidate.seq === row.seq); - if (index >= 0) { - rows[index] = row; - this.memory.initialize(agentId, { rows }); - } - } -} - function createFeature(args: { id: string; label: string; value: boolean }): AgentFeature { return { type: "toggle", @@ -670,12 +599,12 @@ async function startAndSteerThroughManager( return { manager, agentId: agent.id, workdir }; } -test("uses an injected timeline store without making it a production requirement", async () => { +test("commits timeline items through an injected asynchronous backend", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-timeline-store-")); - const store = new RecordingTimelineStore(); + const backend = new DeferredTimelineBackend(); const manager = new AgentManager({ clients: { codex: new TestAgentClient() }, - durableTimelineStore: store, + timeline: new AgentTimeline(backend), logger, }); let agentId: string | null = null; @@ -687,7 +616,7 @@ test("uses an injected timeline store without making it a production requirement await manager.appendTimelineItem(agent.id, { type: "assistant_message", text: "stored" }); await manager.flush(); - expect(store.writes.flat()).toContainEqual( + expect(backend.appends.flat()).toContainEqual( expect.objectContaining({ item: { type: "assistant_message", text: "stored" } }), ); await expect(manager.getTimelineRows(agent.id)).resolves.toContainEqual( @@ -699,9 +628,113 @@ test("uses an injected timeline store without making it a production requirement } }); -test("refreshing an agent replaces the injected timeline store instead of appending a second copy", async () => { +test("a subscriber that throws on publication leaves the committed item in history", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-publish-failure-")); + const logs: Array> = []; + const manager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + timeline: new AgentTimeline(new DeferredTimelineBackend()), + logger: pino({ level: "error" }, { write: (line: string) => logs.push(JSON.parse(line)) }), + }); + let throwOnTimeline = true; + manager.subscribe( + (event) => { + if (throwOnTimeline && event.type === "agent_stream" && event.event.type === "timeline") { + throwOnTimeline = false; + throw new Error("subscriber failed"); + } + }, + { replayState: false }, + ); + let agentId: string | null = null; + try { + const agent = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + agentId = agent.id; + await expect( + manager.appendTimelineItem(agent.id, { type: "assistant_message", text: "kept" }), + ).resolves.toMatchObject({ seq: 1 }); + await expect( + manager.appendTimelineItem(agent.id, { type: "assistant_message", text: " and next" }), + ).resolves.toMatchObject({ seq: 2 }); + expect(await manager.getLastAssistantMessage(agent.id)).toBe("kept and next"); + expect(logs).toContainEqual( + expect.objectContaining({ + msg: "Failed to publish committed timeline item", + agentId: agent.id, + err: expect.objectContaining({ message: "subscriber failed" }), + }), + ); + } finally { + if (agentId) await manager.closeAgent(agentId).catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + } +}); + +test("an accepted prompt the timeline fails to commit surfaces a history error", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-timeline-failure-")); + const backend = new DeferredTimelineBackend(); + const logs: Array> = []; + const manager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + timeline: new AgentTimeline(backend), + logger: pino({ level: "error" }, { write: (line: string) => logs.push(JSON.parse(line)) }), + }); + const published: AgentTimelineItem[] = []; + manager.subscribe( + (event) => { + if (event.type === "agent_stream" && event.event.type === "timeline") { + published.push(event.event.item); + } + }, + { replayState: false }, + ); + let agentId: string | null = null; + try { + const agent = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + agentId = agent.id; + const earlier: AgentTimelineItem = { type: "assistant_message", text: "Earlier reply" }; + await manager.appendTimelineItem(agent.id, earlier); + await manager.flush(); + backend.failNextAppend = true; + await manager.runAgent(agent.id, "hello", { clientMessageId: "hello-client" }); + await manager.flush(); + + expect(manager.getAgent(agent.id)).toMatchObject({ + lifecycle: "error", + lastError: "Conversation history could not be saved: disk full", + attention: { requiresAttention: true, attentionReason: "error" }, + }); + expect(await manager.getTimeline(agent.id)).toEqual([earlier]); + expect(published).toEqual([earlier]); + expect(logs).toContainEqual( + expect.objectContaining({ + msg: "Failed to record timeline item", + agentId: agent.id, + err: expect.objectContaining({ message: "disk full" }), + }), + ); + + // The next run clears the error and records normally. + await manager.runAgent(agent.id, "again", { clientMessageId: "again-client" }); + await manager.flush(); + expect(manager.getAgent(agent.id)).toMatchObject({ lifecycle: "idle" }); + expect(manager.getAgent(agent.id)?.lastError).toBeUndefined(); + expect(await manager.getTimeline(agent.id)).toEqual([ + earlier, + expect.objectContaining({ type: "user_message", clientMessageId: "again-client" }), + ]); + } finally { + if (agentId) await manager.closeAgent(agentId).catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + } +}); + +test("refreshing an agent replaces its timeline instead of appending a second copy", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-timeline-rehydrate-")); - const store = new RecordingTimelineStore(); class HistorySession extends TestAgentSession { override async *streamHistory(): AsyncGenerator { yield { @@ -730,7 +763,7 @@ test("refreshing an agent replaces the injected timeline store instead of append } const manager = new AgentManager({ clients: { codex: new HistoryClient() }, - durableTimelineStore: store, + timeline: new AgentTimeline(new DeferredTimelineBackend()), logger, }); let agentId: string | null = null; @@ -758,9 +791,8 @@ test("refreshing an agent replaces the injected timeline store instead of append } }); -test("a failed history replay leaves the committed timeline intact", async () => { +test("a failed history replay commits none of the replay", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-timeline-replay-fail-")); - const store = new RecordingTimelineStore(); let failReplay = false; class FlakyHistorySession extends TestAgentSession { override async *streamHistory(): AsyncGenerator { @@ -793,7 +825,7 @@ test("a failed history replay leaves the committed timeline intact", async () => } const manager = new AgentManager({ clients: { codex: new FlakyHistoryClient() }, - durableTimelineStore: store, + timeline: new AgentTimeline(new DeferredTimelineBackend()), logger, }); let agentId: string | null = null; @@ -814,7 +846,13 @@ test("a failed history replay leaves the committed timeline intact", async () => ).rejects.toThrow("history stream closed"); await manager.flush(); - expect(await manager.getTimelineRows(agent.id)).toEqual(committed); + // Reload with rehydration discards the active history before the replay (phase 1 keeps + // that); the failed replay must not leave the rows it streamed before failing. + expect(committed.map((row) => row.item)).toEqual([ + { type: "user_message", text: "continue" }, + { type: "assistant_message", text: "done" }, + ]); + expect(await manager.getTimelineRows(agent.id)).toEqual([]); // A retry once the provider recovers replaces that history rather than stacking on it. failReplay = false; @@ -871,7 +909,7 @@ test("retries provider history hydration after a stream failure", async () => { await manager.hydrateTimelineFromProvider(agent.id); expect(attempts).toBe(2); - expect(manager.getTimeline(agent.id)).toContainEqual({ + expect(await manager.getTimeline(agent.id)).toContainEqual({ type: "assistant_message", text: "provider history", }); @@ -924,7 +962,7 @@ test("notification batches share the run they start or join", async () => { session.pushEvent({ type: "turn_completed", provider: "codex", turnId: "active-turn-2" }); await second; expect( - manager.fetchTimeline(agent.id, { limit: 0 }).rows.map((row) => ({ + (await manager.fetchTimeline(agent.id, { limit: 0 })).rows.map((row) => ({ turnId: row.turnId, id: row.item.type === "tool_call" ? row.item.callId : row.item.type, })), @@ -948,7 +986,7 @@ test("unavailable steer interrupts once and starts one replacement turn", async try { expect(session.interruptCount).toBe(1); expect(session.startCount).toBe(2); - expect(manager.getTimeline(agentId)).toContainEqual( + expect(await manager.getTimeline(agentId)).toContainEqual( expect.objectContaining({ type: "user_message", clientMessageId: "replacement-client" }), ); } finally { @@ -1006,9 +1044,8 @@ test("steers a send that arrives while the previous turn is still starting", asy expect(session.startPrompts).toEqual(["FIRST"]); expect(session.interruptCount).toBe(0); expect( - manager - .fetchTimeline(agent.id, { limit: 0 }) - .rows.filter((row) => row.item.type === "user_message") + (await manager.fetchTimeline(agent.id, { limit: 0 })).rows + .filter((row) => row.item.type === "user_message") .map((row) => ({ text: row.item.type === "user_message" ? row.item.text : null, turnId: row.turnId, @@ -1137,14 +1174,14 @@ test("orders an accepted steer before output emitted while acknowledgement is pe provider: "codex", turnId: "active-turn-1", }); - expect(manager.getTimeline(agent.id)).not.toContainEqual( + expect(await manager.getTimeline(agent.id)).not.toContainEqual( expect.objectContaining({ type: "assistant_message", text: "terminal output" }), ); release.resolve(); await expect(steer).resolves.toEqual({ status: "accepted" }); await consume; - const rows = manager.fetchTimeline(agent.id, { limit: 0 }).rows.filter((row) => { + const rows = (await manager.fetchTimeline(agent.id, { limit: 0 })).rows.filter((row) => { return ( (row.item.type === "user_message" && row.item.text === "hello") || (row.item.type === "assistant_message" && row.item.text === "terminal output") @@ -1201,7 +1238,7 @@ test("orders buffered pre-steer output before an immediately accepted steer", as manager.steerAgentRun(agent.id, "hello", { clientMessageId: "hello-client" }), ).resolves.toEqual({ status: "accepted" }); - const rows = manager.fetchTimeline(agent.id, { limit: 0 }).rows.filter((row) => { + const rows = (await manager.fetchTimeline(agent.id, { limit: 0 })).rows.filter((row) => { return ( (row.item.type === "assistant_message" && row.item.text === "pre-steer output") || (row.item.type === "user_message" && row.item.text === "hello") @@ -1281,9 +1318,8 @@ test("orders a concurrent replacement after a pending accepted steer", async () expect(session.interruptCount).toBe(1); expect( - manager - .fetchTimeline(agent.id, { limit: 0 }) - .rows.filter((row) => row.item.type === "user_message") + (await manager.fetchTimeline(agent.id, { limit: 0 })).rows + .filter((row) => row.item.type === "user_message") .map((row) => row.item), ).toMatchObject([ { text: "hello", clientMessageId: "hello-client" }, @@ -1351,9 +1387,9 @@ test("does not replace a newer foreground turn after unavailable steer fallback expect(session.interruptCount).toBe(0); expect(session.startPrompts).not.toContain("hello"); expect( - manager - .getTimeline(agent.id) - .some((item) => item.type === "user_message" && item.clientMessageId === "hello-client"), + (await manager.getTimeline(agent.id)).some( + (item) => item.type === "user_message" && item.clientMessageId === "hello-client", + ), ).toBe(false); } finally { release.resolve(); @@ -1395,7 +1431,7 @@ test("steers a tracked autonomous turn without creating a replacement run", asyn expect(session.interruptCount).toBe(0); expect(manager.getAgent(agent.id)?.activeTurnId).toBe("active-turn-0"); expect(manager.getAgent(agent.id)?.activeForegroundTurnId).toBeNull(); - expect(manager.getTimeline(agent.id)).toContainEqual( + expect(await manager.getTimeline(agent.id)).toContainEqual( expect.objectContaining({ type: "user_message", text: "autonomous follow-up", @@ -1421,7 +1457,7 @@ test("isolated rewind falls back from steering to the normal replacement path", await manager.waitForAgentRunStart(agentId); expect(session.interruptCount).toBe(2); expect(session.startPrompts).toContain("/rewind submitted-message-id"); - expect(manager.getTimeline(agentId)).toContainEqual( + expect(await manager.getTimeline(agentId)).toContainEqual( expect.objectContaining({ type: "user_message", clientMessageId: "rewind-client" }), ); } finally { @@ -1436,7 +1472,7 @@ test("missing steer operation interrupts once and starts one replacement turn", try { expect(session.interruptCount).toBe(1); expect(session.startCount).toBe(2); - expect(manager.getTimeline(agentId)).toContainEqual( + expect(await manager.getTimeline(agentId)).toContainEqual( expect.objectContaining({ type: "user_message", clientMessageId: "replacement-client" }), ); } finally { @@ -1476,7 +1512,7 @@ test("ambiguous steer failure leaves the active turn untouched", async () => { ).rejects.toThrow("connection lost after delivery"); expect(session.interruptCount).toBe(0); expect(session.startCount).toBe(1); - expect(manager.getTimeline(agent.id)).not.toContainEqual( + expect(await manager.getTimeline(agent.id)).not.toContainEqual( expect.objectContaining({ clientMessageId: "ambiguous-client" }), ); } finally { @@ -1510,7 +1546,9 @@ test("steering records concurrent early echoes as canonical submitted prompts", manager.steerAgentRun(agent.id, "one", { clientMessageId: "client-one" }), manager.steerAgentRun(agent.id, "two", { clientMessageId: "client-two" }), ]); - const rows = manager.getTimeline(agent.id).filter((item) => item.type === "user_message"); + const rows = (await manager.getTimeline(agent.id)).filter( + (item) => item.type === "user_message", + ); expect(rows).toEqual( expect.arrayContaining([ expect.objectContaining({ messageId: "client-one", clientMessageId: "client-one" }), @@ -4235,7 +4273,7 @@ test("a prompt after provider replacement reopens the stale session", async () = const result = await manager.waitForAgentEvent(created.id); expect(result.status).toBe("idle"); expect(replacement.resumeOverrides).toHaveLength(1); - const timeline = manager.getTimeline(created.id); + const timeline = await manager.getTimeline(created.id); expect(timeline.filter((item) => item.type === "user_message")).toMatchObject([ { text: "continue after reload", clientMessageId: "stale-session-retry" }, ]); @@ -4590,7 +4628,7 @@ test("importProviderSession imports the selected session without listing and pub }); expect(imported.lifecycle).toBe("idle"); expect(imported.historyPrimed).toBe(true); - expect(manager.getTimeline(imported.id)).toEqual([ + expect(await manager.getTimeline(imported.id)).toEqual([ { type: "user_message", text: "Trace provider imports" }, { type: "assistant_message", text: "Done" }, limitAgentTimelineItemContent({ @@ -4610,7 +4648,7 @@ test("importProviderSession imports the selected session without listing and pub expect(manager.listProviderSubagents(imported.id)).toEqual([ expect.objectContaining({ id: "thread-child", title: "Imported child", status: "completed" }), ]); - expect(manager.fetchProviderSubagentTimeline(imported.id, "thread-child").rows).toEqual([ + expect((await manager.fetchProviderSubagentTimeline(imported.id, "thread-child")).rows).toEqual([ expect.objectContaining({ item: { type: "assistant_message", text: "Child result" } }), ]); expect(events).toHaveLength(3); @@ -4780,18 +4818,18 @@ test("reloadAgentSession preserves timeline and does not force history replay", text: "keep this timeline in memory", }); await manager.hydrateTimelineFromProvider(snapshot.id); - const beforeReload = manager.getTimeline(snapshot.id); + const beforeReload = await manager.getTimeline(snapshot.id); expect(beforeReload).toHaveLength(1); await manager.reloadAgentSession(snapshot.id, { systemPrompt: "reloaded prompt", }); - const afterReload = manager.getTimeline(snapshot.id); + const afterReload = await manager.getTimeline(snapshot.id); expect(afterReload).toEqual(beforeReload); // If reload resets historyPrimed, this would replay provider history and append another item. await manager.hydrateTimelineFromProvider(snapshot.id); - const afterHydrate = manager.getTimeline(snapshot.id); + const afterHydrate = await manager.getTimeline(snapshot.id); expect(afterHydrate).toEqual(beforeReload); }); @@ -5961,13 +5999,13 @@ test("fetchTimeline returns a bounded reset window when cursor epoch is stale", text: "three", }); - const baseline = manager.fetchTimeline(snapshot.id, { + const baseline = await manager.fetchTimeline(snapshot.id, { direction: "tail", limit: 2, }); expect(baseline.rows).toHaveLength(2); - const result = manager.fetchTimeline(snapshot.id, { + const result = await manager.fetchTimeline(snapshot.id, { direction: "after", cursor: { epoch: "stale-epoch", @@ -5984,7 +6022,7 @@ test("fetchTimeline returns a bounded reset window when cursor epoch is stale", expect(result.rows[result.rows.length - 1]?.seq).toBe(3); expect(result.hasOlder).toBe(true); - const older = manager.fetchTimeline(snapshot.id, { + const older = await manager.fetchTimeline(snapshot.id, { direction: "before", cursor: { epoch: result.epoch, @@ -6081,7 +6119,7 @@ test("getAgent does not expose committed history internals once manager owns the expect("timelineRows" in live).toBe(false); expect("timelineNextSeq" in live).toBe(false); - expect(manager.getTimeline(snapshot.id)).toEqual([ + expect(await manager.getTimeline(snapshot.id)).toEqual([ { type: "user_message", text: "hello boundary", @@ -6178,7 +6216,7 @@ test("streams coalesced assistant chunks and retains the projected message", asy epoch: expect.any(String), }); - expect(manager.getTimeline(snapshot.id)).toEqual([ + expect(await manager.getTimeline(snapshot.id)).toEqual([ { type: "assistant_message", text: "final reply", @@ -6252,7 +6290,7 @@ test("fetchTimeline supports older-history pagination with before seq", async () const result = await manager.fetchTimeline(snapshot.id, { direction: "before", cursor: { - epoch: manager.fetchTimeline(snapshot.id).epoch, + epoch: (await manager.fetchTimeline(snapshot.id)).epoch, seq: 5, }, limit: 2, @@ -6396,7 +6434,7 @@ test("hydrateTimeline preserves assistant chunk, reasoning, and tool timeline hi await manager.hydrateTimelineFromProvider(snapshot.id, { force: true }); - expect(manager.getTimeline(snapshot.id)).toEqual([ + expect(await manager.getTimeline(snapshot.id)).toEqual([ { type: "assistant_message", text: "chunk one chunk two" }, { type: "reasoning", text: "internal" }, { @@ -6478,7 +6516,7 @@ test("hydrateTimeline preserves reasoning between assistant chunks", async () => await manager.hydrateTimelineFromProvider(snapshot.id, { force: true }); - expect(manager.getTimeline(snapshot.id)).toEqual([ + expect(await manager.getTimeline(snapshot.id)).toEqual([ { type: "assistant_message", text: "before reasoning ", @@ -7084,13 +7122,11 @@ test("replaceAgentRun stays running when a stale old terminal arrives before the capturedSession!.pushEvent({ type: "turn_completed", provider: "codex", turnId: "turn-1" }); await new Promise((resolve) => setTimeout(resolve, 0)); expect( - manager - .fetchTimeline(snapshot.id, { direction: "tail", limit: 0 }) - .rows.find( - (row) => - row.item.type === "user_message" && - row.item.clientMessageId === "replacement-client-message", - )?.item, + (await manager.fetchTimeline(snapshot.id, { direction: "tail", limit: 0 })).rows.find( + (row) => + row.item.type === "user_message" && + row.item.clientMessageId === "replacement-client-message", + )?.item, ).toMatchObject({ messageId: "replacement-client-message" }); allowSecondTurnToComplete.resolve(); await firstRunDrain; @@ -7196,7 +7232,7 @@ test("applies live autonomous events and preserves usage omitted from completion contextWindowMaxTokens: 200_000, contextWindowUsedTokens: 175, }); - expect(manager.getTimeline(snapshot.id)).toContainEqual({ + expect(await manager.getTimeline(snapshot.id)).toContainEqual({ type: "assistant_message", text: "AUTONOMOUS_PUMP_MESSAGE", }); @@ -7258,7 +7294,7 @@ test("ignores stale autonomous terminals without lowering the active turn lifecy name: "Current turn permission", kind: "tool", }); - const timelineBeforeStaleTerminals = manager.getTimeline(snapshot.id); + const timelineBeforeStaleTerminals = await manager.getTimeline(snapshot.id); const staleTerminals: AgentStreamEvent[] = [ { @@ -7294,7 +7330,7 @@ test("ignores stale autonomous terminals without lowering the active turn lifecy expect(stillRunning?.lastError).toBe("turn-b marker"); expect(stillRunning?.lastUsage).toEqual({ inputTokens: 7 }); expect(stillRunning?.pendingPermissions.has("turn-b-permission")).toBe(true); - expect(manager.getTimeline(snapshot.id)).toEqual(timelineBeforeStaleTerminals); + expect(await manager.getTimeline(snapshot.id)).toEqual(timelineBeforeStaleTerminals); } capturedSession!.pushEvent({ type: "turn_completed", provider: "codex", turnId: "turn-b" }); @@ -7339,7 +7375,7 @@ test("preserves terminal fallback when no active turn identity was observed", as expect(failed?.lifecycle).toBe("error"); expect(failed?.lastError).toBe("untracked failure"); }); - expect(manager.getTimeline(snapshot.id)).toContainEqual( + expect(await manager.getTimeline(snapshot.id)).toContainEqual( expect.objectContaining({ type: "assistant_message", text: expect.stringContaining("untracked failure"), @@ -7643,8 +7679,8 @@ test("autonomous events arriving during foreground run are processed via subscri turnId: autonomousTurnId, }); - await vi.waitFor(() => - expect(manager.getTimeline(snapshot.id)).toContainEqual({ + await vi.waitFor(async () => + expect(await manager.getTimeline(snapshot.id)).toContainEqual({ type: "assistant_message", text: "AUTONOMOUS_DURING_FOREGROUND", }), @@ -7670,7 +7706,7 @@ test("autonomous events arriving during foreground run are processed via subscri ).toBe(false); // Autonomous timeline item should still be recorded in the agent timeline - expect(manager.getTimeline(snapshot.id)).toContainEqual({ + expect(await manager.getTimeline(snapshot.id)).toContainEqual({ type: "assistant_message", text: "AUTONOMOUS_DURING_FOREGROUND", }); @@ -7751,7 +7787,7 @@ test("subscribe error isolation: throwing subscriber does not break event flow", expect(receivedEvents).toContain("turn_started"); expect(receivedEvents).toContain("timeline"); expect(receivedEvents).toContain("turn_completed"); - expect(manager.getTimeline(snapshot.id)).toContainEqual({ + expect(await manager.getTimeline(snapshot.id)).toContainEqual({ type: "assistant_message", text: "EVENT_AFTER_ERROR", }); @@ -8371,9 +8407,9 @@ test("subscribe hides provider subagents of internal parents from global subscri expect(() => manager.getProviderSubagent(internalAgentId, "hidden-child")).toThrow( `Unknown agent '${internalAgentId}'`, ); - expect(() => manager.fetchProviderSubagentTimeline(internalAgentId, "hidden-child")).toThrow( - `Unknown agent '${internalAgentId}'`, - ); + await expect( + manager.fetchProviderSubagentTimeline(internalAgentId, "hidden-child"), + ).rejects.toThrow(`Unknown agent '${internalAgentId}'`); expect(manager.listProviderSubagentActivity()).toEqual([]); }); @@ -8677,7 +8713,7 @@ test("streamAgent clears pending run when startTurn fails before a turn id exist clientMessageId: "rejected-prompt", }), ).rejects.toThrow("Invalid request: missing field `text`"); - expect(manager.getTimeline(agent.id)).toEqual([ + expect(await manager.getTimeline(agent.id)).toEqual([ { type: "user_message", text: "fail before turn id", @@ -9577,12 +9613,10 @@ test("turn_failed emits a system error assistant timeline message and keeps erro expect(snapshot?.lifecycle).toBe("error"); expect(snapshot?.lastError).toBe("invalid model id"); - const systemErrors = manager - .getTimeline(agent.id) - .filter( - (item): item is Extract => - item.type === "assistant_message" && item.text.includes("[System Error]"), - ); + const systemErrors = (await manager.getTimeline(agent.id)).filter( + (item): item is Extract => + item.type === "assistant_message" && item.text.includes("[System Error]"), + ); expect(systemErrors).toHaveLength(1); expect(systemErrors[0]?.text).toContain("invalid model id"); }); @@ -9653,12 +9687,10 @@ test("turn_failed surfaces provider code and diagnostic in system error message" expect(manager.getAgent(agent.id)?.lastError).toBe("Provider execution failed"); - const systemError = manager - .getTimeline(agent.id) - .find( - (item): item is Extract => - item.type === "assistant_message" && item.text.includes("[System Error]"), - ); + const systemError = (await manager.getTimeline(agent.id)).find( + (item): item is Extract => + item.type === "assistant_message" && item.text.includes("[System Error]"), + ); expect(systemError?.text).toContain("Provider execution failed"); expect(systemError?.text).toContain("code: 126"); expect(systemError?.text).toContain("No preset version installed for command claude"); @@ -10913,7 +10945,7 @@ test("hydrateTimeline keeps provider user_message items when no canonical user h await manager.hydrateTimelineFromProvider(snapshot.id, { force: true }); - const timeline = manager.getTimeline(snapshot.id); + const timeline = await manager.getTimeline(snapshot.id); const userMessages = timeline.filter((item) => item.type === "user_message"); const assistantMessages = timeline.filter((item) => item.type === "assistant_message"); expect(userMessages).toHaveLength(2); @@ -10977,7 +11009,7 @@ test("hydrateTimeline preserves provider replay timestamps and marks missing one ); await manager.hydrateTimelineFromProvider(snapshot.id, { force: true }); - const timeline = manager.fetchTimeline(snapshot.id, { direction: "tail", limit: 0 }).rows; + const timeline = (await manager.fetchTimeline(snapshot.id, { direction: "tail", limit: 0 })).rows; expect(timeline).toHaveLength(2); expect(timeline[0]).toMatchObject({ @@ -11042,7 +11074,7 @@ test("provider user_message is recorded from the live stream", async () => { await manager.runAgent(snapshot.id, { text: "do something" }); - const timeline = manager.getTimeline(snapshot.id); + const timeline = await manager.getTimeline(snapshot.id); const userMessages = timeline.filter((item) => item.type === "user_message"); // Provider's user_message should be recorded (no canonical to dedup against) @@ -11178,7 +11210,8 @@ test("canonical submitted prompt keeps wire identity while rewind resolves provi { type: "turn_completed" }, ]); - const timeline = manager.fetchTimeline(snapshot.id, { direction: "tail", limit: 20 }).rows; + const timeline = (await manager.fetchTimeline(snapshot.id, { direction: "tail", limit: 20 })) + .rows; expect(timeline).toMatchObject([ { seq: 1, @@ -11267,6 +11300,8 @@ test("authoritative timeline records a daemon-handled submitted prompt before it runOptions: { clientMessageId: "msg-client-daemon-handled" }, }); await commandCompleted.promise; + // The command's output publishes once the timeline commits it. + await manager.flush(); expect( events.flatMap((event) => @@ -11281,7 +11316,8 @@ test("authoritative timeline records a daemon-handled submitted prompt before it { type: "assistant_message", text: "Handled by the daemon" }, ]); - const timeline = manager.fetchTimeline(snapshot.id, { direction: "tail", limit: 20 }).rows; + const timeline = (await manager.fetchTimeline(snapshot.id, { direction: "tail", limit: 20 })) + .rows; expect(timeline.map((row) => row.item)).toEqual([ { type: "user_message", @@ -11695,7 +11731,7 @@ test("system-injected prompts appear as trigger tools in the live timeline", asy await manager.runAgent(snapshot.id, { text: "do something" }); - const timeline = manager.getTimeline(snapshot.id); + const timeline = await manager.getTimeline(snapshot.id); expect(timeline).toContainEqual( expect.objectContaining({ type: "tool_call", name: "heartbeat", status: "completed" }), ); @@ -11739,7 +11775,7 @@ test("system-injected prompts appear as trigger tools during history replay", as await manager.hydrateTimelineFromProvider(snapshot.id, { force: true }); - const timeline = manager.getTimeline(snapshot.id); + const timeline = await manager.getTimeline(snapshot.id); expect(timeline).toContainEqual( expect.objectContaining({ type: "tool_call", name: "heartbeat", status: "completed" }), ); @@ -11985,10 +12021,10 @@ test("commits startup notices once on create and after restored history", async return new NotifyingSession({ provider: "codex", cwd: workdir }); } })(); - const store = new RecordingTimelineStore(); + const backend = new DeferredTimelineBackend(); const manager = new AgentManager({ clients: { codex: client }, - durableTimelineStore: store, + timeline: new AgentTimeline(backend), logger, }); const ids: string[] = []; @@ -12015,7 +12051,7 @@ test("commits startup notices once on create and after restored history", async notice, ]); await manager.flush(); - expect(store.writes.flat()).toContainEqual(expect.objectContaining({ item: notice })); + expect(backend.appends.flat()).toContainEqual(expect.objectContaining({ item: notice })); } finally { for (const id of ids) await manager.closeAgent(id); rmSync(workdir, { recursive: true, force: true }); @@ -12113,7 +12149,7 @@ test.each(["live", "history"])( }); if (path === "live") await manager.runAgent(agent.id, { text: "start" }); else await manager.hydrateTimelineFromProvider(agent.id, { force: true }); - expect(manager.getTimeline(agent.id)).toContainEqual( + expect(await manager.getTimeline(agent.id)).toContainEqual( expect.objectContaining({ type: "tool_call", callId: "paseo-agent-message:delivery-1", @@ -12131,9 +12167,9 @@ test.each(["live", "history"])( }, }), ); - expect(manager.getTimeline(agent.id).filter((entry) => entry.type === "user_message")).toEqual( - [], - ); + expect( + (await manager.getTimeline(agent.id)).filter((entry) => entry.type === "user_message"), + ).toEqual([]); expect(manager.getAgent(agent.id)?.lastUserMessageAt).toBeNull(); }, ); @@ -12168,15 +12204,17 @@ test.each([false, true])( clientMessageId: delivery.messageId, })) { } - expect(manager.getTimeline(agent.id).filter((item) => item.type === "tool_call")).toEqual([ + expect( + (await manager.getTimeline(agent.id)).filter((item) => item.type === "tool_call"), + ).toEqual([ expect.objectContaining({ callId: "paseo-agent-message:accepted-delivery", status: "completed", }), ]); - expect(manager.getTimeline(agent.id).filter((item) => item.type === "user_message")).toEqual( - [], - ); + expect( + (await manager.getTimeline(agent.id)).filter((item) => item.type === "user_message"), + ).toEqual([]); expect(manager.getAgent(agent.id)?.lastUserMessageAt).toBeNull(); } finally { await manager.closeAgent(agent.id); @@ -12211,7 +12249,9 @@ test("a rejected agent prompt never becomes a delivered notification", async () clientMessageId: delivery.messageId, }); await expect(run.next()).rejects.toThrow("provider rejected delivery"); - expect(manager.getTimeline(agent.id).filter((item) => item.type === "tool_call")).toEqual([]); + expect( + (await manager.getTimeline(agent.id)).filter((item) => item.type === "tool_call"), + ).toEqual([]); expect(manager.getAgent(agent.id)?.lastUserMessageAt).toBeNull(); } finally { await manager.closeAgent(agent.id); @@ -12248,7 +12288,7 @@ test.each([false, true])( clientMessageId: delivery.messageId, })) { } - expect(manager.getTimeline(agent.id)).toEqual([ + expect(await manager.getTimeline(agent.id)).toEqual([ { type: "user_message", text: example, @@ -12295,22 +12335,15 @@ test.each([ ])( "lifecycle hooks receive no conversation for $event.type and stream dispatch never reads timeline items", async ({ event: terminalEvent, outcome }) => { - class CountingTimelineStore extends InMemoryAgentTimelineStore { - itemReads = 0; - override getItems(agentId: string): AgentTimelineItem[] { - this.itemReads += 1; - return super.getItems(agentId); - } - } const workdir = mkdtempSync(join(tmpdir(), "agent-hook-payload-")); const client = new SessionRecordingAgentClient(); - const timelineStore = new CountingTimelineStore(); + const timelineStore = new DeferredTimelineBackend(); const events: Array<{ name: string; event: unknown }> = []; const manager = new AgentManager({ clients: { codex: client }, registry: new AgentStorage(join(workdir, "agents"), logger), logger, - timelineStore, + timeline: new AgentTimeline(timelineStore), pluginLifecycle: { before: async (_name, request) => request, emit: (name, event) => { @@ -12330,14 +12363,14 @@ test.each([ type: "assistant_message", text: "A long conversation".repeat(10_000), }); - expect(manager.getTimeline(agent.id)).toHaveLength(1); - const readsBeforeDispatch = timelineStore.itemReads; + expect(await manager.getTimeline(agent.id)).toHaveLength(1); + const readsBeforeDispatch = timelineStore.readRowsCalls; const session = client.sessions[0]!; session.pushEvent({ type: "turn_started", provider: "codex", turnId: "hook-turn" }); await expect .poll(() => events.filter(({ name }) => name === "agent.turn.before").length) .toBe(1); - expect(timelineStore.itemReads).toBe(readsBeforeDispatch); + expect(timelineStore.readRowsCalls).toBe(readsBeforeDispatch); session.pushEvent(terminalEvent); await expect .poll(() => events.filter(({ name }) => name === "agent.turn.after")) @@ -12359,7 +12392,7 @@ test.each([ }, }, ]); - expect(timelineStore.itemReads).toBe(readsBeforeDispatch); + expect(timelineStore.readRowsCalls).toBe(readsBeforeDispatch); }, ); @@ -12408,8 +12441,7 @@ test.each(["live", "history", "accepted"] as const)( } } else await manager.runAgent(agent.id, { text: "start" }); expect( - manager - .getTimeline(agent.id) + (await manager.getTimeline(agent.id)) .filter((row) => row.type === "tool_call") .map((row) => ({ callId: row.callId, agentMessage: row.agentMessage })), ).toEqual([ @@ -12430,9 +12462,9 @@ test.each(["live", "history", "accepted"] as const)( }, }, ]); - expect(manager.getTimeline(agent.id).filter((row) => row.type === "user_message")).toEqual( - [], - ); + expect( + (await manager.getTimeline(agent.id)).filter((row) => row.type === "user_message"), + ).toEqual([]); expect(manager.getAgent(agent.id)?.lastUserMessageAt).toBeNull(); } finally { await manager.closeAgent(agent.id); @@ -12442,17 +12474,21 @@ test.each(["live", "history", "accepted"] as const)( ); test("reads the last durable assistant response without loading a stored-only child", async () => { - const store = new RecordingTimelineStore(); - await store.appendCommitted("stored-child", { - type: "assistant_message", - text: "Partial work saved before restart", - }); - const manager = new AgentManager({ clients: {}, durableTimelineStore: store, logger }); + const backend = new DeferredTimelineBackend(); + // History a previous daemon process committed. + await backend.create( + { agentId: "stored-child" }, + { + entries: [{ item: { type: "assistant_message", text: "Partial work saved before restart" } }], + }, + ); + const manager = new AgentManager({ clients: {}, timeline: new AgentTimeline(backend), logger }); expect(manager.getAgent("stored-child")).toBeNull(); expect(await manager.getLastAssistantMessage("stored-child")).toBe( "Partial work saved before restart", ); expect(manager.getAgent("stored-child")).toBeNull(); + expect(backend.readRowsCalls).toBe(0); }); function runtimeCleanup(manager: AgentManager) { diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index a761e723c1..69694738d3 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -2,7 +2,6 @@ import type { AttachedAgentTool, PluginAgentToolsRuntime } from "../plugins/agen import type { AgentToolResult } from "@getpaseo/plugin/server"; import type { RuntimeCleanupCandidate } from "./idle-runtime-cleanup/index.js"; import { ensureAgentLoaded } from "./agent-loading.js"; -import { projectTimelineRows } from "./timeline-projection.js"; import type { PluginLifecycle } from "../plugins/lifecycle/index.js"; import { describeHookAgent, publishAgentStream } from "../plugins/lifecycle/index.js"; import type { PluginSessionOpenRequest } from "@getpaseo/plugin/server"; @@ -62,16 +61,20 @@ import { buildArchivedAgentRecord, type ArchivedStoredAgentRecord } from "./agen import { toStoredAgentRecord } from "./agent-projections.js"; import type { StoredAgentRecord, AgentStorage } from "./agent-storage.js"; import type { AgentOwner } from "./agent-owner.js"; -import { - InMemoryAgentTimelineStore, - type SeedAgentTimelineOptions, -} from "./agent-timeline-store.js"; import type { AgentTimelineFetchOptions, AgentTimelineFetchResult, AgentTimelineRow, - AgentTimelineStore, } from "./agent-timeline-store-types.js"; +import { + AgentTimeline, + type TimelineAppendResult, + type TimelineCommit, + type TimelineEntry, + type TimelineIngestResult, + type TimelineScope, + type TimelineSeed, +} from "./timeline/index.js"; import { AGENT_STREAM_COALESCE_DEFAULT_WINDOW_MS, AgentStreamCoalescer, @@ -238,7 +241,13 @@ export type AgentManagerEvent = | { type: "agent_state"; agent: ManagedAgent } | { type: "agent_removed"; agentId: string; workspaceId?: string; internal: boolean } | { type: "provider_subagent"; event: ProviderSubagentStoreEvent } - | { type: "timeline_replacement"; agentId: string; epoch: string } + | { + type: "timeline_replacement"; + agentId: string; + epoch: string; + /** The whole replacement history, for clients that cannot refetch it. */ + rows: AgentTimelineFetchResult["rows"]; + } | { type: "agent_stream"; agentId: string; @@ -335,8 +344,7 @@ export interface AgentManagerOptions { registry?: AgentStorage; onAgentAttention?: AgentAttentionCallback; onWorkspaceStateMayHaveChanged?: (params: { cwd: string }) => void; - timelineStore?: InMemoryAgentTimelineStore; - durableTimelineStore?: AgentTimelineStore; + timeline?: AgentTimeline; terminalManager?: TerminalManager | null; mcpBaseUrl?: string; mcpAuthToken?: string; @@ -444,6 +452,11 @@ interface ManagedAgentBase { activeTurnStartedAt: Date | null; lastUsage?: AgentUsage; lastError?: string; + /** + * Set when a timeline write the run did not wait for failed, so the history is missing items. + * Kept as `lastError` and an error lifecycle through the turn's end; the next run clears it. + */ + historySaveError?: string; attention: AttentionState; foregroundTurnWaiters: Set; finalizedForegroundTurnIds: Set; @@ -565,6 +578,7 @@ interface AgentMetadataPatch { } const SYSTEM_ERROR_PREFIX = "[System Error]"; +const HISTORY_SAVE_ERROR_PREFIX = "Conversation history could not be saved"; function normalizePersistenceForConfig( handle: AgentPersistenceHandle | null, @@ -654,7 +668,7 @@ function buildExplicitTimelineSeedForRegister( updatedAt?: Date; } | undefined, -): SeedAgentTimelineOptions | null { +): TimelineSeed | null { const hasTimeline = Boolean(options?.timeline?.length); const hasTimelineRows = Boolean(options?.timelineRows?.length); const hasTimelineNextSeq = options?.timelineNextSeq !== undefined; @@ -662,13 +676,17 @@ function buildExplicitTimelineSeedForRegister( return null; } return { - items: options?.timeline, + entries: options?.timeline?.map((item) => ({ item })), rows: options?.timelineRows, nextSeq: options?.timelineNextSeq, timestamp: (options?.updatedAt ?? options?.createdAt ?? now).toISOString(), }; } +function toTimelineEntry(event: Extract): TimelineEntry { + return event.timestamp ? { item: event.item, timestamp: event.timestamp } : { item: event.item }; +} + function buildImportedTimelineRows(entries: readonly ImportedTimelineEntry[]): AgentTimelineRow[] { const rows: AgentTimelineRow[] = []; for (const entry of entries) { @@ -756,8 +774,8 @@ export class AgentManager { private readonly providerEnabled = new Map(); private readonly providerDefinitions = new Map(); private readonly agents = new Map(); - private readonly timelineStore: InMemoryAgentTimelineStore; - private readonly providerSubagents = new ProviderSubagentStore(); + private readonly timeline: AgentTimeline; + private readonly providerSubagents: ProviderSubagentStore; private readonly agentsAwaitingInitialSnapshotPersist = new Set(); private readonly sessionEventTails = new Map>(); private readonly steerEventBarriers = new Map(); @@ -766,7 +784,6 @@ export class AgentManager { private readonly subscribers = new Set(); private readonly idFactory: () => string; private readonly registry?: AgentStorage; - private readonly durableTimelineStore?: AgentTimelineStore; private readonly previousStatuses = new Map(); private readonly backgroundTasks = new Set>(); private readonly agentRegistrationTasks = new Set>(); @@ -800,10 +817,10 @@ export class AgentManager { constructor(options: AgentManagerOptions) { this.pluginLifecycle = options.pluginLifecycle; this.pluginAgentTools = options.pluginAgentTools; - this.timelineStore = options.timelineStore ?? new InMemoryAgentTimelineStore(); + this.timeline = options.timeline ?? new AgentTimeline(); + this.providerSubagents = new ProviderSubagentStore(this.timeline); this.idFactory = options.idFactory ?? (() => randomUUID()); this.registry = options?.registry; - this.durableTimelineStore = options?.durableTimelineStore; this.onAgentAttention = options?.onAgentAttention; this.onWorkspaceStateMayHaveChanged = options?.onWorkspaceStateMayHaveChanged; this.mcpBaseUrl = options?.mcpBaseUrl ?? null; @@ -823,8 +840,12 @@ export class AgentManager { windowMs: options.agentStreamCoalesceWindowMs ?? AGENT_STREAM_COALESCE_DEFAULT_WINDOW_MS, timers: { setTimeout, clearTimeout }, onFlush: ({ agentId, item, provider, turnId }) => { - const event = this.recordAndDispatchTimelineItem(agentId, item, provider, turnId); - this.notifyForegroundTurnWaiters(agentId, event); + this.trackTimelineWrite( + agentId, + this.recordAndDispatchTimelineItem(agentId, item, provider, turnId, { + notifyWaiters: true, + }), + ); }, }); this.updateProviderRegistry({ @@ -991,11 +1012,11 @@ export class AgentManager { withActiveForegroundTurn++; } - if (!this.timelineStore.has(agent.id)) { + const len = this.timeline.residentSize({ agentId: agent.id }); + if (len === null) { continue; } - const len = this.timelineStore.getItems(agent.id).length; totalItems += len; if (len > maxItemsPerAgent) { maxItemsPerAgent = len; @@ -1291,25 +1312,23 @@ export class AgentManager { await this.inFlightAgentCloses?.get(agentId)?.catch(() => undefined); } - getTimeline(id: string): AgentTimelineItem[] { + async getTimeline(id: string): Promise { this.requireAgent(id); - return this.timelineStore.getItems(id); + const rows = await this.timeline.readRows({ agentId: id }); + return rows.map((row) => row.item); } async getTimelineRows(id: string): Promise { this.requireAgent(id); - if (this.durableTimelineStore) { - return projectTimelineRows({ - rows: await this.durableTimelineStore.getCommittedRows(id), - mode: "projected", - }).map((entry) => Object.assign({ seq: entry.seqEnd }, entry)); - } - return this.timelineStore.getRows(id); + return await this.timeline.readRows({ agentId: id }); } - fetchTimeline(id: string, options?: AgentTimelineFetchOptions): AgentTimelineFetchResult { + async fetchTimeline( + id: string, + options?: AgentTimelineFetchOptions, + ): Promise { this.requireAgent(id); - return this.timelineStore.fetch(id, options); + return await this.timeline.fetch({ agentId: id }, options); } listProviderSubagents(parentAgentId: string): ProviderSubagentDescriptor[] { @@ -1336,11 +1355,11 @@ export class AgentManager { return this.providerSubagents.get(parentAgentId, subagentId); } - fetchProviderSubagentTimeline( + async fetchProviderSubagentTimeline( parentAgentId: string, subagentId: string, options?: AgentTimelineFetchOptions, - ): AgentTimelineFetchResult { + ): Promise { this.requirePublicAgent(parentAgentId); return this.providerSubagents.fetchTimeline(parentAgentId, subagentId, options); } @@ -1611,7 +1630,7 @@ export class AgentManager { publishWhenReady: true, }); for (const event of imported.providerSubagentEvents ?? []) { - const update = this.providerSubagents.apply(agent.id, event.provider, event.event); + const update = await this.providerSubagents.apply(agent.id, event.provider, event.event); this.dispatch({ type: "provider_subagent", event: update }); } return agent; @@ -1700,8 +1719,8 @@ export class AgentManager { // A persisted thread can have only one writer, even when its turn is idle. await this.closeReloadedSession(existing.session, agentId); await this.drainSessionEvents(agentId); - this.cancelRunningProviderSubagents(agentId); - closedExisting = this.prepareAgentForClosure(existing, "agent reloaded"); + await this.cancelRunningProviderSubagents(agentId); + closedExisting = await this.prepareAgentForClosure(existing, "agent reloaded"); await this.persistSnapshot(closedExisting); this.assertAcceptingAgentRegistrations(); @@ -1715,8 +1734,8 @@ export class AgentManager { if (rehydrateFromDisk) { // Wipe the in-memory timeline so registerSession mints a new epoch and // hydrateTimelineFromProvider re-streams the freshly read provider history. - this.timelineStore.delete(agentId); - for (const event of this.providerSubagents.deleteParent(agentId)) { + await this.timeline.delete({ agentId }); + for (const event of await this.providerSubagents.deleteParent(agentId)) { this.dispatch({ type: "provider_subagent", event }); } } @@ -1947,8 +1966,8 @@ export class AgentManager { // Retain ownership until shutdown succeeds. A failed close may still own a // native writer, so publishing a resumable closed snapshot would orphan it. await agent.session.close(); - this.cancelRunningProviderSubagents(agentId); - const closedAgent = this.prepareAgentForClosure(agent, "agent closed"); + await this.cancelRunningProviderSubagents(agentId); + const closedAgent = await this.prepareAgentForClosure(agent, "agent closed"); let persistError: unknown; try { @@ -1973,12 +1992,12 @@ export class AgentManager { return true; } - private cancelRunningProviderSubagents(parentAgentId: string): void { + private async cancelRunningProviderSubagents(parentAgentId: string): Promise { for (const subagent of this.providerSubagents.list(parentAgentId)) { if (subagent.status !== "running") { continue; } - const event = this.providerSubagents.apply(parentAgentId, subagent.provider, { + const event = await this.providerSubagents.apply(parentAgentId, subagent.provider, { type: "upsert", id: subagent.id, status: "canceled", @@ -2029,7 +2048,7 @@ export class AgentManager { agent.updatedAt = new Date(archivedAt); await this.closeAgentRuntime(agentId); await this.syncNativeArchiveState(stored.provider, stored.persistence, "archive"); - this.discardRetainedAgentState(agentId); + await this.discardRetainedAgentState(agentId); await this.cascadeArchiveChildren(stored); @@ -2464,7 +2483,7 @@ export class AgentManager { await this.syncNativeArchiveState(record.provider, record.persistence, "archive"); - this.discardRetainedAgentState(agentId); + await this.discardRetainedAgentState(agentId); if (!nextRecord.internal) this.dispatchStoredAgentState(nextRecord); await this.fireAgentArchived(agentId); @@ -2648,23 +2667,32 @@ export class AgentManager { } this.touchUpdatedAt(agent); if (options?.clientMessageId) { - this.recordSubmittedPrompt(agent, prompt, options.clientMessageId, "accepted"); + await this.recordSubmittedPrompt(agent, prompt, options.clientMessageId, "accepted"); this.emitState(agent); } + const scope: TimelineScope = { agentId: agent.id }; const dispatch = (event: AgentStreamEvent): void => { // Persist timeline items so they show up in fetchAgentTimeline; broadcast - // for live subscribers. Other event types are broadcast only. + // for live subscribers. Other event types are broadcast only, after the + // timeline items emitted before them. if (event.type === "timeline") { this.touchUpdatedAt(agent); - const row = this.recordTimeline(agent.id, event.item); - this.dispatchStream(agent.id, event, { - seq: row.seq, - epoch: this.timelineStore.getEpoch(agent.id), - timestamp: row.timestamp, - }); + const write = this.timeline + .ingest( + scope, + { item: event.item }, + { publish: (commit) => this.publishTimelineEvent(agent.id, event, commit) }, + ) + .then((result) => this.reportUnpublished(agent.id, result)); + this.trackTimelineWrite(agent.id, write); return; } - this.dispatchStream(agent.id, event, { timestamp: new Date().toISOString() }); + const timestamp = new Date().toISOString(); + const publish = (async () => { + await this.timeline.settled(scope); + this.dispatchStream(agent.id, event, { timestamp }); + })(); + this.trackTimelineWrite(agent.id, publish); }; void this.runForegroundMutation(agentId, async () => { try { @@ -2688,22 +2716,15 @@ export class AgentManager { const agent = this.requireAgent(agentId); item = limitAgentTimelineItemContent(item); this.touchUpdatedAt(agent); - const row = this.recordTimeline(agentId, item); - this.dispatchStream( - agentId, - { - type: "timeline", - item, - provider: agent.provider, - }, - { - seq: row.seq, - epoch: this.timelineStore.getEpoch(agentId), - timestamp: row.timestamp, - }, + const event: AgentStreamEvent = { type: "timeline", item, provider: agent.provider }; + const result = await this.timeline.ingest( + { agentId }, + { item }, + { publish: (commit) => this.publishTimelineEvent(agentId, event, commit) }, ); + this.reportUnpublished(agentId, result); await this.persistSnapshot(agent); - return { seq: row.seq, epoch: this.timelineStore.getEpoch(agentId) }; + return { seq: result.row.seq, epoch: result.epoch }; } async emitLiveTimelineItem(agentId: string, item: AgentTimelineItem): Promise { @@ -2747,7 +2768,7 @@ export class AgentManager { try { // A terminal rejection belongs after the submitted prompt even though no provider turn exists. if (options?.clientMessageId) { - this.recordSubmittedPrompt(agent, prompt, options.clientMessageId, "rejected", { + await this.recordSubmittedPrompt(agent, prompt, options.clientMessageId, "rejected", { messageId: options.clientMessageId, }); } @@ -2819,6 +2840,7 @@ export class AgentManager { const agent = existingAgent; const isReplacement = agent.pendingReplacement; agent.lastError = undefined; + agent.historySaveError = undefined; const pendingRun = this.runs.createPendingRun(agentId); @@ -2859,14 +2881,24 @@ export class AgentManager { ) : undefined; if (options?.clientMessageId) { - this.recordSubmittedPrompt(agent, prompt, options.clientMessageId, "accepted", { - messageId: options.clientMessageId, - turnId, - providerMessageId: - stagedSubmittedPromptEcho?.item.type === "user_message" - ? stagedSubmittedPromptEcho.item.messageId - : undefined, - }); + // Not awaited: the start is already settled, so awaiting would let live provider events + // run ahead of the staged ones below. The timeline commits writes in call order, and + // stream handling waits for it before publishing anything else. + const write = this.recordSubmittedPrompt( + agent, + prompt, + options.clientMessageId, + "accepted", + { + messageId: options.clientMessageId, + turnId, + providerMessageId: + stagedSubmittedPromptEcho?.item.type === "user_message" + ? stagedSubmittedPromptEcho.item.messageId + : undefined, + }, + ); + this.trackTimelineWrite(agent.id, write); } for (const stagedEvent of pendingRun.stagedEvents.splice(0)) { const isAcceptedTurnStart = @@ -3104,6 +3136,7 @@ export class AgentManager { return this.runForegroundMutation(agent.id, async () => { await this.drainSessionEvents(agent.id); this.agentStreamCoalescer.flushFor(agent.id); + await this.timeline.settled({ agentId: agent.id }); this.assertSteerAdmissionOwnsTurn(agent, expectedTurnId); const barrier: SteerEventBarrier = { events: [] }; this.steerEventBarriers.set(agent.id, barrier); @@ -3171,7 +3204,7 @@ export class AgentManager { if (!clientMessageId) { return; } - this.recordSubmittedPrompt(agent, prompt, clientMessageId, "accepted", { + await this.recordSubmittedPrompt(agent, prompt, clientMessageId, "accepted", { messageId: clientMessageId, turnId: expectedTurnId, }); @@ -3184,7 +3217,12 @@ export class AgentManager { throw new Error(`Agent ${agentId} not found`); } + // A failed start is final even while its failure is still being recorded and the + // lifecycle still reads "running". const pendingRun = this.runs.getPendingRun(agentId); + if (pendingRun?.start.status === "failed") { + throw new Error(pendingRun.start.error); + } if ( (snapshot.lifecycle === "running" || pendingRun?.start.status === "started") && !snapshot.pendingReplacement @@ -3252,6 +3290,11 @@ export class AgentManager { } const currentPendingRun = this.runs.getPendingRun(agentId); + if (currentPendingRun?.start.status === "failed") { + finishErr(new Error(currentPendingRun.start.error)); + return true; + } + if ( (current.lifecycle === "running" || currentPendingRun?.start.status === "started") && !current.pendingReplacement @@ -3260,11 +3303,6 @@ export class AgentManager { return true; } - if (currentPendingRun?.start.status === "failed") { - finishErr(new Error(currentPendingRun.start.error)); - return true; - } - if (current.lifecycle === "error" && !currentPendingRun) { finishErr(new Error(current.lastError ?? `Agent ${agentId} failed to start`)); return true; @@ -3459,24 +3497,19 @@ export class AgentManager { async rewind(agentId: string, messageId: string, mode: RewindMode): Promise { const agent = this.requireSessionAgent(agentId); - const submittedRow = this.timelineStore - .getRows(agentId) - .find( - (row) => - row.item.type === "user_message" && - row.item.messageId === messageId && - row.item.clientMessageId === messageId, - ); - if (submittedRow && !submittedRow.providerMessageId) { - throw new Error("Cannot rewind before the provider acknowledges the submitted prompt"); - } - const providerMessageId = submittedRow?.providerMessageId ?? messageId; - - if (this.hasInFlightRun(agentId)) { - await this.cancelAgentRunBefore(agentId, "rewind"); + // Without an active run, the rewind holds the run from its call, so no prompt starts + // while it reads the history. + const heldFromCall = this.hasInFlightRun(agentId) ? null : this.runs.createPendingRun(agentId); + let providerMessageId: string; + try { + providerMessageId = await this.resolveRewindTarget(agentId, messageId); + if (!heldFromCall) await this.cancelAgentRunBefore(agentId, "rewind"); + } catch (error) { + if (heldFromCall) this.runs.settleForegroundRun(agentId, heldFromCall.token); + throw error; } - const lock = this.runs.createPendingRun(agentId); + const lock = heldFromCall ?? this.runs.createPendingRun(agentId); try { this.logger.info( { agentId, provider: agent.provider, messageId, mode }, @@ -3489,10 +3522,12 @@ export class AgentManager { broadcast: true, broadcastTimeline: false, }); + const replacement = await this.timeline.fetch({ agentId }, { limit: 0 }); this.dispatch({ type: "timeline_replacement", agentId, - epoch: this.timelineStore.getEpoch(agentId), + epoch: replacement.epoch, + rows: replacement.rows, }); } // Rewind stages provider events under the run lock; publish its final state directly. @@ -3515,9 +3550,20 @@ export class AgentManager { } } + /** The provider's id for the rewound prompt. Paseo-submitted prompts need its acknowledgement. */ + private async resolveRewindTarget(agentId: string, messageId: string): Promise { + const submittedRow = await this.timeline.findSubmittedPrompt({ agentId }, messageId); + if (submittedRow?.item.type !== "user_message" || submittedRow.item.messageId !== messageId) { + return messageId; + } + if (!submittedRow.providerMessageId) { + throw new Error("Cannot rewind before the provider acknowledges the submitted prompt"); + } + return submittedRow.providerMessageId; + } + async deleteAgentState(agentId: string): Promise { - this.discardRetainedAgentState(agentId); - await this.deleteCommittedTimeline(agentId); + await this.discardRetainedAgentState(agentId); } notifyAgentRemoved(agent: Pick): void { @@ -3529,29 +3575,19 @@ export class AgentManager { }); } - async deleteCommittedTimeline(agentId: string): Promise { - await this.durableTimelineStore?.deleteAgent(agentId); - } - async getLastAssistantMessage(agentId: string): Promise { - if (!this.timelineStore.has(agentId)) { - return (await this.durableTimelineStore?.getLastAssistantMessage(agentId)) ?? null; + const scope: TimelineScope = { agentId }; + if (!(await this.timeline.describe(scope))) { + return null; } - return await this.getLastAssistantMessageFromStores(agentId); + return await this.timeline.lastAssistantMessage(scope); } private getLastAssistantMessageFromTimeline( timeline: readonly AgentTimelineItem[], ): string | null { - return this.getLastAssistantMessageSegmentFromTimeline(timeline)?.text ?? null; - } - - private getLastAssistantMessageSegmentFromTimeline( - timeline: readonly AgentTimelineItem[], - ): { text: string; startsAtBeginning: boolean } | null { // Collect the last contiguous assistant messages (Claude streams chunks) const chunks: string[] = []; - let startsAtBeginning = false; for (let i = timeline.length - 1; i >= 0; i--) { const item = timeline[i]; if (item.type !== "assistant_message") { @@ -3561,42 +3597,8 @@ export class AgentManager { continue; } chunks.push(item.text); - startsAtBeginning = i === 0; } - - if (!chunks.length) { - return null; - } - - return { - text: chunks.toReversed().join(""), - startsAtBeginning, - }; - } - - private async getLastAssistantMessageFromStores(agentId: string): Promise { - const liveTimeline = this.timelineStore.getItems(agentId); - const liveSegment = this.getLastAssistantMessageSegmentFromTimeline(liveTimeline); - if (!this.durableTimelineStore) { - return liveSegment?.text ?? null; - } - if (!liveSegment) { - return await this.durableTimelineStore.getLastAssistantMessage(agentId); - } - if (!liveSegment.startsAtBeginning) { - return liveSegment.text; - } - const lastDurableItem = await this.durableTimelineStore.getLastItem(agentId); - if (lastDurableItem?.type !== "assistant_message") { - return liveSegment.text; - } - const durableMessage = await this.durableTimelineStore.getLastAssistantMessage(agentId); - return durableMessage ? `${durableMessage}${liveSegment.text}` : liveSegment.text; - } - - private async getLastItemFromStores(agentId: string): Promise { - const lastLiveItem = this.timelineStore.getLastItem(agentId); - return lastLiveItem ?? (await this.durableTimelineStore?.getLastItem(agentId)) ?? null; + return chunks.length ? chunks.toReversed().join("") : null; } async waitForAgentEvent( @@ -3691,10 +3693,13 @@ export class AgentManager { } finished = true; cleanup(); + // The status is the one that finished the wait, not whatever arrives while the + // last message is read. + const status = currentStatus; void this.getLastAssistantMessage(agentId) .then((lastMessage) => { resolvePromise({ - status: currentStatus, + status, permission, lastMessage, }); @@ -3757,6 +3762,8 @@ export class AgentManager { }, { agentId, replayState: true }, ); + // The replayed state can finish the wait before `subscribe` returns. + if (finished) cleanup(); }); } @@ -3804,7 +3811,7 @@ export class AgentManager { ); const now = new Date(); - const { durableTimelineHasRows } = await this.initializeAgentTimelineForRegister({ + const { timelineWasPresent } = await this.initializeAgentTimelineForRegister({ agentId: resolvedAgentId, now, options, @@ -3815,7 +3822,7 @@ export class AgentManager { session, config, now, - durableTimelineHasRows, + timelineWasPresent, options, }); @@ -3838,9 +3845,13 @@ export class AgentManager { // Legacy/imported chats need their existing history before startup rows. await this.primeTimelineFromLegacyProviderHistory(managed, false, startupHistory); } else { - for (const entry of buildImportedTimelineRows(session.initialTimeline)) { - this.recordTimeline(managed.id, entry.item, { timestamp: entry.timestamp }); - } + await this.timeline.append( + { agentId: managed.id }, + buildImportedTimelineRows(session.initialTimeline).map((row) => ({ + item: row.item, + timestamp: row.timestamp, + })), + ); } this.refreshSessionPersistence(managed); } @@ -3932,26 +3943,20 @@ export class AgentManager { updatedAt?: Date; } | undefined; - }): Promise<{ durableTimelineHasRows: boolean }> { + }): Promise<{ timelineWasPresent: boolean }> { const { agentId, now, options } = params; - const timelineAlreadyPrimed = this.timelineStore.has(agentId); + const scope: TimelineScope = { agentId }; const explicitTimelineSeed = buildExplicitTimelineSeedForRegister(now, options); - const shouldSeedFromDurable = - !explicitTimelineSeed && !this.timelineStore.has(agentId) && this.durableTimelineStore; - const durableTimelineSeed = shouldSeedFromDurable - ? await this.loadCommittedTimelineSeed(agentId, now) - : null; - const durableTimelineHasRows = - timelineAlreadyPrimed || - (durableTimelineSeed != null && (durableTimelineSeed.nextSeq ?? 1) > 1); - const timelineSeed = explicitTimelineSeed ?? durableTimelineSeed; - if (timelineSeed || !this.timelineStore.has(agentId)) { - this.timelineStore.initialize(agentId, timelineSeed ?? { timestamp: now.toISOString() }); + if (explicitTimelineSeed) { + const timelineWasPresent = (await this.timeline.describe(scope)) !== null; + await this.timeline.create(scope, explicitTimelineSeed); + return { timelineWasPresent }; } - if (options?.timelineRows?.length) { - this.enqueueDurableTimelineBulkInsert(agentId, options.timelineRows); + const existing = await this.timeline.describe(scope); + if (!existing) { + await this.timeline.create(scope, { timestamp: now.toISOString() }); } - return { durableTimelineHasRows }; + return { timelineWasPresent: existing !== null }; } private buildManagedAgentForRegister(params: { @@ -3959,7 +3964,7 @@ export class AgentManager { session: AgentSession; config: AgentSessionConfig; now: Date; - durableTimelineHasRows: boolean; + timelineWasPresent: boolean; options: | { createdAt?: Date; @@ -3977,7 +3982,7 @@ export class AgentManager { } | undefined; }): ActiveManagedAgent { - const { resolvedAgentId, session, config, now, durableTimelineHasRows, options } = params; + const { resolvedAgentId, session, config, now, timelineWasPresent, options } = params; return { id: resolvedAgentId, provider: config.provider, @@ -4008,7 +4013,7 @@ export class AgentManager { options?.persistence ?? session.describePersistence(), config, ), - historyPrimed: options?.historyPrimed ?? durableTimelineHasRows, + historyPrimed: options?.historyPrimed ?? timelineWasPresent, lastUserMessageAt: options?.lastUserMessageAt ?? null, lastUsage: options?.lastUsage, lastError: options?.lastError, @@ -4018,24 +4023,13 @@ export class AgentManager { } as ActiveManagedAgent; } - private async loadCommittedTimelineSeed( - agentId: string, - now: Date, - ): Promise { - if (!this.durableTimelineStore) { - return { timestamp: now.toISOString() }; - } - return { - nextSeq: (await this.durableTimelineStore.getLatestCommittedSeq(agentId)) + 1, - timestamp: now.toISOString(), - }; - } - - private prepareAgentForClosure( + private async prepareAgentForClosure( agent: LiveManagedAgent, cancelReason: string, - ): ManagedAgentClosed { + ): Promise { this.agentStreamCoalescer.flushAndDiscard(agent.id); + // The flushed output belongs before the closed state. + await this.timeline.settled({ agentId: agent.id }); this.agents.delete(agent.id); this.previousStatuses.delete(agent.id); if (agent.unsubscribeSession) { @@ -4066,11 +4060,11 @@ export class AgentManager { }; } - private discardRetainedAgentState(agentId: string): void { - this.timelineStore.delete(agentId); + private async discardRetainedAgentState(agentId: string): Promise { + await this.timeline.delete({ agentId }); this.paseoToolPolicies.delete(agentId); this.detachPluginTools(agentId); - for (const event of this.providerSubagents.deleteParent(agentId)) { + for (const event of await this.providerSubagents.deleteParent(agentId)) { this.dispatch({ type: "provider_subagent", event }); } } @@ -4178,7 +4172,7 @@ export class AgentManager { event: AgentStreamEvent, ): Promise { if (event.type === "provider_subagent") { - const update = this.providerSubagents.apply(agent.id, event.provider, event.event); + const update = await this.providerSubagents.apply(agent.id, event.provider, event.event); this.dispatch({ type: "provider_subagent", event: update }); return; } @@ -4366,36 +4360,31 @@ export class AgentManager { } this.agentStreamCoalescer.flushAndDiscard(agent.id); - await this.deleteCommittedTimeline(agent.id); - this.timelineStore.delete(agent.id); - this.timelineStore.initialize(agent.id, { timestamp: new Date().toISOString() }); + const scope: TimelineScope = { agentId: agent.id }; + // A new history identity: cursors from the replaced history read as stale. + await this.timeline.create(scope, { timestamp: new Date().toISOString() }); agent.historyPrimed = true; - for (const event of this.providerSubagents.deleteParent(agent.id)) { + for (const event of await this.providerSubagents.deleteParent(agent.id)) { if (broadcast) { this.dispatch({ type: "provider_subagent", event }); } } for (const event of providerSubagentEvents) { - const update = this.providerSubagents.apply(agent.id, event.provider, event.event); + const update = await this.providerSubagents.apply(agent.id, event.provider, event.event); if (broadcast) { this.dispatch({ type: "provider_subagent", event: update }); } } - for (const event of historyEvents) { - const row = this.recordTimeline( - agent.id, - event.item, - event.timestamp ? { timestamp: event.timestamp } : undefined, - ); - if (broadcastTimeline) { - this.dispatchStream(agent.id, event, { - seq: row.seq, - epoch: this.timelineStore.getEpoch(agent.id), - timestamp: row.timestamp, - }); - } - } + const replayed = await this.timeline.append(scope, historyEvents.map(toTimelineEntry), { + publish: (commits) => { + if (!broadcastTimeline) return; + commits.forEach((commit, index) => + this.publishTimelineEvent(agent.id, historyEvents[index], commit), + ); + }, + }); + this.reportUnpublished(agent.id, replayed); this.touchUpdatedAt(agent); this.emitState(agent); } @@ -4433,17 +4422,9 @@ export class AgentManager { throw error; } - // The replay is the timeline, so drop the rows a previous hydration committed. - // Keeping them would leave getTimelineRows reading one copy per hydration. - await this.deleteCommittedTimeline(agent.id); - - const timelineEvents: Array<{ - event: Extract; - row: AgentTimelineRow; - }> = []; const providerSubagentEvents: AgentManagerEvent[] = []; for (const event of historySubagentEvents) { - const update = this.providerSubagents.apply(agent.id, event.provider, event.event); + const update = await this.providerSubagents.apply(agent.id, event.provider, event.event); const managerEvent: AgentManagerEvent = { type: "provider_subagent", event: update }; if (deferredBroadcast) { providerSubagentEvents.push(managerEvent); @@ -4451,22 +4432,19 @@ export class AgentManager { this.dispatch(managerEvent); } } - for (const event of historyEvents) { - const row = this.recordTimeline( - agent.id, - event.item, - event.timestamp ? { timestamp: event.timestamp } : undefined, - ); - if (deferredBroadcast) { - timelineEvents.push({ event, row }); - } else if (broadcast) { - this.dispatchStream(agent.id, event, { - seq: row.seq, - epoch: this.timelineStore.getEpoch(agent.id), - timestamp: row.timestamp, - }); - } - } + const primed = await this.timeline.append( + { agentId: agent.id }, + historyEvents.map(toTimelineEntry), + { + publish: (committed) => { + if (deferredBroadcast || !broadcast) return; + committed.forEach((commit, index) => + this.publishTimelineEvent(agent.id, historyEvents[index], commit), + ); + }, + }, + ); + this.reportUnpublished(agent.id, primed); agent.historyPrimed = true; if (typeof broadcast !== "function" || !broadcast()) { @@ -4475,13 +4453,9 @@ export class AgentManager { for (const event of providerSubagentEvents) { this.dispatch(event); } - for (const { event, row } of timelineEvents) { - this.dispatchStream(agent.id, event, { - seq: row.seq, - epoch: this.timelineStore.getEpoch(agent.id), - timestamp: row.timestamp, - }); - } + primed.commits.forEach((commit, index) => + this.publishTimelineEvent(agent.id, historyEvents[index], commit), + ); } private notifyForegroundTurnWaiters(agentId: string, event: AgentStreamEvent): void { @@ -4535,6 +4509,8 @@ export class AgentManager { return false; } this.agentStreamCoalescer.flushFor(agent.id); + // A flushed chunk is committed and published before this event, e.g. a terminal status. + await this.timeline.settled({ agentId: agent.id }); } let terminalDisposition: ActiveTurnTerminalDisposition = "untracked"; @@ -4764,20 +4740,16 @@ export class AgentManager { if ( item.type === "user_message" && item.clientMessageId && - this.reconcileSubmittedPromptEcho(agent, item, event.turnId) + (await this.reconcileSubmittedPromptEcho(agent, item, event.turnId)) ) continue; if (options?.fromHistory) { - this.recordTimeline( - agent.id, - item, - event.timestamp ? { timestamp: event.timestamp } : undefined, - ); + await this.timeline.append({ agentId: agent.id }, [toTimelineEntry({ ...event, item })]); continue; } - this.recordAndDispatchTimelineItem(agent.id, item, event.provider, event.turnId); + await this.recordAndDispatchTimelineItem(agent.id, item, event.provider, event.turnId); if (item.type === "user_message") { agent.lastUserMessageAt = new Date(); this.emitState(agent); @@ -4812,14 +4784,14 @@ export class AgentManager { // If no usage on turn_completed, keep lastUsage as-is so context window // data accumulated during streaming isn't lost when the provider omits // it from the completion event. - agent.lastError = undefined; + agent.lastError = agent.historySaveError; if ( !isForegroundEvent && !agent.activeForegroundTurnId && agent.lifecycle !== "idle" && !agent.pendingReplacement ) { - (agent as ActiveManagedAgent).lifecycle = "idle"; + (agent as ActiveManagedAgent).lifecycle = agent.historySaveError ? "error" : "idle"; this.emitState(agent); } void this.refreshRuntimeInfo(agent); @@ -4893,9 +4865,9 @@ export class AgentManager { ); if (terminalDisposition === "stale") return; if (!isForegroundEvent && !agent.activeForegroundTurnId && !agent.pendingReplacement) { - agent.lifecycle = "idle"; + agent.lifecycle = agent.historySaveError ? "error" : "idle"; } - agent.lastError = undefined; + agent.lastError = agent.historySaveError; this.resolvePendingPermissionsForAgent(agent, event.provider, options, "Interrupted"); if (!isForegroundEvent && !agent.activeForegroundTurnId) { this.emitState(agent); @@ -4987,87 +4959,151 @@ export class AgentManager { } } - private recordAndDispatchTimelineItem( + private async recordAndDispatchTimelineItem( agentId: string, item: AgentTimelineItem, provider: AgentProvider, turnId?: string, - options?: { providerMessageId?: string }, - ): AgentStreamEvent { - const row = this.recordTimeline(agentId, item, { ...options, turnId }); + options?: { + providerMessageId?: string; + notifyWaiters?: boolean; + onRecorded?: () => void; + }, + ): Promise { const event: AgentStreamEvent = { type: "timeline", item, provider, ...(turnId !== undefined ? { turnId } : {}), }; - this.dispatchStream(agentId, event, { - seq: row.seq, - epoch: this.timelineStore.getEpoch(agentId), - timestamp: row.timestamp, - }); - - if ( - item.type === "tool_call" && - item.status === "completed" && - item.detail?.type === "shell" && - commandMayHaveChangedExternalState(item.detail.command) - ) { - const agent = this.agents.get(agentId); - if (agent) { - this.onWorkspaceStateMayHaveChanged?.({ cwd: agent.cwd }); - } - } - - return event; + const result = await this.timeline.ingest( + { agentId }, + { + item, + ...(turnId !== undefined ? { turnId } : {}), + ...(options?.providerMessageId ? { providerMessageId: options.providerMessageId } : {}), + }, + { + publish: (commit) => { + options?.onRecorded?.(); + this.publishTimelineEvent(agentId, event, commit); + if (options?.notifyWaiters) this.notifyForegroundTurnWaiters(agentId, event); + if ( + item.type === "tool_call" && + item.status === "completed" && + item.detail?.type === "shell" && + commandMayHaveChangedExternalState(item.detail.command) + ) { + const agent = this.agents.get(agentId); + if (agent) { + this.onWorkspaceStateMayHaveChanged?.({ cwd: agent.cwd }); + } + } + }, + }, + ); + this.reportUnpublished(agentId, result); } - private recordSubmittedPrompt( + private publishTimelineEvent(agentId: string, event: AgentStreamEvent, commit: TimelineCommit) { + this.dispatchStream(agentId, event, { + seq: commit.row.seq, + epoch: commit.epoch, + timestamp: commit.row.timestamp, + }); + } + + /** A commit whose publication threw stays committed; the failure is logged, not retried. */ + private reportUnpublished( + agentId: string, + result: TimelineIngestResult | TimelineAppendResult, + ): void { + if (result.kind === "unpublished") { + this.logger.error( + { err: result.error, agentId }, + "Failed to publish committed timeline item", + ); + } + } + + /** For writes that must not be awaited at the call site; `flush()` still drains them. */ + private trackTimelineWrite(agentId: string, write: Promise): void { + this.trackBackgroundTask( + write.then( + () => undefined, + (err: unknown) => { + this.logger.error({ err, agentId }, "Failed to record timeline item"); + this.reportHistorySaveFailure(agentId, err); + }, + ), + ); + } + + /** + * Surfaces an unsaved history item as an agent error. A running turn ends in the error + * lifecycle (see `historySaveError`); an idle agent enters it now. + */ + private reportHistorySaveFailure(agentId: string, err: unknown): void { + const agent = this.agents.get(agentId); + if (!agent) return; + const reason = err instanceof Error ? err.message : String(err); + agent.historySaveError = `${HISTORY_SAVE_ERROR_PREFIX}: ${reason}`; + agent.lastError = agent.historySaveError; + if (agent.lifecycle === "idle") { + (agent as ActiveManagedAgent).lifecycle = "error"; + } + this.emitState(agent); + } + + private async recordSubmittedPrompt( agent: ActiveManagedAgent, prompt: AgentPromptInput, clientMessageId: string, outcome: "accepted" | "rejected", options?: { messageId?: string; providerMessageId?: string; turnId?: string }, - ): void { + ): Promise { const items = projectAgentMessages({ type: "user_message", text: submittedPromptText(prompt), clientMessageId, ...(options?.messageId ? { messageId: options.messageId } : {}), }); + const writes: Promise[] = []; for (const item of items) { // Human attempts stay in history on rejection; delivery notifications require acceptance. if (outcome === "rejected" && item.type !== "user_message") continue; - this.recordSubmittedPromptItem(agent, item, options); + writes.push(this.recordSubmittedPromptItem(agent, item, options)); } + await Promise.all(writes); } + /** A prompt already recorded under its `clientMessageId` is not recorded again. */ private recordSubmittedPromptItem( agent: ActiveManagedAgent, item: AgentTimelineItem, options?: { providerMessageId?: string; turnId?: string }, - ): void { - if ( - item.type === "user_message" && - item.clientMessageId && - this.timelineStore.getSubmittedUserMessage(agent.id, item.clientMessageId) - ) - return; - this.touchUpdatedAt(agent); - if (item.type === "user_message") agent.lastUserMessageAt = new Date(); - this.recordAndDispatchTimelineItem(agent.id, item, agent.provider, options?.turnId, options); + ): Promise { + return this.recordAndDispatchTimelineItem(agent.id, item, agent.provider, options?.turnId, { + ...(options?.providerMessageId ? { providerMessageId: options.providerMessageId } : {}), + onRecorded: () => { + this.touchUpdatedAt(agent); + if (item.type === "user_message") agent.lastUserMessageAt = new Date(); + }, + }); } - private reconcileSubmittedPromptEcho( + /** Returns whether the echo belongs to a prompt Paseo submitted, which it records only once. */ + private async reconcileSubmittedPromptEcho( agent: ActiveManagedAgent, item: Extract, turnId?: string, - ): AgentTimelineRow | null { + ): Promise { const { clientMessageId, messageId } = item; - if (!clientMessageId) return null; - let existing = this.timelineStore.getSubmittedUserMessage(agent.id, clientMessageId); + if (!clientMessageId) return false; + const scope: TimelineScope = { agentId: agent.id }; + const existing = await this.timeline.findSubmittedPrompt(scope, clientMessageId); if (!existing) { - this.recordSubmittedPromptItem( + await this.recordSubmittedPromptItem( agent, { ...item, messageId: clientMessageId }, { @@ -5075,18 +5111,13 @@ export class AgentManager { ...(turnId ? { turnId } : {}), }, ); - existing = this.timelineStore.getSubmittedUserMessage(agent.id, clientMessageId); + return true; } - if (!existing || existing.item.type !== "user_message") return null; + if (existing.item.type !== "user_message") return false; if (messageId) { - const enriched = this.timelineStore.enrichSubmittedUserMessage( - agent.id, - clientMessageId, - messageId, - ); - if (enriched) this.enqueueDurableTimelineUpdate(agent.id, enriched); + await this.timeline.attachProviderMessageId(scope, clientMessageId, messageId); } - return existing; + return true; } private async appendSystemErrorTimelineMessage( @@ -5105,26 +5136,13 @@ export class AgentManager { } const text = `${SYSTEM_ERROR_PREFIX} ${normalized}`; - const lastItem = await this.getLastItemFromStores(agent.id); + const lastItem = await this.timeline.lastItem({ agentId: agent.id }); if (lastItem?.type === "assistant_message" && lastItem.text === text) { return; } const item: AgentTimelineItem = { type: "assistant_message", text }; - const row = this.recordTimeline(agent.id, item); - this.dispatchStream( - agent.id, - { - type: "timeline", - item, - provider, - }, - { - seq: row.seq, - epoch: this.timelineStore.getEpoch(agent.id), - timestamp: row.timestamp, - }, - ); + await this.recordAndDispatchTimelineItem(agent.id, item, provider); } private formatTurnFailedMessage( @@ -5143,21 +5161,6 @@ export class AgentManager { return parts.join("\n\n"); } - private recordTimeline( - agentId: string, - item: AgentTimelineItem, - options?: { - timestamp?: string; - providerMessageId?: string; - turnId?: string; - }, - ): AgentTimelineRow { - item = limitAgentTimelineItemContent(item); - const row = this.timelineStore.append(agentId, item, options); - this.enqueueDurableTimelineAppend(agentId, row); - return row; - } - private emitState(agent: ManagedAgent, options?: { persist?: boolean }): void { // Keep attention as an edge-triggered unread signal, not a level signal. this.checkAndSetAttention(agent); @@ -5240,46 +5243,6 @@ export class AgentManager { this.trackBackgroundTask(task); } - private enqueueDurableTimelineAppend(agentId: string, row: AgentTimelineRow): void { - if (!this.durableTimelineStore) { - return; - } - const task = this.durableTimelineStore.bulkInsert(agentId, [row]).catch((err) => { - this.logger.error( - { err, agentId, seq: row.seq, itemType: row.item.type }, - "Failed to append timeline row to durable store", - ); - }); - this.trackBackgroundTask(task); - } - - private enqueueDurableTimelineBulkInsert( - agentId: string, - rows: readonly AgentTimelineRow[], - ): void { - if (!this.durableTimelineStore || rows.length === 0) { - return; - } - const task = this.durableTimelineStore.bulkInsert(agentId, rows).catch((err) => { - this.logger.error( - { err, agentId, rowCount: rows.length }, - "Failed to seed durable timeline store", - ); - }); - this.trackBackgroundTask(task); - } - - private enqueueDurableTimelineUpdate(agentId: string, row: AgentTimelineRow): void { - if (!this.durableTimelineStore) return; - const task = this.durableTimelineStore.updateCommittedRow(agentId, row).catch((err) => { - this.logger.error( - { err, agentId, seq: row.seq, itemType: row.item.type }, - "Failed to enrich durable timeline row", - ); - }); - this.trackBackgroundTask(task); - } - private trackBackgroundTask(task: Promise): void { this.backgroundTasks.add(task); void task.finally(() => { diff --git a/packages/server/src/server/agent/agent-timeline-store-types.ts b/packages/server/src/server/agent/agent-timeline-store-types.ts index 9ca1ab37b9..9e3f3c41a1 100644 --- a/packages/server/src/server/agent/agent-timeline-store-types.ts +++ b/packages/server/src/server/agent/agent-timeline-store-types.ts @@ -47,22 +47,3 @@ export interface AgentTimelineFetchResult { endSeq: number | null; rows: ProjectedTimelineRow[]; } - -export interface AgentTimelineStore { - appendCommitted( - agentId: string, - item: AgentTimelineItem, - options?: { timestamp?: string; turnId?: string }, - ): Promise; - fetchCommitted( - agentId: string, - options?: AgentTimelineFetchOptions, - ): Promise; - getLatestCommittedSeq(agentId: string): Promise; - getCommittedRows(agentId: string): Promise; - getLastItem(agentId: string): Promise; - getLastAssistantMessage(agentId: string): Promise; - deleteAgent(agentId: string): Promise; - bulkInsert(agentId: string, rows: readonly AgentTimelineRow[]): Promise; - updateCommittedRow(agentId: string, row: AgentTimelineRow): Promise; -} diff --git a/packages/server/src/server/agent/agent-timeline-store.test.ts b/packages/server/src/server/agent/agent-timeline-store.test.ts deleted file mode 100644 index 9ec4df4d55..0000000000 --- a/packages/server/src/server/agent/agent-timeline-store.test.ts +++ /dev/null @@ -1,443 +0,0 @@ -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; -import { fileURLToPath } from "node:url"; -import { describe, expect, it } from "vitest"; -import { selectReferenceProjectedTimelinePage } from "./test-utils/timeline-page-reference.js"; -import type { AgentTimelineItem } from "./agent-sdk-types.js"; -import { InMemoryAgentTimelineStore } from "./agent-timeline-store.js"; - -describe("InMemoryAgentTimelineStore", () => { - it("resets stale forward cursors to complete available turns", () => { - const store = new InMemoryAgentTimelineStore(); - store.initialize("agent", { epoch: "current" }); - for (let index = 0; index < 60; index++) - store.append( - "agent", - { - type: "tool_call", - callId: `call-${index}`, - name: "shell", - status: "completed", - error: null, - detail: { type: "unknown", input: null, output: null }, - }, - { turnId: "run" }, - ); - const page = store.fetch("agent", { - direction: "after", - cursor: { epoch: "old", seq: 10 }, - limit: 40, - wholeTurns: true, - }); - expect(page.rows).toHaveLength(60); - expect(page).toMatchObject({ - reset: true, - staleCursor: true, - startSeq: 1, - endSeq: 60, - hasOlder: false, - hasNewer: false, - }); - }); - - it("clamps an overshooting before cursor into the bounded tail window", () => { - const store = new InMemoryAgentTimelineStore(); - store.initialize("agent-1", { - epoch: "epoch-1", - nextSeq: 8, - rows: [ - { - seq: 5, - timestamp: "2026-01-01T00:00:00.000Z", - item: { type: "assistant_message", text: "five", messageId: "five" }, - }, - { - seq: 6, - timestamp: "2026-01-01T00:00:01.000Z", - item: { type: "assistant_message", text: "six", messageId: "six" }, - }, - { - seq: 7, - timestamp: "2026-01-01T00:00:02.000Z", - item: { type: "assistant_message", text: "seven", messageId: "seven" }, - }, - ], - }); - - const result = store.fetch("agent-1", { - direction: "before", - cursor: { epoch: "epoch-1", seq: 100 }, - limit: 2, - }); - - expect(result).toMatchObject({ - epoch: "epoch-1", - direction: "before", - reset: false, - staleCursor: false, - gap: false, - window: { minSeq: 5, maxSeq: 7, nextSeq: 8 }, - hasOlder: true, - hasNewer: false, - rows: [ - { - seq: 6, - timestamp: "2026-01-01T00:00:01.000Z", - item: { type: "assistant_message", text: "six", messageId: "six" }, - }, - { - seq: 7, - timestamp: "2026-01-01T00:00:02.000Z", - item: { type: "assistant_message", text: "seven", messageId: "seven" }, - }, - ], - }); - }); - - it("returns a bounded reset window when an after cursor is behind retained history", () => { - const store = new InMemoryAgentTimelineStore(); - store.initialize("agent-1", { - epoch: "epoch-1", - nextSeq: 8, - rows: [ - { - seq: 5, - timestamp: "2026-01-01T00:00:00.000Z", - item: { type: "assistant_message", text: "five", messageId: "five" }, - }, - { - seq: 6, - timestamp: "2026-01-01T00:00:01.000Z", - item: { type: "assistant_message", text: "six", messageId: "six" }, - }, - { - seq: 7, - timestamp: "2026-01-01T00:00:02.000Z", - item: { type: "assistant_message", text: "seven", messageId: "seven" }, - }, - ], - }); - - const result = store.fetch("agent-1", { - direction: "after", - cursor: { epoch: "epoch-1", seq: 1 }, - limit: 1, - }); - - expect(result).toMatchObject({ - epoch: "epoch-1", - direction: "after", - reset: true, - staleCursor: false, - gap: true, - window: { minSeq: 5, maxSeq: 7, nextSeq: 8 }, - hasOlder: true, - hasNewer: false, - rows: [ - { - seq: 7, - timestamp: "2026-01-01T00:00:02.000Z", - item: { type: "assistant_message", text: "seven", messageId: "seven" }, - }, - ], - }); - }); -}); - -describe("projected timeline retention", () => { - it("retains one full tool state while streaming every source update", () => { - const store = new InMemoryAgentTimelineStore(); - store.initialize("agent"); - for (let seq = 1; seq <= 2000; seq++) { - const item = { - type: "tool_call" as const, - callId: "child", - name: "task", - status: "running" as const, - error: null, - detail: { type: "plain_text" as const, label: "Child", text: "x".repeat(seq * 128) }, - }; - expect(store.append("agent", item)).toMatchObject({ seq, item }); - } - const result = store.fetch("agent", { limit: 0 }); - expect(result.rows).toHaveLength(1); - expect(JSON.stringify(result.rows).length).toBeLessThan(260_000); - expect(result.window.maxSeq).toBe(2000); - }); - - it("catches up a mid-message cursor with the complete projected message", () => { - const store = new InMemoryAgentTimelineStore(); - store.initialize("agent", { epoch: "e" }); - store.append("agent", { type: "assistant_message", messageId: "m", text: "A" }); - store.append("agent", { type: "assistant_message", messageId: "m", text: "B" }); - expect( - store.fetch("agent", { direction: "after", cursor: { epoch: "e", seq: 1 } }).rows, - ).toMatchObject([ - { - item: { text: "AB" }, - seqStart: 1, - seqEnd: 2, - sourceSeqRanges: [{ startSeq: 1, endSeq: 2 }], - }, - ]); - }); - - it("serves a merged assistant message within the item budget", () => { - const store = new InMemoryAgentTimelineStore(); - store.initialize("agent", { epoch: "e" }); - const chunk = "a".repeat(200 * 1024); - for (let index = 0; index < 10; index++) { - store.append("agent", { type: "assistant_message", messageId: "m", text: chunk }); - } - const [row] = store.fetch("agent", { limit: 1 }).rows; - expect(row).toMatchObject({ seqStart: 1, seqEnd: 10 }); - expect(row.item).toMatchObject({ type: "assistant_message", messageId: "m" }); - expect(Buffer.byteLength(JSON.stringify(row.item))).toBeLessThanOrEqual(256 * 1024); - expect(store.getLastAssistantMessage("agent")).toHaveLength(10 * chunk.length); - }); -}); - -describe("projected sequence ownership", () => { - it("preserves fetched coverage when another source chunk arrives", () => { - const store = new InMemoryAgentTimelineStore(); - store.initialize("a"); - store.append("a", { type: "assistant_message", text: "A" }); - const before = store.fetch("a"); - store.append("a", { type: "assistant_message", text: "B" }); - expect(before.rows[0]).toMatchObject({ - item: { text: "A" }, - seqEnd: 1, - sourceSeqRanges: [{ startSeq: 1, endSeq: 1 }], - }); - expect(store.fetch("a").rows[0]).toMatchObject({ - item: { text: "AB" }, - seqEnd: 2, - sourceSeqRanges: [{ startSeq: 1, endSeq: 2 }], - }); - }); - it("preserves source positions when seeding a projected history whose last update is an earlier tool", () => { - const store = new InMemoryAgentTimelineStore(); - store.initialize("a"); - const tool = { - type: "tool_call" as const, - callId: "t", - name: "shell", - error: null, - detail: { type: "plain_text" as const, label: "work" }, - }; - store.append("a", { ...tool, status: "running" }); - store.append("a", { type: "assistant_message", text: "Answer" }); - store.append("a", { ...tool, status: "completed" }); - store.initialize("b", { rows: store.getRows("a") }); - expect(store.getLastItem("b")).toMatchObject({ type: "tool_call", status: "completed" }); - expect(store.getLastAssistantMessage("b")).toBe("Answer"); - expect(store.append("b", { type: "user_message", text: "next" }).seq).toBe(4); - expect(store.getLastItem("b")).toEqual({ type: "user_message", text: "next" }); - expect(store.fetch("b").rows.map((row) => row.seqStart)).toEqual([1, 2, 4]); - }); - it("returns the last projected assistant message without joining different messages", () => { - const store = new InMemoryAgentTimelineStore(); - store.initialize("a"); - store.append("a", { type: "assistant_message", messageId: "first", text: "First" }); - store.append("a", { type: "assistant_message", messageId: "second", text: "Sec" }); - store.append("a", { type: "assistant_message", messageId: "second", text: "ond" }); - expect(store.getLastAssistantMessage("a")).toBe("Second"); - }); -}); - -it("includes transitive tool updates in a contiguous projected tail", () => { - const store = new InMemoryAgentTimelineStore(); - store.initialize("a"); - const tool = (callId: string, status: "running" | "completed") => ({ - type: "tool_call" as const, - callId, - name: "shell", - status, - error: null, - detail: { type: "plain_text" as const, label: "work" }, - }); - store.append("a", tool("first", "running")); - store.append("a", tool("second", "running")); - store.append("a", { type: "user_message", text: "Continue" }); - store.append("a", tool("first", "completed")); - store.append("a", { type: "assistant_message", text: "Answer" }); - store.append("a", tool("second", "completed")); - const tail = store.fetch("a", { limit: 1 }); - expect(tail.rows.map((row) => row.seqStart)).toEqual([1, 2, 3, 5]); - expect(tail).toMatchObject({ startSeq: 1, endSeq: 6, hasOlder: false }); -}); - -async function measureTimelineTail(wholeTurns: boolean) { - const fixture = fileURLToPath( - new URL("./test-utils/timeline-responsiveness.ts", import.meta.url), - ); - const { stdout, stderr } = await promisify(execFile)( - process.execPath, - ["--expose-gc", "--import", "tsx", fixture, String(wholeTurns)], - { timeout: 15_000 }, - ); - process.stderr.write(stderr); - return JSON.parse(stdout) as { - result: ReturnType; - blockingMs: number; - }; -} - -it("excludes unrelated test-runner CPU from the timeline fetch budget", async () => { - let unrelatedWork: ReturnType; - const burnCpu = () => { - const threadCpuUsage: (previous?: NodeJS.CpuUsage) => NodeJS.CpuUsage = Reflect.get( - process, - "threadCpuUsage", - ); - const start = threadCpuUsage(); - let elapsed: NodeJS.CpuUsage; - do { - elapsed = threadCpuUsage(start); - } while (elapsed.user + elapsed.system < 80_000); - unrelatedWork = setImmediate(burnCpu); - }; - unrelatedWork = setImmediate(burnCpu); - try { - const { result: tail, blockingMs } = await measureTimelineTail(false); - expect(tail.rows).toHaveLength(200); - expect(tail.startSeq).toBe(99_801); - expect(tail.endSeq).toBe(100_000); - expect(blockingMs).toBeLessThan(50); - } finally { - clearImmediate(unrelatedWork); - } -}); - -it.each([false, true])( - "serves tail 200 from 100,000 retained rows without stalling the loop (wholeTurns=%s)", - async (wholeTurns) => { - const { result: tail, blockingMs } = await measureTimelineTail(wholeTurns); - expect(tail.rows).toHaveLength(200); - expect(tail.startSeq).toBe(99_801); - expect(tail.endSeq).toBe(100_000); - expect(blockingMs).toBeLessThan(50); - }, -); - -it("matches the previous reprojecting selector on seeded randomized timelines", () => { - let seed = 0x76aabbcc; - const random = (bound: number) => { - seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; - return Math.floor((seed / 0x1_0000_0000) * bound); - }; - for (let scenario = 0; scenario < 100; scenario++) { - const store = new InMemoryAgentTimelineStore(); - store.initialize("a", { epoch: "e" }); - for (let seq = 1; seq <= 100; seq++) { - const identity = String(random(8)); - const text = String(seq); - const items: AgentTimelineItem[] = [ - { type: "user_message", text }, - { type: "assistant_message", text }, - { type: "assistant_message", text, messageId: identity }, - { type: "reasoning", text }, - { - type: "tool_call", - callId: identity, - name: "shell", - status: "running", - error: null, - detail: { type: "plain_text", label: text }, - }, - { - type: "plugin", - id: identity, - pluginId: "test", - kind: "state", - version: 1, - data: { text }, - }, - ]; - store.append("a", items[random(items.length)], { turnId: String(random(3)) }); - } - // Include rehydration of already-projected rows, with non-monotonic seqEnd. - store.initialize("b", { rows: store.getRows("a"), epoch: "e" }); - for (const agentId of ["a", "b"]) { - assertSelectionMatchesReference(store, agentId); - store.append(agentId, { type: "assistant_message", text: "later" }); - assertSelectionMatchesReference(store, agentId); - } - } -}); - -function assertSelectionMatchesReference(store: InMemoryAgentTimelineStore, agentId: string): void { - for (const direction of ["tail", "before", "after"] as const) { - for (const limit of [0, 1, 7, 200]) { - for (const cursorSeq of [0, 1, 25, 50, 99, 100, 150]) { - const actual = store.fetch(agentId, { - direction, - limit, - cursor: { epoch: "e", seq: cursorSeq }, - }); - const expected = selectReferenceProjectedTimelinePage({ - rows: store.getRows(agentId), - bounds: actual.window, - direction, - limit, - cursorSeq, - }); - expect({ - ...expected, - entries: undefined, - rows: expected.entries.map((entry) => Object.assign({ seq: entry.seqEnd }, entry)), - }).toEqual({ - entries: undefined, - rows: actual.rows, - startSeq: actual.startSeq, - endSeq: actual.endSeq, - hasOlder: actual.hasOlder, - hasNewer: actual.hasNewer, - }); - } - } - } -} - -it("continues a scroll across appends and lifecycle collapse without losing source coverage", () => { - const store = new InMemoryAgentTimelineStore(); - store.initialize("a", { epoch: "e" }); - const tool: AgentTimelineItem = { - type: "tool_call", - callId: "t", - name: "shell", - status: "running", - error: null, - detail: { type: "plain_text", label: "work" }, - }; - store.append("a", tool); - for (let seq = 2; seq <= 5; seq++) store.append("a", { type: "user_message", text: String(seq) }); - const first = store.fetch("a", { direction: "after", cursor: { epoch: "e", seq: 0 }, limit: 2 }); - expect(first.endSeq).toBe(2); - store.append("a", { ...tool, status: "completed" }); - store.append("a", { type: "assistant_message", text: "Done" }); - expect(first.rows[0].item).toMatchObject({ status: "running" }); - const second = store.fetch("a", { - direction: "after", - cursor: { epoch: "e", seq: first.endSeq! }, - limit: 2, - }); - expect(second.rows.map((row) => row.seqStart)).toEqual([3, 4]); - expect(second.endSeq).toBe(4); - const third = store.fetch("a", { - direction: "after", - cursor: { epoch: "e", seq: second.endSeq! }, - limit: 2, - }); - expect(third.rows.map((row) => row.seqStart)).toEqual([1, 5]); - expect(third.rows[0].item).toMatchObject({ status: "completed" }); - expect(third.endSeq).toBe(6); - const last = store.fetch("a", { - direction: "after", - cursor: { epoch: "e", seq: third.endSeq! }, - limit: 2, - }); - expect(last.rows.map((row) => row.item)).toEqual([{ type: "assistant_message", text: "Done" }]); - expect(last.endSeq).toBe(7); - expect(last.hasNewer).toBe(false); -}); diff --git a/packages/server/src/server/agent/agent-timeline-store.ts b/packages/server/src/server/agent/agent-timeline-store.ts deleted file mode 100644 index 4c3b9c5c3a..0000000000 --- a/packages/server/src/server/agent/agent-timeline-store.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { randomUUID } from "node:crypto"; -import { TimelineProjection, type ProjectedTimelineRow } from "./timeline-projection.js"; -import { limitAgentTimelineItemContent } from "./agent-timeline-content.js"; -import type { AgentTimelineItem } from "./agent-sdk-types.js"; -import type { - AgentTimelineFetchOptions, - AgentTimelineFetchResult, - AgentTimelineRow, -} from "./agent-timeline-store-types.js"; - -export interface SeedAgentTimelineOptions { - items?: readonly AgentTimelineItem[]; - rows?: readonly AgentTimelineRow[]; - epoch?: string; - nextSeq?: number; - timestamp?: string; -} - -interface AgentTimelineState { - epoch: string; - projection: TimelineProjection; - minSeq: number; - nextSeq: number; -} -const DEFAULT_TIMELINE_FETCH_LIMIT = 200; -function cloneRow(row: T): T { - return { ...row }; -} - -export class InMemoryAgentTimelineStore { - private readonly states = new Map(); - - has(agentId: string): boolean { - return this.states.has(agentId); - } - - initialize(agentId: string, options?: SeedAgentTimelineOptions): void { - const timestamp = options?.timestamp ?? new Date().toISOString(); - const rows = options?.rows?.length - ? options.rows.map(cloneRow) - : this.buildRowsFromItems(options?.items ?? [], options?.nextSeq ?? 1, timestamp); - const nextSeq = rows.reduce((next, row) => Math.max(next, row.seq + 1), options?.nextSeq ?? 1); - const projection = new TimelineProjection(rows); - this.states.set(agentId, { - epoch: options?.epoch ?? randomUUID(), - projection, - minSeq: projection.getRows()[0]?.seqStart ?? 0, - nextSeq, - }); - } - - delete(agentId: string): void { - this.states.delete(agentId); - } - - getItems(agentId: string): AgentTimelineItem[] { - return this.requireState(agentId) - .projection.getRows() - .map((row) => row.item); - } - - getRows(agentId: string): ProjectedTimelineRow[] { - return this.requireState(agentId).projection.getRows().map(cloneRow); - } - - getSubmittedUserMessage(agentId: string, clientMessageId: string): AgentTimelineRow | null { - const row = this.requireState(agentId) - .projection.getRows() - .find( - (candidate) => - candidate.item.type === "user_message" && - candidate.item.clientMessageId === clientMessageId, - ); - return row ? cloneRow(row) : null; - } - - enrichSubmittedUserMessage( - agentId: string, - clientMessageId: string, - providerMessageId: string, - ): AgentTimelineRow | null { - return this.requireState(agentId).projection.enrichSubmittedUserMessage( - clientMessageId, - providerMessageId, - ); - } - - getEpoch(agentId: string): string { - return this.requireState(agentId).epoch; - } - - fetch(agentId: string, options?: AgentTimelineFetchOptions): AgentTimelineFetchResult { - const state = this.requireState(agentId); - const direction = options?.direction ?? "tail"; - const cursor = options?.cursor; - const hasRows = state.minSeq !== 0; - const window = { minSeq: state.minSeq, maxSeq: state.nextSeq - 1, nextSeq: state.nextSeq }; - const staleCursor = cursor !== undefined && cursor.epoch !== state.epoch; - const gap = - !staleCursor && - direction === "after" && - cursor !== undefined && - hasRows && - cursor.seq < state.minSeq - 1; - const reset = staleCursor || gap; - const page = state.projection.selectPage({ - bounds: window, - direction: reset ? "tail" : direction, - cursorSeq: cursor?.seq, - limit: options?.limit ?? DEFAULT_TIMELINE_FETCH_LIMIT, - wholeTurns: options?.wholeTurns, - }); - return { - epoch: state.epoch, - direction, - reset, - staleCursor, - gap, - window, - hasOlder: page.hasOlder, - hasNewer: page.hasNewer, - startSeq: page.startSeq, - endSeq: page.endSeq, - // Stored rows are already within budget; this bounds assistant and reasoning merges. - rows: page.entries.map((entry) => - Object.assign({ seq: entry.seqEnd }, entry, { - item: limitAgentTimelineItemContent(entry.item), - }), - ), - }; - } - - append( - agentId: string, - item: AgentTimelineItem, - options?: { timestamp?: string; providerMessageId?: string; turnId?: string }, - ): AgentTimelineRow { - const state = this.requireState(agentId); - const row: AgentTimelineRow = { - seq: state.nextSeq, - timestamp: options?.timestamp ?? new Date().toISOString(), - item, - ...(options?.turnId ? { turnId: options.turnId } : {}), - ...(options?.providerMessageId ? { providerMessageId: options.providerMessageId } : {}), - }; - state.nextSeq += 1; - if (state.minSeq === 0) state.minSeq = row.seq; - state.projection.append(row); - return cloneRow(row); - } - - getLastItem(agentId: string): AgentTimelineItem | null { - const state = this.requireState(agentId); - return state.projection.getLastItem(state.nextSeq - 1); - } - - getLastAssistantMessage(agentId: string): string | null { - return this.requireState(agentId).projection.getLastAssistantMessage(); - } - - private requireState(agentId: string): AgentTimelineState { - const state = this.states.get(agentId); - if (!state) { - throw new Error(`Unknown agent '${agentId}'`); - } - return state; - } - - private buildRowsFromItems( - items: readonly AgentTimelineItem[], - startSeq: number, - timestamp: string, - ): AgentTimelineRow[] { - let nextSeq = startSeq; - return items.map((item) => { - const row: AgentTimelineRow = { - seq: nextSeq, - timestamp, - item, - }; - nextSeq += 1; - return row; - }); - } -} diff --git a/packages/server/src/server/agent/import-sessions.test.ts b/packages/server/src/server/agent/import-sessions.test.ts index 9190e6942b..a9b4e1800c 100644 --- a/packages/server/src/server/agent/import-sessions.test.ts +++ b/packages/server/src/server/agent/import-sessions.test.ts @@ -679,7 +679,7 @@ class ProviderImportHarness { return this.snapshot; }, hydrateTimelineFromProvider: async () => {}, - getTimeline: () => this.timeline, + getTimeline: async () => this.timeline, closeAgent: async (agentId: string) => { this.closedAgentIds.push(agentId); this.activeAgent = null; diff --git a/packages/server/src/server/agent/import-sessions.ts b/packages/server/src/server/agent/import-sessions.ts index 492667ab88..c7d7527ebf 100644 --- a/packages/server/src/server/agent/import-sessions.ts +++ b/packages/server/src/server/agent/import-sessions.ts @@ -239,7 +239,7 @@ async function importProviderSessionNow( }); return { snapshot, - timelineSize: input.agentManager.getTimeline(snapshot.id).length, + timelineSize: (await input.agentManager.getTimeline(snapshot.id)).length, }; } catch (error) { await rollbackArchivedImport(input, archivedRecord, archivedRecord.archivedAt); @@ -258,7 +258,7 @@ async function importProviderSessionNow( return { snapshot, - timelineSize: input.agentManager.getTimeline(snapshot.id).length, + timelineSize: (await input.agentManager.getTimeline(snapshot.id)).length, }; } diff --git a/packages/server/src/server/agent/mcp-parity.e2e.test.ts b/packages/server/src/server/agent/mcp-parity.e2e.test.ts index efb14e6d31..091588b774 100644 --- a/packages/server/src/server/agent/mcp-parity.e2e.test.ts +++ b/packages/server/src/server/agent/mcp-parity.e2e.test.ts @@ -565,12 +565,12 @@ describe("Suite A: Core Fixes", () => { const notifications = await waitFor({ timeoutMs: 10000, label: `${outcome} notification`, - check: () => { - const items = daemonHandle.daemon.agentManager - .getTimeline(parentAgentId) - .filter( - (item) => item.type === "tool_call" && item.agentMessage?.sender.id === childId, - ); + check: async () => { + const items = ( + await daemonHandle.daemon.agentManager.getTimeline(parentAgentId) + ).filter( + (item) => item.type === "tool_call" && item.agentMessage?.sender.id === childId, + ); return items.length > 0 ? items : null; }, }); diff --git a/packages/server/src/server/agent/mcp-shared.ts b/packages/server/src/server/agent/mcp-shared.ts index 0936474255..b4eab81ba4 100644 --- a/packages/server/src/server/agent/mcp-shared.ts +++ b/packages/server/src/server/agent/mcp-shared.ts @@ -142,7 +142,7 @@ export async function waitForAgentWithTimeout( } catch (error) { if (error instanceof Error && error.message === "wait timeout") { const snapshot = agentManager.getAgent(agentId); - const timeline = agentManager.getTimeline(agentId); + const timeline = await agentManager.getTimeline(agentId); const recent = selectItemsByProjectedLimit({ items: timeline, direction: "tail", diff --git a/packages/server/src/server/agent/projected-timeline-contract.test.ts b/packages/server/src/server/agent/projected-timeline-contract.test.ts index ee76fe50a7..e300255f8a 100644 --- a/packages/server/src/server/agent/projected-timeline-contract.test.ts +++ b/packages/server/src/server/agent/projected-timeline-contract.test.ts @@ -64,9 +64,13 @@ test("projects Codex child history and confines old-client degradation to the ch .toBe(1); await expect .poll( - () => - ctx.daemon.daemon.agentManager.fetchProviderSubagentTimeline(agent.id, "child-thread") - .window.maxSeq, + async () => + ( + await ctx.daemon.daemon.agentManager.fetchProviderSubagentTimeline( + agent.id, + "child-thread", + ) + ).window.maxSeq, ) .toBe(3); const child = await ctx.client.fetchProviderSubagentTimeline(agent.id, "child-thread", { @@ -170,9 +174,10 @@ test("resuming Codex restores child descriptors and complete messages without hi test("retains only the latest cumulative Pi tool progress payload", async () => { const { parseToolArgs, parseToolResult, mapToolDetail } = await import("./providers/pi/tool-call-mapper.js"); - const { InMemoryAgentTimelineStore } = await import("./agent-timeline-store.js"); - const store = new InMemoryAgentTimelineStore(); - store.initialize("agent"); + const { AgentTimeline } = await import("./timeline/index.js"); + const timeline = new AgentTimeline(); + const scope = { agentId: "agent" }; + await timeline.create(scope); const tool = parseToolArgs("custom_progress", {}); const messages: string[] = []; for (let index = 0; index < 20; index++) { @@ -181,16 +186,20 @@ test("retains only the latest cumulative Pi tool progress payload", async () => content: [{ type: "text", text: "running" }], details: { results: [{ messages: [...messages] }] }, }); - store.append("agent", { - type: "tool_call", - callId: "same-call", - name: "custom_progress", - status: "running", - error: null, - detail: mapToolDetail(tool, result), - }); + await timeline.append(scope, [ + { + item: { + type: "tool_call", + callId: "same-call", + name: "custom_progress", + status: "running", + error: null, + detail: mapToolDetail(tool, result), + }, + }, + ]); } - const history = store.fetch("agent", { limit: 0 }); + const history = await timeline.fetch(scope, { limit: 0 }); expect(history.rows).toHaveLength(1); expect(history.rows[0]).toMatchObject({ seqStart: 1, diff --git a/packages/server/src/server/agent/provider-subagents/store.test.ts b/packages/server/src/server/agent/provider-subagents/store.test.ts index 5c2ca55174..81a1ead3b3 100644 --- a/packages/server/src/server/agent/provider-subagents/store.test.ts +++ b/packages/server/src/server/agent/provider-subagents/store.test.ts @@ -1,11 +1,12 @@ import { describe, expect, test } from "vitest"; +import { AgentTimeline } from "../timeline/index.js"; import { ProviderSubagentStore } from "./store.js"; describe("ProviderSubagentStore", () => { - test("keeps provider children and their timelines scoped to the parent agent", () => { - const subagents = new ProviderSubagentStore(); + test("keeps provider children and their timelines scoped to the parent agent", async () => { + const subagents = new ProviderSubagentStore(new AgentTimeline()); - subagents.apply("parent-a", "codex", { + await subagents.apply("parent-a", "codex", { type: "upsert", id: "child-1", title: "Explore", @@ -13,19 +14,19 @@ describe("ProviderSubagentStore", () => { status: "running", timestamp: "2026-07-12T10:00:00.000Z", }); - subagents.apply("parent-a", "codex", { + await subagents.apply("parent-a", "codex", { type: "timeline", id: "child-1", item: { type: "assistant_message", text: "Found it." }, timestamp: "2026-07-12T10:00:01.000Z", }); - subagents.apply("parent-a", "codex", { + await subagents.apply("parent-a", "codex", { type: "upsert", id: "child-1", status: "completed", timestamp: "2026-07-12T10:00:02.000Z", }); - subagents.apply("parent-b", "claude", { + await subagents.apply("parent-b", "claude", { type: "upsert", id: "child-1", title: "Review", @@ -45,7 +46,7 @@ describe("ProviderSubagentStore", () => { updatedAt: "2026-07-12T10:00:02.000Z", }), ]); - expect(subagents.fetchTimeline("parent-a", "child-1").rows).toMatchObject([ + expect((await subagents.fetchTimeline("parent-a", "child-1")).rows).toMatchObject([ { seq: 1, timestamp: "2026-07-12T10:00:01.000Z", @@ -53,17 +54,17 @@ describe("ProviderSubagentStore", () => { }, ]); expect(subagents.list("parent-b")[0]).toMatchObject({ provider: "claude", title: "Review" }); - expect(subagents.deleteParent("parent-a")).toEqual([ + expect(await subagents.deleteParent("parent-a")).toEqual([ { type: "remove", parentAgentId: "parent-a", subagentId: "child-1" }, ]); expect(subagents.list("parent-a")).toEqual([]); expect(subagents.list("parent-b")).toHaveLength(1); }); - test("limits oversized provider child tool output before storage", () => { - const subagents = new ProviderSubagentStore(); + test("limits oversized provider child tool output before storage", async () => { + const subagents = new ProviderSubagentStore(new AgentTimeline()); const output = "x".repeat(70 * 1024); - const update = subagents.apply("parent-a", "opencode", { + const update = await subagents.apply("parent-a", "opencode", { type: "timeline", id: "child-1", item: { @@ -77,24 +78,24 @@ describe("ProviderSubagentStore", () => { }); expect(update.type).toBe("timeline"); - const [row] = subagents.fetchTimeline("parent-a", "child-1").rows; + const [row] = (await subagents.fetchTimeline("parent-a", "child-1")).rows; expect(row?.item).toMatchObject({ type: "tool_call", detail: { type: "shell", output: expect.stringMatching(/^x+…\[truncated \d+\.\d KB\]$/) }, }); }); - test("pages provider history on projected item boundaries", () => { - const subagents = new ProviderSubagentStore(); + test("pages provider history on projected item boundaries", async () => { + const subagents = new ProviderSubagentStore(new AgentTimeline()); for (let index = 0; index < 101; index += 1) { - subagents.apply("parent-a", "opencode", { + await subagents.apply("parent-a", "opencode", { type: "timeline", id: "child-1", item: { type: "assistant_message", text: String(index) }, }); } - const page = subagents.fetchTimeline("parent-a", "child-1", { + const page = await subagents.fetchTimeline("parent-a", "child-1", { direction: "tail", limit: 1, }); diff --git a/packages/server/src/server/agent/provider-subagents/store.ts b/packages/server/src/server/agent/provider-subagents/store.ts index b04246054a..02e19bcacb 100644 --- a/packages/server/src/server/agent/provider-subagents/store.ts +++ b/packages/server/src/server/agent/provider-subagents/store.ts @@ -1,11 +1,10 @@ import type { AgentProvider, AgentTimelineItem } from "../agent-sdk-types.js"; -import { limitAgentTimelineItemContent } from "../agent-timeline-content.js"; -import { InMemoryAgentTimelineStore } from "../agent-timeline-store.js"; import type { AgentTimelineFetchOptions, AgentTimelineFetchResult, AgentTimelineRow, } from "../agent-timeline-store-types.js"; +import type { AgentTimeline, TimelineScope } from "../timeline/index.js"; export type ProviderSubagentStatus = "running" | "completed" | "failed" | "canceled"; @@ -66,6 +65,10 @@ function storeKey(parentAgentId: string, subagentId: string): string { return `${parentAgentId}\0${subagentId}`; } +function childScope(parentAgentId: string, subagentId: string): TimelineScope { + return { agentId: parentAgentId, subagentId }; +} + /** * Sticky upsert semantics for a descriptor field: an omitted value preserves what is stored, an * explicit `null` clears it. Providers observe these fields incrementally, so a partial upsert @@ -77,41 +80,38 @@ function stickyField(next: T | undefined, previous: T | null | undefined): T export class ProviderSubagentStore { private readonly descriptors = new Map(); - private readonly timelines = new InMemoryAgentTimelineStore(); - apply( + constructor(private readonly timeline: AgentTimeline) {} + + async apply( parentAgentId: string, provider: AgentProvider, event: ProviderSubagentInputEvent, - ): ProviderSubagentStoreEvent { + ): Promise { const key = storeKey(parentAgentId, event.id); + const scope = childScope(parentAgentId, event.id); if (event.type === "remove") { this.descriptors.delete(key); - this.timelines.delete(key); + await this.timeline.delete(scope); return { type: "remove", parentAgentId, subagentId: event.id }; } if (event.type === "timeline") { - if (!this.timelines.has(key)) { - this.timelines.initialize(key); - } - const row = this.timelines.append(key, limitAgentTimelineItemContent(event.item), { - timestamp: event.timestamp, - }); + await this.timeline.open(scope); + const { + commits: [{ row, epoch }], + } = await this.timeline.append(scope, [{ item: event.item, timestamp: event.timestamp }]); return { type: "timeline", parentAgentId, subagentId: event.id, provider, row, - epoch: this.timelines.getEpoch(key), + epoch, }; } const previous = this.descriptors.get(key); - if (!this.timelines.has(key)) { - this.timelines.initialize(key); - } const timestamp = event.timestamp ?? new Date().toISOString(); const subagent: ProviderSubagentDescriptor = { id: event.id, @@ -128,6 +128,7 @@ export class ProviderSubagentStore { parentSubagentId: stickyField(event.parentSubagentId, previous?.parentSubagentId), }; this.descriptors.set(key, subagent); + await this.timeline.open(scope); return { type: "upsert", subagent }; } @@ -149,18 +150,18 @@ export class ProviderSubagentStore { parentAgentId: string, subagentId: string, options?: AgentTimelineFetchOptions, - ): AgentTimelineFetchResult { - return this.timelines.fetch(storeKey(parentAgentId, subagentId), options); + ): Promise { + return this.timeline.fetch(childScope(parentAgentId, subagentId), options); } - deleteParent(parentAgentId: string): ProviderSubagentStoreEvent[] { - const events: ProviderSubagentStoreEvent[] = []; - for (const subagent of this.list(parentAgentId)) { - const key = storeKey(parentAgentId, subagent.id); - this.descriptors.delete(key); - this.timelines.delete(key); - events.push({ type: "remove", parentAgentId, subagentId: subagent.id }); + async deleteParent(parentAgentId: string): Promise { + const removed = this.list(parentAgentId).map((subagent) => subagent.id); + for (const subagentId of removed) { + this.descriptors.delete(storeKey(parentAgentId, subagentId)); } - return events; + await Promise.all( + removed.map((subagentId) => this.timeline.delete(childScope(parentAgentId, subagentId))), + ); + return removed.map((subagentId) => ({ type: "remove", parentAgentId, subagentId })); } } diff --git a/packages/server/src/server/agent/providers/claude/subagents/observation.test.ts b/packages/server/src/server/agent/providers/claude/subagents/observation.test.ts index 3690334af0..e3a724b633 100644 --- a/packages/server/src/server/agent/providers/claude/subagents/observation.test.ts +++ b/packages/server/src/server/agent/providers/claude/subagents/observation.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { ProviderSubagentStore } from "../../../provider-subagents/store.js"; +import { AgentTimeline } from "../../../timeline/index.js"; import { foldSubagentObservations, type SubagentObservation } from "./observation.js"; describe("foldSubagentObservations", () => { @@ -66,9 +67,9 @@ describe("foldSubagentObservations", () => { ); }); - it("accumulates through the store so a later status keeps the declared identity", () => { + it("accumulates through the store so a later status keeps the declared identity", async () => { // The property the whole design rests on: sources emit sparse facts, the store merges them. - const store = new ProviderSubagentStore(); + const store = new ProviderSubagentStore(new AgentTimeline()); const events = foldSubagentObservations([ { kind: "declared", id: "toolu_1", title: "Explore", description: "Find the code" }, { kind: "status", id: "toolu_1", status: "completed" }, @@ -76,7 +77,7 @@ describe("foldSubagentObservations", () => { let descriptor = null; for (const event of events) { - const applied = store.apply("parent", "claude", event); + const applied = await store.apply("parent", "claude", event); if (applied.type === "upsert") descriptor = applied.subagent; } @@ -102,15 +103,15 @@ describe("subtitle observations", () => { expect(event).not.toHaveProperty("status"); }); - it("does not revert a finished subagent when presentation changes", () => { - const store = new ProviderSubagentStore(); + it("does not revert a finished subagent when presentation changes", async () => { + const store = new ProviderSubagentStore(new AgentTimeline()); let descriptor = null; for (const event of foldSubagentObservations([ { kind: "declared", id: "toolu_1", title: "Explore" }, { kind: "status", id: "toolu_1", status: "completed" }, { kind: "subtitle", id: "toolu_1", subtitle: "Explore · Opus 5 · 32.3k tokens" }, ])) { - const applied = store.apply("parent", "claude", event); + const applied = await store.apply("parent", "claude", event); if (applied.type === "upsert") descriptor = applied.subagent; } expect(descriptor).toMatchObject({ @@ -119,14 +120,14 @@ describe("subtitle observations", () => { }); }); - it("keeps identity when a later subtitle arrives", () => { - const store = new ProviderSubagentStore(); + it("keeps identity when a later subtitle arrives", async () => { + const store = new ProviderSubagentStore(new AgentTimeline()); let descriptor = null; for (const event of foldSubagentObservations([ { kind: "declared", id: "toolu_1", title: "Explore", description: "Find the code" }, { kind: "subtitle", id: "toolu_1", subtitle: "Explore · Opus 5" }, ])) { - const applied = store.apply("parent", "claude", event); + const applied = await store.apply("parent", "claude", event); if (applied.type === "upsert") descriptor = applied.subagent; } expect(descriptor).toMatchObject({ diff --git a/packages/server/src/server/agent/providers/claude/subagents/replay-source.test.ts b/packages/server/src/server/agent/providers/claude/subagents/replay-source.test.ts index 3bde4b813b..a60feb8ba4 100644 --- a/packages/server/src/server/agent/providers/claude/subagents/replay-source.test.ts +++ b/packages/server/src/server/agent/providers/claude/subagents/replay-source.test.ts @@ -11,6 +11,7 @@ import { ProviderSubagentStore, type ProviderSubagentDescriptor, } from "../../../provider-subagents/store.js"; +import { AgentTimeline } from "../../../timeline/index.js"; import { ClaudeTaskProtocolSource } from "./live-source.js"; import { foldSubagentObservations, type SubagentObservation } from "./observation.js"; import { @@ -40,11 +41,13 @@ function parentWithTaskCall(failed = false): ClaudeReplayParentFacts { }; } -function applyToStore(observations: SubagentObservation[]): ProviderSubagentDescriptor | null { - const store = new ProviderSubagentStore(); +async function applyToStore( + observations: SubagentObservation[], +): Promise { + const store = new ProviderSubagentStore(new AgentTimeline()); let descriptor: ProviderSubagentDescriptor | null = null; for (const event of foldSubagentObservations(observations)) { - const applied = store.apply("parent", "claude", event); + const applied = await store.apply("parent", "claude", event); if (applied.type === "upsert") descriptor = applied.subagent; } return descriptor; @@ -203,7 +206,7 @@ describe("observeReplaySubagents", () => { it("drops a terminal subagent the parent has no Task call for", async () => { // A legacy grandchild: its toolUseId names a Task call made inside a sibling's transcript, so // no tool_result for it can ever reach the parent. Left running, it never stops. - const descriptor = applyToStore( + const descriptor = await applyToStore( ( await observeReplaySubagents({ subagents: [ @@ -231,7 +234,7 @@ describe("observeReplaySubagents", () => { it("drops a terminal skill-spawned subagent that has no link at all", async () => { // A /code-review run: no Task tool_use exists to name it, so its sidecar carries only // agentType and its id appears nowhere in the parent transcript. - const descriptor = applyToStore( + const descriptor = await applyToStore( ( await observeReplaySubagents({ subagents: [ @@ -257,7 +260,7 @@ describe("observeReplaySubagents", () => { }); it("reports a scraped link's failure as failed", async () => { - const descriptor = applyToStore( + const descriptor = await applyToStore( ( await observeReplaySubagents({ subagents: [{ agentId: AGENT_ID, meta: null, entries: [] }], @@ -299,7 +302,7 @@ describe("observeReplaySubagents", () => { }); it("finishes a terminal Task child when the parent has not recorded its outcome", async () => { - const descriptor = applyToStore( + const descriptor = await applyToStore( ( await observeReplaySubagents({ subagents: [ @@ -575,7 +578,7 @@ describe("replay usage", () => { }); it("lands the complete provider-owned subtitle on the descriptor", async () => { - const store = new ProviderSubagentStore(); + const store = new ProviderSubagentStore(new AgentTimeline()); const observations = ( await observeReplaySubagents({ subagents: [ @@ -596,7 +599,7 @@ describe("replay usage", () => { }) ).observations; for (const event of foldSubagentObservations(observations)) { - store.apply("parent", "claude", event); + await store.apply("parent", "claude", event); } expect(store.get("parent", TOOL_USE_ID)?.subtitle).toBe("general-purpose · 7 tokens"); @@ -645,8 +648,8 @@ describe("live and replay agree", () => { }) ).observations; - const fromWire = applyToStore(liveObservations); - const fromDisk = applyToStore(replayObservations); + const fromWire = await applyToStore(liveObservations); + const fromDisk = await applyToStore(replayObservations); const identity = (descriptor: ProviderSubagentDescriptor | null) => ({ id: descriptor?.id, diff --git a/packages/server/src/server/agent/providers/claude/subagents/workflow-replay-source.test.ts b/packages/server/src/server/agent/providers/claude/subagents/workflow-replay-source.test.ts index edb673c664..dc1418a5f1 100644 --- a/packages/server/src/server/agent/providers/claude/subagents/workflow-replay-source.test.ts +++ b/packages/server/src/server/agent/providers/claude/subagents/workflow-replay-source.test.ts @@ -7,6 +7,7 @@ import { import { describe, expect, it } from "vitest"; import { ProviderSubagentStore } from "../../../provider-subagents/store.js"; +import { AgentTimeline } from "../../../timeline/index.js"; import { foldSubagentObservations } from "./observation.js"; import { observeReplayWorkflows, @@ -82,9 +83,9 @@ describe("Claude workflow replay", () => { ], parentEntries: parentEntries(), }); - const store = new ProviderSubagentStore(); + const store = new ProviderSubagentStore(new AgentTimeline()); for (const event of foldSubagentObservations(observations)) { - store.apply("parent", "claude", event); + await store.apply("parent", "claude", event); } expect(store.get("parent", TOOL_CALL_ID)).toMatchObject({ @@ -107,9 +108,9 @@ describe("Claude workflow replay", () => { }, ]), }); - const store = new ProviderSubagentStore(); + const store = new ProviderSubagentStore(new AgentTimeline()); for (const event of foldSubagentObservations(observations)) { - store.apply("parent", "claude", event); + await store.apply("parent", "claude", event); } expect(store.get("parent", TOOL_CALL_ID)).toMatchObject({ diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts index 62e6bb0404..7366d91fca 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts @@ -625,7 +625,7 @@ function createReopenableCodexAgent(threads: Record) { // Cold-loads the agent, closes its runtime, and reopens it while Paseo keeps its timeline. async loadCloseAndReopen(): Promise { await open(); - const projected = manager.getTimeline(agentId); + const projected = await manager.getTimeline(agentId); await manager.closeAgent(agentId); await open(); return projected; @@ -2072,7 +2072,7 @@ describe("Codex app-server provider", () => { "thread/read", ]); expect(codex.threadMethods(1)).toEqual(["thread/loaded/list", "thread/resume"]); - expect(codex.manager.getTimeline(codex.agentId)).toEqual(projected); + expect(await codex.manager.getTimeline(codex.agentId)).toEqual(projected); } finally { await codex.close(); } @@ -2098,7 +2098,7 @@ describe("Codex app-server provider", () => { expect(codex.latestAppServer().recordedRollbacks).toEqual([ { threadId: "forked-thread", numTurns: 1 }, ]); - expect(codex.manager.getTimeline(codex.agentId)).toEqual([ + expect(await codex.manager.getTimeline(codex.agentId)).toEqual([ expect.objectContaining({ type: "user_message", messageId: "user-1", text: "First" }), expect.objectContaining({ type: "assistant_message", text: "First answer." }), ]); @@ -2151,9 +2151,9 @@ describe("Codex app-server provider", () => { appServer.completeTurn({ threadId: "saved-thread" }); await run; - const toolCalls = codex.manager - .getTimeline(codex.agentId) - .filter((item) => item.type === "tool_call"); + const toolCalls = (await codex.manager.getTimeline(codex.agentId)).filter( + (item) => item.type === "tool_call", + ); expect(new Set(toolCalls.map((item) => item.callId))).toEqual(new Set(["spawn-child"])); expect(toolCalls.at(-1)).toMatchObject({ callId: "spawn-child", status: "completed" }); expect(codex.manager.getProviderSubagent(codex.agentId, "child-thread")).toMatchObject({ @@ -2161,9 +2161,9 @@ describe("Codex app-server provider", () => { status: "completed", }); expect( - codex.manager - .fetchProviderSubagentTimeline(codex.agentId, "child-thread") - .rows.map((row) => row.item), + (await codex.manager.fetchProviderSubagentTimeline(codex.agentId, "child-thread")).rows.map( + (row) => row.item, + ), ).toEqual([ expect.objectContaining({ type: "assistant_message", text: "Child started." }), expect.objectContaining({ type: "assistant_message", text: "Child replied." }), diff --git a/packages/server/src/server/agent/providers/mock-load-test-agent.test.ts b/packages/server/src/server/agent/providers/mock-load-test-agent.test.ts index da10deb869..396e7876af 100644 --- a/packages/server/src/server/agent/providers/mock-load-test-agent.test.ts +++ b/packages/server/src/server/agent/providers/mock-load-test-agent.test.ts @@ -535,7 +535,7 @@ describe("MockLoadTestAgentClient", () => { await vi.advanceTimersByTimeAsync(10_000); await resultPromise; - const timeline = manager.getTimeline(agent.id); + const timeline = await manager.getTimeline(agent.id); const assistantMessages = timeline.filter((item) => item.type === "assistant_message"); const toolCalls = timeline.filter((item) => item.type === "tool_call"); diff --git a/packages/server/src/server/agent/providers/pi/extensions/edxeth-pi-subagents/index.test.ts b/packages/server/src/server/agent/providers/pi/extensions/edxeth-pi-subagents/index.test.ts index 86692ab32e..a0d5d2512f 100644 --- a/packages/server/src/server/agent/providers/pi/extensions/edxeth-pi-subagents/index.test.ts +++ b/packages/server/src/server/agent/providers/pi/extensions/edxeth-pi-subagents/index.test.ts @@ -8,6 +8,7 @@ import { join } from "node:path"; import { edxethPiSubagents } from "./index.js"; import { streamPiHistory } from "../../history-mapper.js"; import { ProviderSubagentStore } from "../../../../provider-subagents/store.js"; +import { AgentTimeline } from "../../../../timeline/index.js"; const args = { agent: "scout", @@ -274,13 +275,13 @@ describe("edxeth/pi-subagents adapter", () => { from: `/tmp/paseo-edxeth-fixture/${mode}/child-session.jsonl`, to: fileURLToPath(new URL(`./fixtures/${mode}-child-session.jsonl`, import.meta.url)), }); - const store = new ProviderSubagentStore(); + const store = new ProviderSubagentStore(new AgentTimeline()); for await (const event of streamPiHistory("pi", fixture.messages)) { - if (event.type === "provider_subagent") store.apply("parent", "pi", event.event); + if (event.type === "provider_subagent") await store.apply("parent", "pi", event.event); } const children = store.list("parent"); expect(children).toEqual([expect.objectContaining({ status: "completed", title: "marker" })]); - const transcript = store.fetchTimeline("parent", children[0]!.id); + const transcript = await store.fetchTimeline("parent", children[0]!.id); expect(transcript.rows).toContainEqual( expect.objectContaining({ item: expect.objectContaining({ diff --git a/packages/server/src/server/agent/providers/pi/extensions/gotgenes-pi-subagents/index.test.ts b/packages/server/src/server/agent/providers/pi/extensions/gotgenes-pi-subagents/index.test.ts index 7bb62a9d1f..3e7648d8a4 100644 --- a/packages/server/src/server/agent/providers/pi/extensions/gotgenes-pi-subagents/index.test.ts +++ b/packages/server/src/server/agent/providers/pi/extensions/gotgenes-pi-subagents/index.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "vitest"; import { fileURLToPath } from "node:url"; import { ProviderSubagentStore } from "../../../../provider-subagents/store.js"; +import { AgentTimeline } from "../../../../timeline/index.js"; import { createPiExtensionHost } from "../index.js"; import { readSubagentFixture, verifySubagentFixture } from "../subagent-fixture-test.js"; import { streamPiHistory } from "../../history-mapper.js"; @@ -43,9 +44,13 @@ describe("@gotgenes/pi-subagents adapter", () => { if (block.type === "toolCall") block.arguments = start.args; } const events = await verifySubagentFixture(fixture); - const store = new ProviderSubagentStore(); - store.apply("parent", "pi", { type: "upsert", id: "previous-child", status: "completed" }); - for (const event of events) store.apply("parent", "pi", event.event); + const store = new ProviderSubagentStore(new AgentTimeline()); + await store.apply("parent", "pi", { + type: "upsert", + id: "previous-child", + status: "completed", + }); + for (const event of events) await store.apply("parent", "pi", event.event); expect(events.map(({ event }) => event.type)).toEqual(["upsert", "remove"]); expect(store.list("parent")).toMatchObject([{ id: "previous-child", status: "completed" }]); }, diff --git a/packages/server/src/server/agent/rewind/rewind.test.ts b/packages/server/src/server/agent/rewind/rewind.test.ts index df5c242251..1c596ed4cd 100644 --- a/packages/server/src/server/agent/rewind/rewind.test.ts +++ b/packages/server/src/server/agent/rewind/rewind.test.ts @@ -75,9 +75,9 @@ describe("AgentManager rewind", () => { expect(session.recordedRewinds).toEqual([{ mode: "conversation", messageId: "message-1" }]); expect(session.historyReadCount).toBe(1); - expect(manager.fetchTimeline(agentId, { limit: 0 }).rows.map((row) => row.item)).toEqual([ - { type: "user_message", text: "before", messageId: "message-1" }, - ]); + expect( + (await manager.fetchTimeline(agentId, { limit: 0 })).rows.map((row) => row.item), + ).toEqual([{ type: "user_message", text: "before", messageId: "message-1" }]); }); test("replaces the canonical epoch without replaying reconstructed parent rows", async () => { @@ -87,7 +87,7 @@ describe("AgentManager rewind", () => { text: `rewound ${index}`, })); session.history = history; - const epochBefore = manager.fetchTimeline(agentId, { limit: 0 }).epoch; + const epochBefore = (await manager.fetchTimeline(agentId, { limit: 0 })).epoch; const events: string[] = []; const unsubscribe = manager.subscribe((event) => events.push(event.type), { replayState: false, @@ -96,7 +96,7 @@ describe("AgentManager rewind", () => { await manager.rewind(agentId, "message-1", "conversation"); unsubscribe(); - const replacement = manager.fetchTimeline(agentId, { limit: 0 }); + const replacement = await manager.fetchTimeline(agentId, { limit: 0 }); expect(replacement.epoch).not.toBe(epochBefore); expect(replacement.rows).toEqual([ expect.objectContaining({ diff --git a/packages/server/src/server/agent/test-utils/plugin-stream-memory.ts b/packages/server/src/server/agent/test-utils/plugin-stream-memory.ts index 948202bae2..0807b1ec45 100644 --- a/packages/server/src/server/agent/test-utils/plugin-stream-memory.ts +++ b/packages/server/src/server/agent/test-utils/plugin-stream-memory.ts @@ -6,7 +6,7 @@ import { setTimeout as delay } from "node:timers/promises"; import type { ProviderEvent, ProviderRegistration } from "@getpaseo/plugin/server/provider"; import pino from "pino"; import { PluginAgentClientRegistry } from "../plugin-provider.js"; -import { InMemoryAgentTimelineStore } from "../agent-timeline-store.js"; +import { AgentTimeline } from "../timeline/index.js"; import { AgentManager } from "../agent-manager.js"; import { AgentStorage } from "../agent-storage.js"; import { buildProviderRegistry } from "../provider-registry.js"; @@ -67,12 +67,12 @@ const registrations: ProviderRegistration[] = Array.from({ length: 3 }, (_, inde }, })); plugins.replace(registrations); -const store = new InMemoryAgentTimelineStore(); +const timeline = new AgentTimeline(); const manager = new AgentManager({ clients: plugins.clients(), providerDefinitions: buildProviderRegistry(logger, { pluginProviders: plugins.definitions() }), registry: new AgentStorage(root, logger), - timelineStore: store, + timeline, logger, }); const agents = await Promise.all( @@ -143,7 +143,7 @@ global.gc(); const retained = process.memoryUsage().heapUsed - before; assert(retained < 24 * 1024 * 1024, `Retained heap exceeded 24 MiB: ${retained} bytes`); for (let index = 0; index < sessions.length; index++) { - const items = store.getItems(agents[index]!.id); + const items = (await timeline.readRows({ agentId: agents[index]!.id })).map((row) => row.item); assert.equal(items.length, steps * 2); for (let step = 0; step < steps; step++) { const item = items[step * 2]!; @@ -160,7 +160,7 @@ for (let index = 0; index < sessions.length; index++) { assert.equal(rootItems, steps * 2); assert.equal(childItems, steps * 2); await manager.archiveAgent(agents[index]!.id); - assert.equal(store.has(agents[index]!.id), false); + assert.equal(await timeline.describe({ agentId: agents[index]!.id }), null); assert.equal( manager.listAgents().some((agent) => agent.id === agents[index]!.id), false, diff --git a/packages/server/src/server/agent/test-utils/timeline-responsiveness.ts b/packages/server/src/server/agent/test-utils/timeline-responsiveness.ts index bfe5ee59c4..3e57ed0531 100644 --- a/packages/server/src/server/agent/test-utils/timeline-responsiveness.ts +++ b/packages/server/src/server/agent/test-utils/timeline-responsiveness.ts @@ -1,23 +1,27 @@ import { setImmediate as nextTurn } from "node:timers/promises"; import { measureMainThreadWork } from "../../../test-utils/main-thread-work.js"; -import { InMemoryAgentTimelineStore } from "../agent-timeline-store.js"; +import { AgentTimeline } from "../timeline/index.js"; // Measure a cold fetch on its own thread, without Vitest reporting or callbacks. const wholeTurns = process.argv[2] === "true"; -const store = new InMemoryAgentTimelineStore(); -store.initialize("large"); -for (let index = 0; index < 100_000; index++) { - store.append("large", { type: "user_message", text: String(index) }); -} +const timeline = new AgentTimeline(); +const scope = { agentId: "large" }; +await timeline.create(scope); +await timeline.append( + scope, + Array.from({ length: 100_000 }, (_, index) => ({ + item: { type: "user_message" as const, text: String(index) }, + })), +); // Finish fixture allocation and collect its temporary garbage before measuring. globalThis.gc!(); await nextTurn(); const measured = await measureMainThreadWork( `timeline tail main-thread work (wholeTurns=${wholeTurns})`, - () => { - let page = store.fetch("large", { limit: 200, wholeTurns }); + async () => { + let page = await timeline.fetch(scope, { limit: 200, wholeTurns }); for (let repeat = 1; repeat < 8; repeat++) - page = store.fetch("large", { limit: 200, wholeTurns }); + page = await timeline.fetch(scope, { limit: 200, wholeTurns }); return page; }, ); diff --git a/packages/server/src/server/agent/timeline-projection.ts b/packages/server/src/server/agent/timeline-projection.ts index bd5357e581..7333882adc 100644 --- a/packages/server/src/server/agent/timeline-projection.ts +++ b/packages/server/src/server/agent/timeline-projection.ts @@ -308,7 +308,6 @@ interface ProjectedTimelinePageOptions { export class TimelineProjection { private readonly rows: ProjectedTimelineRow[] = []; private readonly identities = new Map(); - private lastAssistantRowIndex: number | null = null; // Disjoint source-coverage spans in display order. A tail starting inside a // span must include its first anchor to cover interleaved lifecycle updates. private readonly spans: { start: number; endSeq: number }[] = []; @@ -356,7 +355,6 @@ export class TimelineProjection { return this.rows.length - 1; } if (identity !== null) this.identities.set(identity, this.rows.length); - if (entry.item.type === "assistant_message") this.lastAssistantRowIndex = this.rows.length; this.rows.push({ ...entry, seq: entry.seqEnd }); const last = this.spans.at(-1); if (last && last.endSeq >= entry.seqStart) { @@ -394,6 +392,10 @@ export class TimelineProjection { return this.rows.map((row) => ({ ...row })); } + get size(): number { + return this.rows.length; + } + selectPage(input: ProjectedTimelinePageOptions): ProjectedTimelinePageSelection { if (this.rows.length === 0) { const cursor = @@ -522,9 +524,20 @@ export class TimelineProjection { return last?.endSeq === seq ? this.rows[last.rowIndex].item : null; } + /** + * The text of the trailing run of assistant rows, joined in order. Rows after the run are + * skipped; the first non-assistant row before it ends the run. + */ getLastAssistantMessage(): string | null { - const row = this.lastAssistantRowIndex === null ? null : this.rows[this.lastAssistantRowIndex]; - return row?.item.type === "assistant_message" ? row.item.text : null; + let end = this.rows.length; + while (end > 0 && this.rows[end - 1].item.type !== "assistant_message") end -= 1; + let start = end; + while (start > 0 && this.rows[start - 1].item.type === "assistant_message") start -= 1; + if (start === end) return null; + return this.rows + .slice(start, end) + .map((row) => (row.item.type === "assistant_message" ? row.item.text : "")) + .join(""); } enrichSubmittedUserMessage( diff --git a/packages/server/src/server/agent/timeline/agent-timeline.test.ts b/packages/server/src/server/agent/timeline/agent-timeline.test.ts new file mode 100644 index 0000000000..4feabaf584 --- /dev/null +++ b/packages/server/src/server/agent/timeline/agent-timeline.test.ts @@ -0,0 +1,659 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import type { AgentTimelineItem } from "../agent-sdk-types.js"; +import { limitAgentTimelineItemContent } from "../agent-timeline-content.js"; +import type { AgentTimelineFetchResult, AgentTimelineRow } from "../agent-timeline-store-types.js"; +import { DeferredTimelineBackend } from "./testing.js"; +import { selectReferenceProjectedTimelinePage } from "../test-utils/timeline-page-reference.js"; +import { AgentTimeline, type TimelineCommit, type TimelineScope } from "./index.js"; +import { UnknownTimelineError } from "./internal/backend.js"; + +const root = (agentId: string): TimelineScope => ({ agentId }); + +async function append( + timeline: AgentTimeline, + agentId: string, + item: AgentTimelineItem, + options?: { turnId?: string }, +): Promise { + const { commits } = await timeline.append(root(agentId), [{ item, ...options }]); + const [commit] = commits; + return commit.row; +} + +describe.each([ + ["memory", () => new AgentTimeline()], + ["deferred", () => new AgentTimeline(new DeferredTimelineBackend())], +])("AgentTimeline over a %s backend", (_name, createTimeline) => { + it("lets a read observe writes called before it without awaiting them", async () => { + const timeline = createTimeline(); + void timeline.create(root("a"), { epoch: "e" }); + const published: number[] = []; + for (const text of ["one", "two", "three"]) { + void timeline.ingest( + root("a"), + { item: { type: "user_message", text } }, + { publish: ({ row }) => published.push(row.seq) }, + ); + } + const rows = await timeline.readRows(root("a")); + expect(rows.map((row) => row.item)).toEqual([ + { type: "user_message", text: "one" }, + { type: "user_message", text: "two" }, + { type: "user_message", text: "three" }, + ]); + expect(published).toEqual([1, 2, 3]); + }); + + it("publishes each commit before the next operation starts", async () => { + const timeline = createTimeline(); + await timeline.create(root("a"), { epoch: "e" }); + const order: string[] = []; + const first = timeline.ingest( + root("a"), + { item: { type: "assistant_message", text: "A" } }, + { publish: () => order.push("publish A") }, + ); + const settled = timeline.settled(root("a")).then(() => order.push("settled")); + const second = timeline.ingest( + root("a"), + { item: { type: "assistant_message", text: "B" } }, + { publish: () => order.push("publish B") }, + ); + await Promise.all([first, settled, second]); + expect(order).toEqual(["publish A", "settled", "publish B"]); + }); + + it("rejects a failed write without publishing it or consuming a sequence number", async () => { + const backend = new DeferredTimelineBackend(); + const timeline = new AgentTimeline(backend); + await timeline.create(root("a"), { epoch: "e" }); + const published: TimelineCommit[] = []; + backend.failNextAppend = true; + const failed = timeline.ingest( + root("a"), + { item: { type: "user_message", text: "lost" } }, + { publish: (commit) => published.push(commit) }, + ); + const next = timeline.ingest( + root("a"), + { item: { type: "user_message", text: "kept" } }, + { publish: (commit) => published.push(commit) }, + ); + await expect(failed).rejects.toThrow("disk full"); + await expect(next).resolves.toMatchObject({ kind: "recorded", row: { seq: 1 } }); + expect(published.map(({ row }) => row.item)).toEqual([{ type: "user_message", text: "kept" }]); + expect((await timeline.fetch(root("a"))).rows.map((row) => row.seq)).toEqual([1]); + }); + + it("records a submitted prompt once per client message id", async () => { + const timeline = createTimeline(); + await timeline.create(root("a"), { epoch: "e" }); + const prompt: AgentTimelineItem = { type: "user_message", text: "Hi", clientMessageId: "c1" }; + const published: number[] = []; + const publish = ({ row }: TimelineCommit) => published.push(row.seq); + const first = await timeline.ingest(root("a"), { item: prompt, turnId: "t1" }, { publish }); + const echo = await timeline.ingest(root("a"), { item: prompt }, { publish }); + expect(first).toMatchObject({ kind: "recorded", epoch: "e", row: { seq: 1, turnId: "t1" } }); + expect(echo).toMatchObject({ kind: "duplicate", epoch: "e", row: { seq: 1 } }); + expect(published).toEqual([1]); + await timeline.attachProviderMessageId(root("a"), "c1", "provider-1"); + expect(await timeline.findSubmittedPrompt(root("a"), "c1")).toMatchObject({ + seq: 1, + providerMessageId: "provider-1", + }); + }); + + it("keeps a commit whose publication throws and lets later writes proceed", async () => { + const timeline = createTimeline(); + await timeline.create(root("a"), { epoch: "e" }); + const prompt: AgentTimelineItem = { type: "user_message", text: "Hi", clientMessageId: "c1" }; + const failure = new Error("subscriber failed"); + const throwing = () => { + throw failure; + }; + const published: number[] = []; + const unpublished = timeline.ingest(root("a"), { item: prompt }, { publish: throwing }); + const next = timeline.ingest( + root("a"), + { item: { type: "assistant_message", text: "reply" } }, + { publish: ({ row }) => published.push(row.seq) }, + ); + const batch = timeline.append(root("a"), [{ item: { type: "user_message", text: "batch" } }], { + publish: throwing, + }); + await expect(unpublished).resolves.toMatchObject({ + kind: "unpublished", + epoch: "e", + row: { seq: 1, item: prompt }, + error: failure, + }); + await expect(next).resolves.toMatchObject({ kind: "recorded", row: { seq: 2 } }); + await expect(batch).resolves.toMatchObject({ + kind: "unpublished", + commits: [{ row: { seq: 3 } }], + error: failure, + }); + expect(published).toEqual([2]); + // The committed prompt is not recorded twice when ingested again. + await expect(timeline.ingest(root("a"), { item: prompt })).resolves.toMatchObject({ + kind: "duplicate", + row: { seq: 1 }, + }); + expect((await timeline.readRows(root("a"))).map((row) => row.seq)).toEqual([1, 2, 3]); + }); + + it("reads the trailing run of assistant messages as the last assistant message", async () => { + const tool = (callId: string): AgentTimelineItem => ({ + type: "tool_call", + callId, + name: "shell", + status: "completed", + error: null, + detail: { type: "unknown", input: null, output: null }, + }); + const timeline = createTimeline(); + await timeline.create(root("a")); + expect(await timeline.lastAssistantMessage(root("a"))).toBeNull(); + await timeline.append(root("a"), [{ item: tool("only") }]); + expect(await timeline.lastAssistantMessage(root("a"))).toBeNull(); + await timeline.append(root("a"), [ + { item: { type: "assistant_message", text: "Earlier reply" } }, + { item: { type: "user_message", text: "Next" } }, + { item: { type: "assistant_message", text: "Before the tool. " } }, + { item: tool("call-1") }, + { item: { type: "assistant_message", text: "Hel", messageId: "m1" } }, + { item: { type: "assistant_message", text: "lo", messageId: "m1" } }, + { item: { type: "assistant_message", text: " world", messageId: "m2" } }, + { item: tool("call-2") }, + ]); + expect(await timeline.lastAssistantMessage(root("a"))).toBe("Hello world"); + await timeline.append(root("a"), [ + { item: { type: "user_message", text: "Again" } }, + { item: { type: "assistant_message", text: "Bye" } }, + ]); + expect(await timeline.lastAssistantMessage(root("a"))).toBe("Bye"); + }); + + it("keeps provider child history apart from its parent's", async () => { + const timeline = createTimeline(); + const child = { agentId: "a", subagentId: "child" }; + await timeline.create(root("a"), { epoch: "parent" }); + await timeline.open(child); + await append(timeline, "a", { type: "assistant_message", text: "parent" }); + await timeline.append(child, [{ item: { type: "assistant_message", text: "child" } }]); + expect((await timeline.readRows(child)).map((row) => [row.seq, row.item])).toEqual([ + [1, { type: "assistant_message", text: "child" }], + ]); + await timeline.delete(child); + expect(await timeline.describe(child)).toBeNull(); + expect(await timeline.describe(root("a"))).toEqual({ epoch: "parent", size: 1 }); + await expect(timeline.readRows(child)).rejects.toBeInstanceOf(UnknownTimelineError); + }); + + it("gives a recreated history a new identity", async () => { + const timeline = createTimeline(); + const before = await timeline.create(root("a")); + await append(timeline, "a", { type: "user_message", text: "old" }); + const after = await timeline.create(root("a")); + expect(after.epoch).not.toBe(before.epoch); + expect(after.size).toBe(0); + expect(timeline.residentSize(root("a"))).toBe(0); + expect(await timeline.open(root("a"))).toEqual(after); + }); + + it("bounds ingested content", async () => { + const timeline = createTimeline(); + await timeline.create(root("a")); + const text = "a".repeat(400 * 1024); + const { row } = await timeline.ingest(root("a"), { + item: { type: "assistant_message", text }, + }); + expect(Buffer.byteLength(JSON.stringify(row.item))).toBeLessThanOrEqual(256 * 1024); + }); +}); + +describe("history paging", () => { + it("resets stale forward cursors to complete available turns", async () => { + const timeline = new AgentTimeline(); + await timeline.create(root("agent"), { epoch: "current" }); + for (let index = 0; index < 60; index++) + await append( + timeline, + "agent", + { + type: "tool_call", + callId: `call-${index}`, + name: "shell", + status: "completed", + error: null, + detail: { type: "unknown", input: null, output: null }, + }, + { turnId: "run" }, + ); + const page = await timeline.fetch(root("agent"), { + direction: "after", + cursor: { epoch: "old", seq: 10 }, + limit: 40, + wholeTurns: true, + }); + expect(page.rows).toHaveLength(60); + expect(page).toMatchObject({ + reset: true, + staleCursor: true, + startSeq: 1, + endSeq: 60, + hasOlder: false, + hasNewer: false, + }); + }); + + const seededRows: AgentTimelineRow[] = [ + { + seq: 5, + timestamp: "2026-01-01T00:00:00.000Z", + item: { type: "assistant_message", text: "five", messageId: "five" }, + }, + { + seq: 6, + timestamp: "2026-01-01T00:00:01.000Z", + item: { type: "assistant_message", text: "six", messageId: "six" }, + }, + { + seq: 7, + timestamp: "2026-01-01T00:00:02.000Z", + item: { type: "assistant_message", text: "seven", messageId: "seven" }, + }, + ]; + + it("clamps an overshooting before cursor into the bounded tail window", async () => { + const timeline = new AgentTimeline(); + await timeline.create(root("agent-1"), { epoch: "epoch-1", nextSeq: 8, rows: seededRows }); + + const result = await timeline.fetch(root("agent-1"), { + direction: "before", + cursor: { epoch: "epoch-1", seq: 100 }, + limit: 2, + }); + + expect(result).toMatchObject({ + epoch: "epoch-1", + direction: "before", + reset: false, + staleCursor: false, + gap: false, + window: { minSeq: 5, maxSeq: 7, nextSeq: 8 }, + hasOlder: true, + hasNewer: false, + rows: [ + { + seq: 6, + timestamp: "2026-01-01T00:00:01.000Z", + item: { type: "assistant_message", text: "six", messageId: "six" }, + }, + { + seq: 7, + timestamp: "2026-01-01T00:00:02.000Z", + item: { type: "assistant_message", text: "seven", messageId: "seven" }, + }, + ], + }); + }); + + it("returns a bounded reset window when an after cursor is behind retained history", async () => { + const timeline = new AgentTimeline(); + await timeline.create(root("agent-1"), { epoch: "epoch-1", nextSeq: 8, rows: seededRows }); + + const result = await timeline.fetch(root("agent-1"), { + direction: "after", + cursor: { epoch: "epoch-1", seq: 1 }, + limit: 1, + }); + + expect(result).toMatchObject({ + epoch: "epoch-1", + direction: "after", + reset: true, + staleCursor: false, + gap: true, + window: { minSeq: 5, maxSeq: 7, nextSeq: 8 }, + hasOlder: true, + hasNewer: false, + rows: [ + { + seq: 7, + timestamp: "2026-01-01T00:00:02.000Z", + item: { type: "assistant_message", text: "seven", messageId: "seven" }, + }, + ], + }); + }); +}); + +describe("projected timeline retention", () => { + it("retains one full tool state while streaming every source update", async () => { + const timeline = new AgentTimeline(); + await timeline.create(root("agent")); + for (let seq = 1; seq <= 2000; seq++) { + const item = { + type: "tool_call" as const, + callId: "child", + name: "task", + status: "running" as const, + error: null, + detail: { type: "plain_text" as const, label: "Child", text: "x".repeat(seq * 128) }, + }; + expect(await append(timeline, "agent", item)).toMatchObject({ + seq, + item: limitAgentTimelineItemContent(item), + }); + } + const result = await timeline.fetch(root("agent"), { limit: 0 }); + expect(result.rows).toHaveLength(1); + expect(JSON.stringify(result.rows).length).toBeLessThan(260_000); + expect(result.window.maxSeq).toBe(2000); + }); + + it("catches up a mid-message cursor with the complete projected message", async () => { + const timeline = new AgentTimeline(); + await timeline.create(root("agent"), { epoch: "e" }); + await append(timeline, "agent", { type: "assistant_message", messageId: "m", text: "A" }); + await append(timeline, "agent", { type: "assistant_message", messageId: "m", text: "B" }); + expect( + (await timeline.fetch(root("agent"), { direction: "after", cursor: { epoch: "e", seq: 1 } })) + .rows, + ).toMatchObject([ + { + item: { text: "AB" }, + seqStart: 1, + seqEnd: 2, + sourceSeqRanges: [{ startSeq: 1, endSeq: 2 }], + }, + ]); + }); + + it("serves a merged assistant message within the item budget", async () => { + const timeline = new AgentTimeline(); + await timeline.create(root("agent"), { epoch: "e" }); + const chunk = "a".repeat(200 * 1024); + for (let index = 0; index < 10; index++) { + await append(timeline, "agent", { type: "assistant_message", messageId: "m", text: chunk }); + } + const [row] = (await timeline.fetch(root("agent"), { limit: 1 })).rows; + expect(row).toMatchObject({ seqStart: 1, seqEnd: 10 }); + expect(row.item).toMatchObject({ type: "assistant_message", messageId: "m" }); + expect(Buffer.byteLength(JSON.stringify(row.item))).toBeLessThanOrEqual(256 * 1024); + // The resident projection keeps the whole message for completion notifications. + const [resident] = await timeline.readRows(root("agent")); + expect(resident.item.type === "assistant_message" && resident.item.text).toHaveLength( + 10 * chunk.length, + ); + }); +}); + +describe("projected sequence ownership", () => { + it("preserves fetched coverage when another source chunk arrives", async () => { + const timeline = new AgentTimeline(); + await timeline.create(root("a")); + await append(timeline, "a", { type: "assistant_message", text: "A" }); + const before = await timeline.fetch(root("a")); + await append(timeline, "a", { type: "assistant_message", text: "B" }); + expect(before.rows[0]).toMatchObject({ + item: { text: "A" }, + seqEnd: 1, + sourceSeqRanges: [{ startSeq: 1, endSeq: 1 }], + }); + expect((await timeline.fetch(root("a"))).rows[0]).toMatchObject({ + item: { text: "AB" }, + seqEnd: 2, + sourceSeqRanges: [{ startSeq: 1, endSeq: 2 }], + }); + }); + + it("preserves source positions when seeding a projected history whose last update is an earlier tool", async () => { + const timeline = new AgentTimeline(); + await timeline.create(root("a")); + const tool = { + type: "tool_call" as const, + callId: "t", + name: "shell", + error: null, + detail: { type: "plain_text" as const, label: "work" }, + }; + await append(timeline, "a", { ...tool, status: "running" }); + await append(timeline, "a", { type: "assistant_message", text: "Answer" }); + await append(timeline, "a", { ...tool, status: "completed" }); + await timeline.create(root("b"), { rows: await timeline.readRows(root("a")) }); + expect(await timeline.lastItem(root("b"))).toMatchObject({ + type: "tool_call", + status: "completed", + }); + expect((await timeline.readRows(root("b")))[1]?.item).toEqual({ + type: "assistant_message", + text: "Answer", + }); + expect((await append(timeline, "b", { type: "user_message", text: "next" })).seq).toBe(4); + expect(await timeline.lastItem(root("b"))).toEqual({ type: "user_message", text: "next" }); + expect((await timeline.fetch(root("b"))).rows.map((row) => row.seqStart)).toEqual([1, 2, 4]); + }); + + it("keeps differently identified assistant messages as separate rows", async () => { + const timeline = new AgentTimeline(); + await timeline.create(root("a")); + await append(timeline, "a", { type: "assistant_message", messageId: "first", text: "First" }); + await append(timeline, "a", { type: "assistant_message", messageId: "second", text: "Sec" }); + await append(timeline, "a", { type: "assistant_message", messageId: "second", text: "ond" }); + expect((await timeline.readRows(root("a"))).map((row) => row.item)).toEqual([ + { type: "assistant_message", messageId: "first", text: "First" }, + { type: "assistant_message", messageId: "second", text: "Second" }, + ]); + }); +}); + +it("includes transitive tool updates in a contiguous projected tail", async () => { + const timeline = new AgentTimeline(); + await timeline.create(root("a")); + const tool = (callId: string, status: "running" | "completed") => ({ + type: "tool_call" as const, + callId, + name: "shell", + status, + error: null, + detail: { type: "plain_text" as const, label: "work" }, + }); + await append(timeline, "a", tool("first", "running")); + await append(timeline, "a", tool("second", "running")); + await append(timeline, "a", { type: "user_message", text: "Continue" }); + await append(timeline, "a", tool("first", "completed")); + await append(timeline, "a", { type: "assistant_message", text: "Answer" }); + await append(timeline, "a", tool("second", "completed")); + const tail = await timeline.fetch(root("a"), { limit: 1 }); + expect(tail.rows.map((row) => row.seqStart)).toEqual([1, 2, 3, 5]); + expect(tail).toMatchObject({ startSeq: 1, endSeq: 6, hasOlder: false }); +}); + +async function measureTimelineTail(wholeTurns: boolean) { + const fixture = fileURLToPath( + new URL("../test-utils/timeline-responsiveness.ts", import.meta.url), + ); + const { stdout, stderr } = await promisify(execFile)( + process.execPath, + ["--expose-gc", "--import", "tsx", fixture, String(wholeTurns)], + { timeout: 15_000 }, + ); + process.stderr.write(stderr); + return JSON.parse(stdout) as { + result: AgentTimelineFetchResult; + blockingMs: number; + }; +} + +it("excludes unrelated test-runner CPU from the timeline fetch budget", async () => { + let unrelatedWork: ReturnType; + const burnCpu = () => { + const threadCpuUsage: (previous?: NodeJS.CpuUsage) => NodeJS.CpuUsage = Reflect.get( + process, + "threadCpuUsage", + ); + const start = threadCpuUsage(); + let elapsed: NodeJS.CpuUsage; + do { + elapsed = threadCpuUsage(start); + } while (elapsed.user + elapsed.system < 80_000); + unrelatedWork = setImmediate(burnCpu); + }; + unrelatedWork = setImmediate(burnCpu); + try { + const { result: tail, blockingMs } = await measureTimelineTail(false); + expect(tail.rows).toHaveLength(200); + expect(tail.startSeq).toBe(99_801); + expect(tail.endSeq).toBe(100_000); + expect(blockingMs).toBeLessThan(50); + } finally { + clearImmediate(unrelatedWork); + } +}); + +it.each([false, true])( + "serves tail 200 from 100,000 retained rows without stalling the loop (wholeTurns=%s)", + async (wholeTurns) => { + const { result: tail, blockingMs } = await measureTimelineTail(wholeTurns); + expect(tail.rows).toHaveLength(200); + expect(tail.startSeq).toBe(99_801); + expect(tail.endSeq).toBe(100_000); + expect(blockingMs).toBeLessThan(50); + }, +); + +it("matches the previous reprojecting selector on seeded randomized timelines", async () => { + let seed = 0x76aabbcc; + const random = (bound: number) => { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; + return Math.floor((seed / 0x1_0000_0000) * bound); + }; + for (let scenario = 0; scenario < 100; scenario++) { + const timeline = new AgentTimeline(); + await timeline.create(root("a"), { epoch: "e" }); + for (let seq = 1; seq <= 100; seq++) { + const identity = String(random(8)); + const text = String(seq); + const items: AgentTimelineItem[] = [ + { type: "user_message", text }, + { type: "assistant_message", text }, + { type: "assistant_message", text, messageId: identity }, + { type: "reasoning", text }, + { + type: "tool_call", + callId: identity, + name: "shell", + status: "running", + error: null, + detail: { type: "plain_text", label: text }, + }, + { + type: "plugin", + id: identity, + pluginId: "test", + kind: "state", + version: 1, + data: { text }, + }, + ]; + await append(timeline, "a", items[random(items.length)], { turnId: String(random(3)) }); + } + // Include rehydration of already-projected rows, with non-monotonic seqEnd. + await timeline.create(root("b"), { rows: await timeline.readRows(root("a")), epoch: "e" }); + for (const agentId of ["a", "b"]) { + await assertSelectionMatchesReference(timeline, agentId); + await append(timeline, agentId, { type: "assistant_message", text: "later" }); + await assertSelectionMatchesReference(timeline, agentId); + } + } +}); + +async function assertSelectionMatchesReference( + timeline: AgentTimeline, + agentId: string, +): Promise { + const rows = await timeline.readRows(root(agentId)); + for (const direction of ["tail", "before", "after"] as const) { + for (const limit of [0, 1, 7, 200]) { + for (const cursorSeq of [0, 1, 25, 50, 99, 100, 150]) { + const actual = await timeline.fetch(root(agentId), { + direction, + limit, + cursor: { epoch: "e", seq: cursorSeq }, + }); + const expected = selectReferenceProjectedTimelinePage({ + rows, + bounds: actual.window, + direction, + limit, + cursorSeq, + }); + expect({ + ...expected, + entries: undefined, + rows: expected.entries.map((entry) => Object.assign({ seq: entry.seqEnd }, entry)), + }).toEqual({ + entries: undefined, + rows: actual.rows, + startSeq: actual.startSeq, + endSeq: actual.endSeq, + hasOlder: actual.hasOlder, + hasNewer: actual.hasNewer, + }); + } + } + } +} + +it("continues a scroll across appends and lifecycle collapse without losing source coverage", async () => { + const timeline = new AgentTimeline(); + await timeline.create(root("a"), { epoch: "e" }); + const tool: AgentTimelineItem = { + type: "tool_call", + callId: "t", + name: "shell", + status: "running", + error: null, + detail: { type: "plain_text", label: "work" }, + }; + await append(timeline, "a", tool); + for (let seq = 2; seq <= 5; seq++) + await append(timeline, "a", { type: "user_message", text: String(seq) }); + const first = await timeline.fetch(root("a"), { + direction: "after", + cursor: { epoch: "e", seq: 0 }, + limit: 2, + }); + expect(first.endSeq).toBe(2); + await append(timeline, "a", { ...tool, status: "completed" }); + await append(timeline, "a", { type: "assistant_message", text: "Done" }); + expect(first.rows[0].item).toMatchObject({ status: "running" }); + const second = await timeline.fetch(root("a"), { + direction: "after", + cursor: { epoch: "e", seq: first.endSeq! }, + limit: 2, + }); + expect(second.rows.map((row) => row.seqStart)).toEqual([3, 4]); + expect(second.endSeq).toBe(4); + const third = await timeline.fetch(root("a"), { + direction: "after", + cursor: { epoch: "e", seq: second.endSeq! }, + limit: 2, + }); + expect(third.rows.map((row) => row.seqStart)).toEqual([1, 5]); + expect(third.rows[0].item).toMatchObject({ status: "completed" }); + expect(third.endSeq).toBe(6); + const last = await timeline.fetch(root("a"), { + direction: "after", + cursor: { epoch: "e", seq: third.endSeq! }, + limit: 2, + }); + expect(last.rows.map((row) => row.item)).toEqual([{ type: "assistant_message", text: "Done" }]); + expect(last.endSeq).toBe(7); + expect(last.hasNewer).toBe(false); +}); diff --git a/packages/server/src/server/agent/timeline/index.ts b/packages/server/src/server/agent/timeline/index.ts new file mode 100644 index 0000000000..958098de67 --- /dev/null +++ b/packages/server/src/server/agent/timeline/index.ts @@ -0,0 +1,10 @@ +export { + AgentTimeline, + type TimelineAppendResult, + type TimelineCommit, + type TimelineEntry, + type TimelineHead, + type TimelineIngestResult, + type TimelineScope, + type TimelineSeed, +} from "./internal/agent-timeline.js"; diff --git a/packages/server/src/server/agent/timeline/internal/agent-timeline.ts b/packages/server/src/server/agent/timeline/internal/agent-timeline.ts new file mode 100644 index 0000000000..18936c7039 --- /dev/null +++ b/packages/server/src/server/agent/timeline/internal/agent-timeline.ts @@ -0,0 +1,242 @@ +import type { AgentTimelineItem } from "../../agent-sdk-types.js"; +import { limitAgentTimelineItemContent } from "../../agent-timeline-content.js"; +import type { + AgentTimelineFetchOptions, + AgentTimelineFetchResult, + AgentTimelineRow, +} from "../../agent-timeline-store-types.js"; +import type { ProjectedTimelineRow } from "../../timeline-projection.js"; +import type { + AgentTimelineBackend, + TimelineEntry, + TimelineHead, + TimelineScope, + TimelineSeed, +} from "./backend.js"; +import { UnknownTimelineError } from "./backend.js"; +import { MemoryTimelineBackend } from "./memory-backend.js"; + +export type { TimelineEntry, TimelineHead, TimelineScope, TimelineSeed } from "./backend.js"; + +export interface TimelineCommit { + row: AgentTimelineRow; + epoch: string; +} + +export type TimelineIngestResult = + | { kind: "recorded"; row: AgentTimelineRow; epoch: string } + /** + * The row is committed but `publish` threw. The commit stands and nothing retries the + * publication; ingesting the same submitted prompt again returns `duplicate`. + */ + | { kind: "unpublished"; row: AgentTimelineRow; epoch: string; error: unknown } + /** A submitted prompt with this `clientMessageId` is already recorded; nothing was published. */ + | { kind: "duplicate"; row: AgentTimelineRow; epoch: string }; + +export type TimelineAppendResult = + | { kind: "recorded"; commits: TimelineCommit[] } + /** The rows are committed but `publish` threw; see `TimelineIngestResult`. */ + | { kind: "unpublished"; commits: TimelineCommit[]; error: unknown }; + +export interface TimelineIngestOptions { + /** + * Runs right after the commit, before any later operation on the agent's timeline starts, so + * publications leave in commit order. Not called when the commit fails or is a duplicate. + * A throw does not undo the commit: the write resolves `unpublished` with the error. + */ + publish?: (commit: TimelineCommit) => void; +} + +export interface TimelineAppendOptions { + /** See `TimelineIngestOptions.publish`. */ + publish?: (commits: TimelineCommit[]) => void; +} + +/** + * Owner of agent timeline history: item identity, sequence numbers, projection, and reads. + * + * Operations on one agent (its root history and its provider children) run in call order. A read + * observes every write called before it, whether or not the caller awaited that write. A write + * that fails rejects, consumes no sequence number, and publishes nothing. + */ +export class AgentTimeline { + private readonly tails = new Map>(); + private readonly sizes = new Map(); + + constructor(private readonly backend: AgentTimelineBackend = new MemoryTimelineBackend()) {} + + describe(scope: TimelineScope): Promise { + return this.run(scope, () => this.backend.head(scope)); + } + + /** Creates the scope's history with a new identity, replacing any existing history. */ + create(scope: TimelineScope, seed: TimelineSeed = {}): Promise { + return this.run(scope, async () => this.track(scope, await this.backend.create(scope, seed))); + } + + /** Returns the existing history, or creates an empty one. */ + open(scope: TimelineScope): Promise { + return this.run(scope, async () => { + const existing = await this.backend.head(scope); + return existing ?? this.track(scope, await this.backend.create(scope, {})); + }); + } + + delete(scope: TimelineScope): Promise { + return this.run(scope, async () => { + await this.backend.delete(scope); + this.sizes.delete(sizeKey(scope)); + }); + } + + ingest( + scope: TimelineScope, + entry: TimelineEntry, + options?: TimelineIngestOptions, + ): Promise { + return this.run(scope, async () => { + const clientMessageId = submittedPromptId(entry.item); + if (clientMessageId) { + const existing = await this.backend.findSubmittedPrompt(scope, clientMessageId); + if (existing) return { kind: "duplicate", row: existing, epoch: await this.epoch(scope) }; + } + const [commit] = await this.commit(scope, [entry]); + const failure = publishCommitted(options?.publish, commit); + return failure + ? { kind: "unpublished", ...commit, error: failure.error } + : { kind: "recorded", ...commit }; + }); + } + + /** Appends provider history entries in one commit, without submitted-prompt deduplication. */ + append( + scope: TimelineScope, + entries: readonly TimelineEntry[], + options?: TimelineAppendOptions, + ): Promise { + return this.run(scope, async () => { + const commits = entries.length === 0 ? [] : await this.commit(scope, entries); + const failure = publishCommitted(options?.publish, commits); + return failure + ? { kind: "unpublished", commits, error: failure.error } + : { kind: "recorded", commits }; + }); + } + + attachProviderMessageId( + scope: TimelineScope, + clientMessageId: string, + providerMessageId: string, + ): Promise { + return this.run(scope, () => + this.backend.attachProviderMessageId(scope, clientMessageId, providerMessageId), + ); + } + + findSubmittedPrompt( + scope: TimelineScope, + clientMessageId: string, + ): Promise { + return this.run(scope, () => this.backend.findSubmittedPrompt(scope, clientMessageId)); + } + + fetch( + scope: TimelineScope, + options: AgentTimelineFetchOptions = {}, + ): Promise { + return this.run(scope, () => this.backend.fetch(scope, options)); + } + + readRows(scope: TimelineScope): Promise { + return this.run(scope, () => this.backend.readRows(scope)); + } + + lastItem(scope: TimelineScope): Promise { + return this.run(scope, () => this.backend.lastItem(scope)); + } + + /** + * The text of the trailing run of assistant messages, joined in order (providers stream one + * reply as several messages). Non-assistant items after the run are skipped; the first one + * before it ends the run. Null when the history has no assistant message. + */ + lastAssistantMessage(scope: TimelineScope): Promise { + return this.run(scope, () => this.backend.lastAssistantMessage(scope)); + } + + /** + * Resolves once every operation called before it has finished, including its publication. + * Callers use it so an event published outside the timeline cannot overtake queued writes. + */ + settled(scope: TimelineScope): Promise { + return this.run(scope, settledOperation); + } + + /** Projected row count as of the last committed write, for diagnostics. */ + residentSize(scope: TimelineScope): number | null { + return this.sizes.get(sizeKey(scope)) ?? null; + } + + private async commit( + scope: TimelineScope, + entries: readonly TimelineEntry[], + ): Promise { + const bounded = entries.map((entry) => ({ + ...entry, + item: limitAgentTimelineItemContent(entry.item), + })); + const { rows, head } = await this.backend.append(scope, bounded); + this.track(scope, head); + return rows.map((row) => ({ row, epoch: head.epoch })); + } + + private async epoch(scope: TimelineScope): Promise { + const head = await this.backend.head(scope); + if (!head) throw new UnknownTimelineError(scope); + return head.epoch; + } + + private track(scope: TimelineScope, head: TimelineHead): TimelineHead { + this.sizes.set(sizeKey(scope), head.size); + return head; + } + + private run(scope: TimelineScope, operation: () => T | Promise): Promise { + const key = scope.agentId; + const previous = this.tails.get(key) ?? Promise.resolve(); + const result = previous.then(operation); + const tail = result.then(settledOperation, settledOperation); + this.tails.set(key, tail); + void tail.finally(() => { + if (this.tails.get(key) === tail) this.tails.delete(key); + }); + return result; + } +} + +function settledOperation(): void {} + +interface PublishFailure { + error: unknown; +} + +/** Returns what `publish` threw, or null. A committed write stays committed either way. */ +function publishCommitted( + publish: ((committed: T) => void) | undefined, + committed: T, +): PublishFailure | null { + try { + publish?.(committed); + return null; + } catch (error) { + return { error }; + } +} + +function sizeKey(scope: TimelineScope): string { + return scope.subagentId === undefined ? scope.agentId : `${scope.agentId}\0${scope.subagentId}`; +} + +function submittedPromptId(item: AgentTimelineItem): string | null { + return item.type === "user_message" && item.clientMessageId ? item.clientMessageId : null; +} diff --git a/packages/server/src/server/agent/timeline/internal/backend.ts b/packages/server/src/server/agent/timeline/internal/backend.ts new file mode 100644 index 0000000000..5e350297ea --- /dev/null +++ b/packages/server/src/server/agent/timeline/internal/backend.ts @@ -0,0 +1,88 @@ +import type { AgentTimelineItem } from "../../agent-sdk-types.js"; +import type { + AgentTimelineFetchOptions, + AgentTimelineFetchResult, + AgentTimelineRow, +} from "../../agent-timeline-store-types.js"; +import type { ProjectedTimelineRow } from "../../timeline-projection.js"; + +/** Root history uses `agentId` alone; provider-child history adds the child's id. */ +export interface TimelineScope { + agentId: string; + subagentId?: string; +} + +export interface TimelineEntry { + item: AgentTimelineItem; + timestamp?: string; + turnId?: string; + providerMessageId?: string; +} + +export interface TimelineSeed { + entries?: readonly TimelineEntry[]; + /** Already-sequenced rows, e.g. an imported session. Takes precedence over `entries`. */ + rows?: readonly AgentTimelineRow[]; + nextSeq?: number; + epoch?: string; + timestamp?: string; +} + +export interface TimelineHead { + epoch: string; + /** Projected (displayed) row count. */ + size: number; +} + +export interface TimelineBackendAppendResult { + rows: AgentTimelineRow[]; + head: TimelineHead; +} + +type MaybePromise = T | Promise; + +/** + * Storage for the timeline owner. Each mutating call is one atomic commit: it either applies + * fully and returns, or throws having changed nothing. The owner serializes calls per agent, so + * a backend never sees concurrent calls for one agent. Callers outside the timeline module never + * see a backend. + */ +export interface AgentTimelineBackend { + create(scope: TimelineScope, seed: TimelineSeed): MaybePromise; + delete(scope: TimelineScope): MaybePromise; + head(scope: TimelineScope): MaybePromise; + /** Allocates the next sequence numbers and folds the entries into the projection. */ + append( + scope: TimelineScope, + entries: readonly TimelineEntry[], + ): MaybePromise; + attachProviderMessageId( + scope: TimelineScope, + clientMessageId: string, + providerMessageId: string, + ): MaybePromise; + findSubmittedPrompt( + scope: TimelineScope, + clientMessageId: string, + ): MaybePromise; + fetch( + scope: TimelineScope, + options: AgentTimelineFetchOptions, + ): MaybePromise; + readRows(scope: TimelineScope): MaybePromise; + lastItem(scope: TimelineScope): MaybePromise; + /** The trailing run of assistant messages joined, skipping later non-assistant rows. */ + lastAssistantMessage(scope: TimelineScope): MaybePromise; +} + +/** The scope has no history: it was never created, or it was deleted. */ +export class UnknownTimelineError extends Error { + constructor(readonly scope: TimelineScope) { + super( + scope.subagentId === undefined + ? `Unknown agent '${scope.agentId}'` + : `Unknown timeline '${scope.agentId}/${scope.subagentId}'`, + ); + this.name = "UnknownTimelineError"; + } +} diff --git a/packages/server/src/server/agent/timeline/internal/memory-backend.ts b/packages/server/src/server/agent/timeline/internal/memory-backend.ts new file mode 100644 index 0000000000..9efc43d842 --- /dev/null +++ b/packages/server/src/server/agent/timeline/internal/memory-backend.ts @@ -0,0 +1,177 @@ +import { randomUUID } from "node:crypto"; +import { limitAgentTimelineItemContent } from "../../agent-timeline-content.js"; +import type { AgentTimelineItem } from "../../agent-sdk-types.js"; +import type { + AgentTimelineFetchOptions, + AgentTimelineFetchResult, + AgentTimelineRow, +} from "../../agent-timeline-store-types.js"; +import { TimelineProjection, type ProjectedTimelineRow } from "../../timeline-projection.js"; +import type { + AgentTimelineBackend, + TimelineBackendAppendResult, + TimelineEntry, + TimelineHead, + TimelineScope, + TimelineSeed, +} from "./backend.js"; +import { UnknownTimelineError } from "./backend.js"; + +interface MemoryTimelineState { + epoch: string; + projection: TimelineProjection; + minSeq: number; + nextSeq: number; +} + +const DEFAULT_TIMELINE_FETCH_LIMIT = 200; + +function scopeKey(scope: TimelineScope): string { + return scope.subagentId === undefined ? scope.agentId : `${scope.agentId}\0${scope.subagentId}`; +} + +function cloneRow(row: T): T { + return { ...row }; +} + +function buildRow(entry: TimelineEntry, seq: number, timestamp: string): AgentTimelineRow { + return { + seq, + timestamp: entry.timestamp ?? timestamp, + item: entry.item, + ...(entry.turnId ? { turnId: entry.turnId } : {}), + ...(entry.providerMessageId ? { providerMessageId: entry.providerMessageId } : {}), + }; +} + +/** Volatile backend: history lives in process memory and is lost when the daemon exits. */ +export class MemoryTimelineBackend implements AgentTimelineBackend { + private readonly states = new Map(); + + create(scope: TimelineScope, seed: TimelineSeed): TimelineHead { + const timestamp = seed.timestamp ?? new Date().toISOString(); + const startSeq = seed.nextSeq ?? 1; + const rows = seed.rows?.length + ? seed.rows.map(cloneRow) + : (seed.entries ?? []).map((entry, index) => buildRow(entry, startSeq + index, timestamp)); + const nextSeq = rows.reduce((next, row) => Math.max(next, row.seq + 1), startSeq); + const projection = new TimelineProjection(rows); + const state: MemoryTimelineState = { + epoch: seed.epoch ?? randomUUID(), + projection, + minSeq: projection.getRows()[0]?.seqStart ?? 0, + nextSeq, + }; + this.states.set(scopeKey(scope), state); + return this.describe(state); + } + + delete(scope: TimelineScope): void { + this.states.delete(scopeKey(scope)); + } + + head(scope: TimelineScope): TimelineHead | null { + const state = this.states.get(scopeKey(scope)); + return state ? this.describe(state) : null; + } + + append(scope: TimelineScope, entries: readonly TimelineEntry[]): TimelineBackendAppendResult { + const state = this.requireState(scope); + const timestamp = new Date().toISOString(); + const rows = entries.map((entry, index) => buildRow(entry, state.nextSeq + index, timestamp)); + for (const row of rows) { + state.projection.append(row); + state.nextSeq = row.seq + 1; + if (state.minSeq === 0) state.minSeq = row.seq; + } + return { rows: rows.map(cloneRow), head: this.describe(state) }; + } + + attachProviderMessageId( + scope: TimelineScope, + clientMessageId: string, + providerMessageId: string, + ): AgentTimelineRow | null { + return this.requireState(scope).projection.enrichSubmittedUserMessage( + clientMessageId, + providerMessageId, + ); + } + + findSubmittedPrompt(scope: TimelineScope, clientMessageId: string): AgentTimelineRow | null { + const row = this.requireState(scope) + .projection.getRows() + .find( + (candidate) => + candidate.item.type === "user_message" && + candidate.item.clientMessageId === clientMessageId, + ); + return row ? cloneRow(row) : null; + } + + fetch(scope: TimelineScope, options: AgentTimelineFetchOptions): AgentTimelineFetchResult { + const state = this.requireState(scope); + const direction = options.direction ?? "tail"; + const cursor = options.cursor; + const hasRows = state.minSeq !== 0; + const window = { minSeq: state.minSeq, maxSeq: state.nextSeq - 1, nextSeq: state.nextSeq }; + const staleCursor = cursor !== undefined && cursor.epoch !== state.epoch; + const gap = + !staleCursor && + direction === "after" && + cursor !== undefined && + hasRows && + cursor.seq < state.minSeq - 1; + const reset = staleCursor || gap; + const page = state.projection.selectPage({ + bounds: window, + direction: reset ? "tail" : direction, + cursorSeq: cursor?.seq, + limit: options.limit ?? DEFAULT_TIMELINE_FETCH_LIMIT, + wholeTurns: options.wholeTurns, + }); + return { + epoch: state.epoch, + direction, + reset, + staleCursor, + gap, + window, + hasOlder: page.hasOlder, + hasNewer: page.hasNewer, + startSeq: page.startSeq, + endSeq: page.endSeq, + // Stored rows are already within budget; this bounds assistant and reasoning merges. + rows: page.entries.map((entry) => + Object.assign({ seq: entry.seqEnd }, entry, { + item: limitAgentTimelineItemContent(entry.item), + }), + ), + }; + } + + readRows(scope: TimelineScope): ProjectedTimelineRow[] { + return this.requireState(scope).projection.getRows().map(cloneRow); + } + + lastItem(scope: TimelineScope): AgentTimelineItem | null { + const state = this.requireState(scope); + return state.projection.getLastItem(state.nextSeq - 1); + } + + lastAssistantMessage(scope: TimelineScope): string | null { + return this.requireState(scope).projection.getLastAssistantMessage(); + } + + private describe(state: MemoryTimelineState): TimelineHead { + return { epoch: state.epoch, size: state.projection.size }; + } + + private requireState(scope: TimelineScope): MemoryTimelineState { + const state = this.states.get(scopeKey(scope)); + if (!state) { + throw new UnknownTimelineError(scope); + } + return state; + } +} diff --git a/packages/server/src/server/agent/timeline/testing.ts b/packages/server/src/server/agent/timeline/testing.ts new file mode 100644 index 0000000000..f666bcc10e --- /dev/null +++ b/packages/server/src/server/agent/timeline/testing.ts @@ -0,0 +1,90 @@ +import { setImmediate as nextTurn } from "node:timers/promises"; +import type { AgentTimelineItem } from "../agent-sdk-types.js"; +import type { + AgentTimelineFetchOptions, + AgentTimelineFetchResult, + AgentTimelineRow, +} from "../agent-timeline-store-types.js"; +import type { ProjectedTimelineRow } from "../timeline-projection.js"; +import type { + AgentTimelineBackend, + TimelineBackendAppendResult, + TimelineEntry, + TimelineHead, + TimelineScope, + TimelineSeed, +} from "./internal/backend.js"; +import { MemoryTimelineBackend } from "./internal/memory-backend.js"; + +/** + * Testing entry point of the timeline module. Production code imports `./index.js`. + * + * Defers every call by a turn of the event loop, the shape of an I/O-backed store, and records + * the committed appends and whole-history reads. + */ +export class DeferredTimelineBackend implements AgentTimelineBackend { + private readonly memory = new MemoryTimelineBackend(); + readonly appends: TimelineEntry[][] = []; + readRowsCalls = 0; + failNextAppend = false; + + private async defer(call: () => T): Promise { + await nextTurn(); + return call(); + } + + create(scope: TimelineScope, seed: TimelineSeed): Promise { + return this.defer(() => this.memory.create(scope, seed)); + } + delete(scope: TimelineScope): Promise { + return this.defer(() => this.memory.delete(scope)); + } + head(scope: TimelineScope): Promise { + return this.defer(() => this.memory.head(scope)); + } + append( + scope: TimelineScope, + entries: readonly TimelineEntry[], + ): Promise { + return this.defer(() => { + if (this.failNextAppend) { + this.failNextAppend = false; + throw new Error("disk full"); + } + const result = this.memory.append(scope, entries); + this.appends.push([...entries]); + return result; + }); + } + attachProviderMessageId( + scope: TimelineScope, + clientMessageId: string, + providerMessageId: string, + ): Promise { + return this.defer(() => + this.memory.attachProviderMessageId(scope, clientMessageId, providerMessageId), + ); + } + findSubmittedPrompt( + scope: TimelineScope, + clientMessageId: string, + ): Promise { + return this.defer(() => this.memory.findSubmittedPrompt(scope, clientMessageId)); + } + fetch( + scope: TimelineScope, + options: AgentTimelineFetchOptions, + ): Promise { + return this.defer(() => this.memory.fetch(scope, options)); + } + readRows(scope: TimelineScope): Promise { + this.readRowsCalls += 1; + return this.defer(() => this.memory.readRows(scope)); + } + lastItem(scope: TimelineScope): Promise { + return this.defer(() => this.memory.lastItem(scope)); + } + lastAssistantMessage(scope: TimelineScope): Promise { + return this.defer(() => this.memory.lastAssistantMessage(scope)); + } +} diff --git a/packages/server/src/server/agent/tools/paseo-tools.ts b/packages/server/src/server/agent/tools/paseo-tools.ts index 0ca6c5a3e4..feb73ab34f 100644 --- a/packages/server/src/server/agent/tools/paseo-tools.ts +++ b/packages/server/src/server/agent/tools/paseo-tools.ts @@ -3173,7 +3173,7 @@ function createToolCatalog( agentStorage, logger: childLogger, }); - const timeline = agentManager.getTimeline(agentId); + const timeline = await agentManager.getTimeline(agentId); const snapshot = agentManager.getAgent(agentId); return { diff --git a/packages/server/src/server/daemon-e2e/checkout-ship.e2e.test.ts b/packages/server/src/server/daemon-e2e/checkout-ship.e2e.test.ts index a9becb6473..2424667048 100644 --- a/packages/server/src/server/daemon-e2e/checkout-ship.e2e.test.ts +++ b/packages/server/src/server/daemon-e2e/checkout-ship.e2e.test.ts @@ -183,13 +183,15 @@ describe("daemon checkout ship loop", () => { expect(diffUncommitted.error).toBeNull(); expect(diffUncommitted.files.length).toBeGreaterThan(0); - const timelineBeforeCommit = ctx.daemon.daemon.agentManager.getTimeline(agent.id).length; + const timelineBeforeCommit = (await ctx.daemon.daemon.agentManager.getTimeline(agent.id)) + .length; const commitResult = await ctx.client.checkoutCommit(worktree.worktreePath, { addAll: true, }); expect(commitResult.error).toBeNull(); expect(commitResult.success).toBe(true); - const timelineAfterCommit = ctx.daemon.daemon.agentManager.getTimeline(agent.id).length; + const timelineAfterCommit = (await ctx.daemon.daemon.agentManager.getTimeline(agent.id)) + .length; expect(timelineAfterCommit).toBe(timelineBeforeCommit); const diffAfterCommit = await ctx.client.getCheckoutDiff(worktree.worktreePath, { @@ -203,13 +205,14 @@ describe("daemon checkout ship loop", () => { }); expect(baseDiff.files.length).toBeGreaterThan(0); - const timelineBeforePr = ctx.daemon.daemon.agentManager.getTimeline(agent.id).length; + const timelineBeforePr = (await ctx.daemon.daemon.agentManager.getTimeline(agent.id)) + .length; const prCreate = await ctx.client.checkoutPrCreate(worktree.worktreePath, { baseRef: "main", }); expect(prCreate.error).toBeNull(); expect(prCreate.url).toContain(repoName); - const timelineAfterPr = ctx.daemon.daemon.agentManager.getTimeline(agent.id).length; + const timelineAfterPr = (await ctx.daemon.daemon.agentManager.getTimeline(agent.id)).length; expect(timelineAfterPr).toBe(timelineBeforePr); const prStatus = await ctx.client.checkoutPrStatus(worktree.worktreePath); diff --git a/packages/server/src/server/daemon-e2e/timeline-window.e2e.test.ts b/packages/server/src/server/daemon-e2e/timeline-window.e2e.test.ts index 78134d7865..7648722c55 100644 --- a/packages/server/src/server/daemon-e2e/timeline-window.e2e.test.ts +++ b/packages/server/src/server/daemon-e2e/timeline-window.e2e.test.ts @@ -333,10 +333,12 @@ describe("daemon E2E - timeline window", () => { text: `row ${seq}`, }); } - const epoch = ctx.daemon.daemon.agentManager.fetchTimeline(agent.id, { - direction: "tail", - limit: 1, - }).epoch; + const epoch = ( + await ctx.daemon.daemon.agentManager.fetchTimeline(agent.id, { + direction: "tail", + limit: 1, + }) + ).epoch; const timeline = await ctx.client.fetchAgentTimeline(agent.id, { direction: "after", @@ -374,10 +376,12 @@ describe("daemon E2E - timeline window", () => { text: `message ${seq - 200}`, }); } - const epoch = ctx.daemon.daemon.agentManager.fetchTimeline(agent.id, { - direction: "tail", - limit: 1, - }).epoch; + const epoch = ( + await ctx.daemon.daemon.agentManager.fetchTimeline(agent.id, { + direction: "tail", + limit: 1, + }) + ).epoch; const timeline = await ctx.client.fetchAgentTimeline(agent.id, { direction: "after", @@ -476,10 +480,12 @@ describe("daemon E2E - timeline window", () => { text: `row ${seq}`, }); } - const epoch = ctx.daemon.daemon.agentManager.fetchTimeline(agent.id, { - direction: "tail", - limit: 1, - }).epoch; + const epoch = ( + await ctx.daemon.daemon.agentManager.fetchTimeline(agent.id, { + direction: "tail", + limit: 1, + }) + ).epoch; const timeline = await ctx.client.fetchAgentTimeline(agent.id, { direction: "before", diff --git a/packages/server/src/server/hub/provider-policy.real.e2e.test.ts b/packages/server/src/server/hub/provider-policy.real.e2e.test.ts index d59d1a8b39..77a68911ce 100644 --- a/packages/server/src/server/hub/provider-policy.real.e2e.test.ts +++ b/packages/server/src/server/hub/provider-policy.real.e2e.test.ts @@ -149,7 +149,10 @@ describe.skipIf(!RUN_REAL_HUB_POLICY)("Hub provider policy (real providers)", () provider, scenario: "replying_worker", actions: sink.calls, - agent: redactDiagnostic(hub.ownedAgentDiagnostic(created.agentId), hub.repoRoot()), + agent: redactDiagnostic( + await hub.ownedAgentDiagnostic(created.agentId), + hub.repoRoot(), + ), permissionsRequested: permissionRequests(events), toolOutcomes: toolOutcomes(events), })}`, diff --git a/packages/server/src/server/hub/test-utils/relationship-harness.ts b/packages/server/src/server/hub/test-utils/relationship-harness.ts index 84726d0b02..115745492c 100644 --- a/packages/server/src/server/hub/test-utils/relationship-harness.ts +++ b/packages/server/src/server/hub/test-utils/relationship-harness.ts @@ -1036,15 +1036,15 @@ export class HubRelationshipHarness { ); } - ownedAgentDiagnostic(agentId: string): { + async ownedAgentDiagnostic(agentId: string): Promise<{ lifecycle: string; lastError?: string; assistantText?: string; timelineTypes: string[]; - } { + }> { const agent = this.daemon!.agentManager.getAgent(agentId); if (!agent) throw new Error(`Owned agent ${agentId} does not exist`); - const timeline = this.daemon!.agentManager.getTimeline(agentId); + const timeline = await this.daemon!.agentManager.getTimeline(agentId); const assistantText = timeline .flatMap((item) => (item.type === "assistant_message" ? [item.text] : [])) .join(""); diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 42c676b50b..72c7268612 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -1989,7 +1989,7 @@ export class Session { this.unsubscribeAgentEvents = this.agentManager.subscribe( (event) => { if (event.type === "timeline_replacement") { - this.deliverTimelineReplacement(event.agentId, this.rewindInitiators.get(event.agentId)); + this.deliverTimelineReplacement(event, this.rewindInitiators.get(event.agentId)); return; } @@ -4642,7 +4642,7 @@ export class Session { await unarchiveAgentState(this.agentStorage, this.agentManager, snapshot.id); await this.agentManager.hydrateTimelineFromProvider(snapshot.id); await this.agentUpdates.forwardLiveAgent(snapshot); - const timelineSize = this.agentManager.getTimeline(snapshot.id).length; + const timelineSize = (await this.agentManager.getTimeline(snapshot.id)).length; if (requestId) { const agentPayload = await this.buildAgentPayload(snapshot); this.emit({ @@ -4791,7 +4791,7 @@ export class Session { } await this.agentManager.hydrateTimelineFromProvider(agentId, { broadcast: true }); await this.agentUpdates.forwardLiveAgent(snapshot); - const timelineSize = this.agentManager.getTimeline(agentId).length; + const timelineSize = (await this.agentManager.getTimeline(agentId)).length; if (requestId) { this.emit({ type: "status", @@ -4910,11 +4910,13 @@ export class Session { } } - private deliverTimelineReplacement(agentId: string, initiatingSource?: object): void { + private deliverTimelineReplacement( + timeline: Extract, + initiatingSource?: object, + ): void { + const { agentId, epoch } = timeline; const agent = this.agentManager.getAgent(agentId); if (!agent) return; - const timeline = this.agentManager.fetchTimeline(agentId, { limit: 0 }); - const epoch = timeline.epoch; if (this.clientSources.size === 0 || !this.onMessageToSource) { if (!this.supports(CLIENT_CAPS.timelineReplacementInvalidation)) { @@ -7843,7 +7845,7 @@ export class Session { }); const agentPayload = await this.buildAgentPayload(snapshot); - const fetchedControlTimeline = this.agentManager.fetchTimeline(msg.agentId, { + const fetchedControlTimeline = await this.agentManager.fetchTimeline(msg.agentId, { direction, cursor, limit: msg.limit, @@ -7973,14 +7975,14 @@ export class Session { logger: this.sessionLogger, }); const rows = await this.agentManager.getTimelineRows(msg.agentId); - const { epoch } = this.agentManager.fetchTimeline(msg.agentId, { + const { epoch } = await this.agentManager.fetchTimeline(msg.agentId, { direction: "tail", limit: 1, }); const result = await searchTimeline({ rows, query: msg.query, cursor: msg.cursor }); if ( - this.agentManager.fetchTimeline(msg.agentId, { direction: "tail", limit: 1 }).epoch !== - epoch + (await this.agentManager.fetchTimeline(msg.agentId, { direction: "tail", limit: 1 })) + .epoch !== epoch ) { throw new Error("History changed; search again"); } @@ -8026,7 +8028,7 @@ export class Session { logger: this.sessionLogger, }); const rows = await this.agentManager.getTimelineRows(msg.agentId); - const timeline = this.agentManager.fetchTimeline(msg.agentId, { + const timeline = await this.agentManager.fetchTimeline(msg.agentId, { direction: "tail", limit: 1, }); @@ -8114,7 +8116,7 @@ export class Session { const supportsProjection = source ? this.supportsForSource(CLIENT_CAPS.projectedSubagentTimeline, source) : this.supports(CLIENT_CAPS.projectedSubagentTimeline); - const timeline = this.agentManager.fetchProviderSubagentTimeline( + const timeline = await this.agentManager.fetchProviderSubagentTimeline( msg.parentAgentId, msg.subagentId, { @@ -8205,7 +8207,7 @@ export class Session { logger: this.sessionLogger, }); const agentPayload = await this.buildAgentPayload(snapshot); - const timeline = this.agentManager.fetchTimeline(msg.agentId, { + const timeline = await this.agentManager.fetchTimeline(msg.agentId, { direction: "tail", limit: 0, }); diff --git a/packages/server/src/server/wire-compat.test.ts b/packages/server/src/server/wire-compat.test.ts index 00faf1f441..9328f1f05c 100644 --- a/packages/server/src/server/wire-compat.test.ts +++ b/packages/server/src/server/wire-compat.test.ts @@ -22,7 +22,7 @@ import { OWNER_PERMISSIONS } from "./authorization/index.js"; import { DirectorySyncService } from "./directory-sync/index.js"; import { createProviderSnapshotManagerStub } from "./test-utils/session-stubs.js"; import type { AgentTimelineRow } from "./agent/agent-manager.js"; -import { InMemoryAgentTimelineStore } from "./agent/agent-timeline-store.js"; +import { AgentTimeline } from "./agent/timeline/index.js"; import type { AgentTimelineFetchOptions } from "./agent/agent-timeline-store-types.js"; import { handleCreatePaseoWorktreeRequest } from "./worktree-session.js"; import { createPersistedProjectRecord } from "./workspace-registry.js"; @@ -60,14 +60,18 @@ interface SessionInternals { } class InMemoryAgentManager { - private readonly timeline = new InMemoryAgentTimelineStore(); + private readonly timeline = new AgentTimeline(); constructor(rows: AgentTimelineRow[]) { - this.timeline.initialize("agent-1", { - epoch: "epoch-1", - rows, - nextSeq: (rows.at(-1)?.seq ?? 0) + 1, - }); + // The timeline runs operations in call order, so every fetch observes this seed. + void this.timeline.create( + { agentId: "agent-1" }, + { + epoch: "epoch-1", + rows, + nextSeq: (rows.at(-1)?.seq ?? 0) + 1, + }, + ); } getAgent() { @@ -115,7 +119,7 @@ class InMemoryAgentManager { } fetchTimeline(_agentId: string, options?: AgentTimelineFetchOptions) { - return this.timeline.fetch("agent-1", options); + return this.timeline.fetch({ agentId: "agent-1" }, options); } listAgents() {