diff --git a/docs/timeline-sync.md b/docs/timeline-sync.md index b852e340a..ed22654c2 100644 --- a/docs/timeline-sync.md +++ b/docs/timeline-sync.md @@ -37,6 +37,12 @@ Initialization timeouts guard lack of catch-up progress, not the full multi-page The first load of an agent without a local cursor is different: it fetches a bounded latest tail page. Older history remains user-driven by scrolling upward. +Reaching the history-start threshold loads one older page and preserves the visible content anchor. +Cursor progress does not trigger another page. The user must leave and return to the threshold unless +the anchored page still leaves the viewport at history start, as with short or compacted content; in +that case pagination continues as one loading operation until the page fills the viewport or history +is exhausted. + ## Durable item anchors Provider message IDs are not guaranteed for every displayed item. Paseo-generated system errors are one example. Rendered item indices are not durable either because pagination and projection can merge source rows. @@ -61,6 +67,22 @@ recomposition while the runtime still owns the same directory snapshot and timel Removing the host from the registry is the destructive boundary: it stops the runtime and clears the session and host-scoped setup state together. +The durable replica cache is a display cache, not a synchronization checkpoint. Its timeline record +contains only the focused `agentId` and a truncated item tail. It never persists a cursor, epoch, +older-history availability, authority status, or sync generation because those facts would describe +the complete source dataset rather than the truncated display dataset. + +Restoring that cache produces a painted timeline: the items may render immediately, but the first +daemon timeline request is still `tail`. A successful tail response atomically establishes canonical +items, range, and older-history availability. Live rows received between cache paint and that tail +response stay in the separate live head, do not advance a cursor or trigger gap recovery, and are +reconciled with the authoritative tail and subsequent catch-up. + +Every daemon-derived live item carries its timeline epoch and sequence position. Bootstrap +replacement keeps only positioned rows newer than the page it installs, while unresolved local +submissions remain governed by the submission registry. This prevents a page from duplicating rows +it already covers without making the display replica authoritative. + ## Selective and legacy delivery The app chooses one delivery policy from `server_info.features.selectiveAgentTimeline`: diff --git a/packages/app/e2e/agent-tab-image-stability.spec.ts b/packages/app/e2e/agent-tab-image-stability.spec.ts new file mode 100644 index 000000000..6ad5d0550 --- /dev/null +++ b/packages/app/e2e/agent-tab-image-stability.spec.ts @@ -0,0 +1,85 @@ +import { test as base } from "./fixtures"; +import { + appendSettledTimelineTurns, + createNearTenMegabyteAssistantPng, + createSettledMockAgent, + createSmallAssistantPng, + emitSettledAssistantImage, + expectAssistantImageNotMounted, + expectAssistantImageRendered, + openAssistantImageTimeline, + openExistingImageAgentTabs, + remountAndRecoverAssistantImageFromHistory, + sendFollowUpAndExpectVisibleResponse, + switchAwayAndBackWithoutImageInstability, + userPagesUntilAssistantImageRenders, +} from "./helpers/assistant-images"; +import { seedWorkspace, type SeededWorkspace } from "./helpers/seed-client"; + +const test = base.extend<{ imageWorkspace: SeededWorkspace }>({ + imageWorkspace: async ({ page: _page }, provide) => { + const workspace = await seedWorkspace({ repoPrefix: "agent-tab-image-stability-" }); + try { + await provide(workspace); + } finally { + await workspace.cleanup(); + } + }, +}); + +test("switching between settled agent tabs keeps a real assistant PNG rendered", async ({ + imageWorkspace: workspace, + page, +}) => { + test.setTimeout(120_000); + const image = await createSmallAssistantPng(workspace, { + alt: "Real file image", + fileName: "assistant-preview.png", + }); + const imageAgent = await createSettledMockAgent(workspace, "Image timeline"); + const otherAgent = await createSettledMockAgent(workspace, "Other timeline"); + await emitSettledAssistantImage(workspace.client, imageAgent, image); + + await openExistingImageAgentTabs(page, { imageAgent, otherAgent }); + await expectAssistantImageRendered(page, image); + await switchAwayAndBackWithoutImageInstability(page, { image, imageAgent, otherAgent }); +}); + +test("a real assistant PNG remains reachable through pagination and remount", async ({ + imageWorkspace: workspace, + page, +}) => { + test.setTimeout(120_000); + const image = await createSmallAssistantPng(workspace, { + alt: "Paginated real file image", + fileName: "paginated-assistant-preview.png", + }); + const imageAgent = await createSettledMockAgent(workspace, "Paginated image timeline"); + await emitSettledAssistantImage(workspace.client, imageAgent, image); + await appendSettledTimelineTurns(workspace.client, imageAgent, 40); + + await openAssistantImageTimeline(page, imageAgent); + await expectAssistantImageNotMounted(page, image); + await userPagesUntilAssistantImageRenders(page, image); + await remountAndRecoverAssistantImageFromHistory(page, image); +}); + +test("a near-10 MiB real assistant PNG renders and the app remains responsive", async ({ + imageWorkspace: workspace, + page, +}) => { + test.setTimeout(180_000); + const image = await createNearTenMegabyteAssistantPng(workspace, { + alt: "Large real file image", + fileName: "large-assistant-preview.png", + }); + const imageAgent = await createSettledMockAgent(workspace, "Large image timeline"); + await emitSettledAssistantImage(workspace.client, imageAgent, image); + + await openAssistantImageTimeline(page, imageAgent); + await expectAssistantImageRendered(page, image); + await sendFollowUpAndExpectVisibleResponse(page, { + prompt: "confirm responsiveness: emit 1 coalesced agent stream updates", + response: "stress-update-0", + }); +}); diff --git a/packages/app/e2e/agent-timeline-pagination.spec.ts b/packages/app/e2e/agent-timeline-pagination.spec.ts index 30c0bc379..0f562987f 100644 --- a/packages/app/e2e/agent-timeline-pagination.spec.ts +++ b/packages/app/e2e/agent-timeline-pagination.spec.ts @@ -1,27 +1,169 @@ -import { test } from "./fixtures"; +import { expect, test } from "./fixtures"; import { - expectLoadedTimelineDoesNotScroll, + expectSameOlderHistoryLoadingOperation, + expectTimelineAtHistoryStart, expectTimelinePromptNotMounted, + expectTimelinePromptPositionPreserved, expectTimelinePromptVisible, - holdNextOlderTimelinePage, + holdBootstrapTimelinePage, + holdDaemonHydration, + holdOlderHistoryPages, makeLoadedTimelineFitViewport, openAgentTimeline, + rememberOlderHistoryLoadingOperation, + rememberTimelineViewport, + rememberTimelinePromptPosition, + reloadAgentTimelineFromPersistedReplica, scrollTimelineUntilOlderHistoryIsReachable, + scrollTimelineToNewestLoadedEdge, seedLongMockAgentTimeline, + sendLiveTurnBeforeHydration, + expectTimelineViewportAnchoredAfterPrepend, + userScrollsTimelineToHistoryStart, } from "./helpers/timeline-pagination"; test.describe("Agent timeline pagination", () => { - test("loads older history when the user scrolls to the top of a long agent timeline", async ({ + test("loads one page each time the user returns to history start", async ({ page }) => { + test.setTimeout(120_000); + const agent = await seedLongMockAgentTimeline({ turns: 80 }); + try { + const history = await holdOlderHistoryPages(page, agent); + await openAgentTimeline(page, agent); + await expectTimelinePromptVisible(page, agent.newestPrompt); + await expectTimelinePromptNotMounted(page, agent.oldestPrompt); + + await userScrollsTimelineToHistoryStart(page); + await history.expectRequestedPages(1); + history.releasePage(1); + await history.expectSettledWithRequestedPages(1); + + await userScrollsTimelineToHistoryStart(page); + await history.expectRequestedPages(2); + } finally { + await agent.cleanup(); + } + }); + + test("keeps the visible timeline position anchored while prepending a page", async ({ page }) => { + test.setTimeout(120_000); + const agent = await seedLongMockAgentTimeline({ turns: 80 }); + try { + const history = await holdOlderHistoryPages(page, agent); + await openAgentTimeline(page, agent); + await userScrollsTimelineToHistoryStart(page); + await history.expectRequestedPages(1); + const viewport = await rememberTimelineViewport(page); + + history.releasePage(1); + await expectTimelineViewportAnchoredAfterPrepend(page, viewport); + } finally { + await agent.cleanup(); + } + }); + + test("keeps visible history anchored when live output grows during a prepend", async ({ page, }) => { test.setTimeout(120_000); const agent = await seedLongMockAgentTimeline({ turns: 80 }); try { + const history = await holdOlderHistoryPages(page, agent); await openAgentTimeline(page, agent); - await expectTimelinePromptVisible(page, agent.newestPrompt); - await expectTimelinePromptNotMounted(page, agent.oldestPrompt); + await userScrollsTimelineToHistoryStart(page); + await history.expectRequestedPages(1); + const position = await rememberTimelinePromptPosition(page, agent.initialTailOldestPrompt); - await scrollTimelineUntilOlderHistoryIsReachable(page); + await agent.client.sendAgentMessage( + agent.agentId, + "timeline live during held older page: emit 20 coalesced agent stream updates", + ); + await agent.client.waitForFinish(agent.agentId, 15_000); + history.releasePage(1); + + await expectTimelinePromptPositionPreserved(page, position); + } finally { + await agent.cleanup(); + } + }); + + test("finishes loading an older page while live output continues", async ({ page }) => { + test.setTimeout(120_000); + const agent = await seedLongMockAgentTimeline({ turns: 40 }); + try { + const history = await holdOlderHistoryPages(page, agent); + await openAgentTimeline(page, agent); + await userScrollsTimelineToHistoryStart(page); + await history.expectRequestedPages(1); + + await agent.client.sendAgentMessage(agent.agentId, "keep streaming while history settles"); + await agent.client.waitForAgentUpsert( + agent.agentId, + (snapshot) => snapshot.status === "running", + ); + history.releasePage(1); + + await expect(page.getByTestId("load-older-history-spinner")).toBeHidden({ timeout: 5_000 }); + const running = await agent.client.fetchAgents({ scope: "active" }); + expect(running.entries.find((entry) => entry.agent.id === agent.agentId)?.agent.status).toBe( + "running", + ); + } finally { + await agent.cleanup(); + } + }); + + test("keeps the visible timeline anchored when the final page finishes", async ({ page }) => { + test.setTimeout(120_000); + const agent = await seedLongMockAgentTimeline({ turns: 40 }); + try { + const history = await holdOlderHistoryPages(page, agent); + await openAgentTimeline(page, agent); + await userScrollsTimelineToHistoryStart(page); + await history.expectRequestedPages(1); + const position = await rememberTimelinePromptPosition(page, agent.initialTailOldestPrompt); + + history.releasePage(1); + + await expectTimelinePromptPositionPreserved(page, position); + await history.expectSettledWithRequestedPages(1); + } finally { + await agent.cleanup(); + } + }); + + test("continues one loading operation while older pages still leave history start exposed", async ({ + page, + }) => { + test.setTimeout(120_000); + const agent = await seedLongMockAgentTimeline({ turns: 80 }); + try { + await makeLoadedTimelineFitViewport(page); + const history = await holdOlderHistoryPages(page, agent); + await openAgentTimeline(page, agent); + await history.expectRequestedPages(1); + const loading = await rememberOlderHistoryLoadingOperation(page); + + history.releasePage(1); + await history.expectRequestedPages(2); + await expectTimelineAtHistoryStart(page); + await expectSameOlderHistoryLoadingOperation(page, loading); + } finally { + await agent.cleanup(); + } + }); + + test("keeps complete loaded history reachable after reload", async ({ page }) => { + test.setTimeout(120_000); + const agent = await seedLongMockAgentTimeline({ turns: 80 }); + try { + await openAgentTimeline(page, agent); + await scrollTimelineUntilOlderHistoryIsReachable(page, agent.oldestPrompt); + await expectTimelinePromptVisible(page, agent.oldestPrompt); + + const hydration = await holdDaemonHydration(page); + await reloadAgentTimelineFromPersistedReplica(page, agent); + hydration.release(); + await scrollTimelineUntilOlderHistoryIsReachable(page, agent.oldestPrompt); await expectTimelinePromptVisible(page, agent.oldestPrompt); } finally { @@ -29,18 +171,27 @@ test.describe("Agent timeline pagination", () => { } }); - test("loads older history when the initial page does not fill the viewport", async ({ page }) => { + test("preserves a live row received before replica hydration", async ({ page }) => { test.setTimeout(120_000); - const agent = await seedLongMockAgentTimeline({ turns: 30 }); + const agent = await seedLongMockAgentTimeline({ turns: 80 }); try { - await makeLoadedTimelineFitViewport(page); - const olderPage = await holdNextOlderTimelinePage(page, agent); await openAgentTimeline(page, agent); - await expectTimelinePromptVisible(page, agent.newestPrompt); - await expectLoadedTimelineDoesNotScroll(page); - await olderPage.expectLoading(); - olderPage.release(); + await scrollTimelineUntilOlderHistoryIsReachable(page, agent.oldestPrompt); + + const hydration = await holdBootstrapTimelinePage(page, agent); + await reloadAgentTimelineFromPersistedReplica(page, agent); + await hydration.waitForDelayedResponse(); + const livePrompt = await sendLiveTurnBeforeHydration(agent); + await expectTimelinePromptVisible(page, livePrompt); + + hydration.release(); + await hydration.waitForDelayedCatchUp(); + await expectTimelinePromptVisible(page, livePrompt); + hydration.releaseCatchUp(); + await scrollTimelineUntilOlderHistoryIsReachable(page, agent.oldestPrompt); await expectTimelinePromptVisible(page, agent.oldestPrompt); + await scrollTimelineToNewestLoadedEdge(page); + await expectTimelinePromptVisible(page, livePrompt); } finally { await agent.cleanup(); } diff --git a/packages/app/e2e/helpers/agent-timeline-gate.ts b/packages/app/e2e/helpers/agent-timeline-gate.ts index 511d303db..104f0a159 100644 --- a/packages/app/e2e/helpers/agent-timeline-gate.ts +++ b/packages/app/e2e/helpers/agent-timeline-gate.ts @@ -15,6 +15,21 @@ export interface AgentTimelineResponseGate { waitForDelayedResponse(): Promise; } +export interface OlderTimelinePagesGate { + getRequestCount(): number; + releasePage(pageNumber: number): void; + waitForRequestCount(count: number): Promise; +} + +export interface DaemonHydrationGate { + release(): void; +} + +export interface BootstrapTimelineGate extends AgentTimelineResponseGate { + releaseCatchUp(): void; + waitForDelayedCatchUp(): Promise; +} + function parseWebSocketJson(message: WebSocketMessage): unknown { const rawMessage = typeof message === "string" ? message : message.toString("utf8"); try { @@ -45,6 +60,32 @@ function getPayload(message: Record): Record | : null; } +export async function holdDaemonHydration(page: Page): Promise { + let released = false; + const delayedForwards: Array<() => void> = []; + + await page.routeWebSocket(daemonWsRoutePattern(), (ws) => { + const server = ws.connectToServer(); + ws.onMessage((message) => server.send(message)); + server.onMessage((message) => { + if (released) { + ws.send(message); + return; + } + delayedForwards.push(() => ws.send(message)); + }); + }); + + return { + release() { + released = true; + for (const forward of delayedForwards.splice(0)) { + forward(); + } + }, + }; +} + export async function delayCreatedAgentInitialTailResponse( page: Page, ): Promise { @@ -127,6 +168,142 @@ export async function delayCreatedAgentInitialTailResponse( export async function delayAgentOlderTimelineResponse( page: Page, agentId: string, +): Promise { + return delayAgentTimelineResponse(page, agentId, "before"); +} + +export async function holdAgentOlderTimelinePages( + page: Page, + agentId: string, +): Promise { + let requestCount = 0; + let responseCount = 0; + const releasedPages = new Set(); + const delayedForwards = new Map void>>(); + const requestWaiters = new Map void>>(); + + const resolveRequestWaiters = () => { + for (const [count, resolvers] of requestWaiters) { + if (requestCount < count) continue; + requestWaiters.delete(count); + for (const resolve of resolvers) resolve(); + } + }; + + await page.routeWebSocket(daemonWsRoutePattern(), (ws) => { + const server = ws.connectToServer(); + ws.onMessage((message) => { + const sessionMessage = getSessionMessage(message); + if ( + sessionMessage?.type === "fetch_agent_timeline_request" && + sessionMessage.agentId === agentId && + sessionMessage.direction === "before" + ) { + requestCount += 1; + resolveRequestWaiters(); + } + server.send(message); + }); + server.onMessage((message) => { + const sessionMessage = getSessionMessage(message); + const payload = sessionMessage ? getPayload(sessionMessage) : null; + if ( + sessionMessage?.type === "fetch_agent_timeline_response" && + payload?.agentId === agentId && + payload.direction === "before" + ) { + responseCount += 1; + const pageNumber = responseCount; + if (releasedPages.has(pageNumber)) { + ws.send(message); + return; + } + const forwards = delayedForwards.get(pageNumber) ?? []; + forwards.push(() => ws.send(message)); + delayedForwards.set(pageNumber, forwards); + return; + } + ws.send(message); + }); + }); + + return { + getRequestCount: () => requestCount, + releasePage(pageNumber) { + releasedPages.add(pageNumber); + for (const forward of delayedForwards.get(pageNumber) ?? []) forward(); + delayedForwards.delete(pageNumber); + }, + waitForRequestCount(count) { + if (requestCount >= count) return Promise.resolve(); + return new Promise((resolve) => { + const resolvers = requestWaiters.get(count) ?? []; + resolvers.push(resolve); + requestWaiters.set(count, resolvers); + }); + }, + }; +} + +export async function delayAgentBootstrapTailResponse( + page: Page, + agentId: string, +): Promise { + let tailReleased = false; + let catchUpReleased = false; + const delayedTailForwards: Array<() => void> = []; + const delayedCatchUpForwards: Array<() => void> = []; + let resolveDelayedTail: (() => void) | null = null; + let resolveDelayedCatchUp: (() => void) | null = null; + const delayedTail = new Promise((resolve) => { + resolveDelayedTail = resolve; + }); + const delayedCatchUp = new Promise((resolve) => { + resolveDelayedCatchUp = resolve; + }); + + await page.routeWebSocket(daemonWsRoutePattern(), (ws) => { + const server = ws.connectToServer(); + ws.onMessage((message) => server.send(message)); + server.onMessage((message) => { + const sessionMessage = getSessionMessage(message); + const payload = sessionMessage ? getPayload(sessionMessage) : null; + const isTimelineResponse = + sessionMessage?.type === "fetch_agent_timeline_response" && payload?.agentId === agentId; + if (isTimelineResponse && payload.direction === "tail") { + resolveDelayedTail?.(); + if (tailReleased) ws.send(message); + else delayedTailForwards.push(() => ws.send(message)); + return; + } + if (isTimelineResponse && payload.direction === "after") { + resolveDelayedCatchUp?.(); + if (catchUpReleased) ws.send(message); + else delayedCatchUpForwards.push(() => ws.send(message)); + return; + } + ws.send(message); + }); + }); + + return { + release() { + tailReleased = true; + for (const forward of delayedTailForwards.splice(0)) forward(); + }, + releaseCatchUp() { + catchUpReleased = true; + for (const forward of delayedCatchUpForwards.splice(0)) forward(); + }, + waitForDelayedResponse: () => delayedTail, + waitForDelayedCatchUp: () => delayedCatchUp, + }; +} + +async function delayAgentTimelineResponse( + page: Page, + agentId: string, + direction: "before" | "tail", ): Promise { let releaseRequested = false; let delayedResponseSeen = false; @@ -148,7 +325,7 @@ export async function delayAgentOlderTimelineResponse( !delayedResponseSeen && sessionMessage?.type === "fetch_agent_timeline_response" && payload?.agentId === agentId && - payload.direction === "before" + payload.direction === direction ) { delayedResponseSeen = true; resolveDelayedResponse?.(); diff --git a/packages/app/e2e/helpers/assistant-images.ts b/packages/app/e2e/helpers/assistant-images.ts new file mode 100644 index 000000000..00016f0c0 --- /dev/null +++ b/packages/app/e2e/helpers/assistant-images.ts @@ -0,0 +1,343 @@ +import { writeFile } from "node:fs/promises"; +import path from "node:path"; +import { deflateSync } from "node:zlib"; +import { expect, type Page } from "@playwright/test"; +import type { ArchiveTabAgent } from "./archive-tab"; +import { openWorkspaceWithAgents } from "./archive-tab"; +import { submitMessage } from "./composer"; +import type { SeedDaemonClient, SeededWorkspace } from "./seed-client"; +import { openAgentRoute } from "./mock-agent"; +import { rememberTimelineViewport, userScrollsTimelineToHistoryStart } from "./timeline-pagination"; + +const IMAGE_PREVIEW_ERROR = "Unable to load image preview."; +const SMALL_PNG = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==", + "base64", +); +const TEN_MEBIBYTES = 10 * 1024 * 1024; + +export interface AssistantImageFixture { + alt: string; + height: number; + relativePath: string; + size: number; + width: number; +} + +export async function createSmallAssistantPng( + workspace: SeededWorkspace, + input: { alt: string; fileName: string }, +): Promise { + await writeFile(path.join(workspace.repoPath, input.fileName), SMALL_PNG); + return { + alt: input.alt, + height: 1, + relativePath: input.fileName, + size: SMALL_PNG.byteLength, + width: 1, + }; +} + +export async function createNearTenMegabyteAssistantPng( + workspace: SeededWorkspace, + input: { alt: string; fileName: string }, +): Promise { + const width = 2_048; + const height = 1_280; + const bytes = encodeDeterministicPng(width, height); + if (bytes.byteLength < TEN_MEBIBYTES * 0.95 || bytes.byteLength > TEN_MEBIBYTES * 1.05) { + throw new Error(`Expected a PNG near 10 MiB, encoded ${bytes.byteLength} bytes`); + } + await writeFile(path.join(workspace.repoPath, input.fileName), bytes); + return { + alt: input.alt, + height, + relativePath: input.fileName, + size: bytes.byteLength, + width, + }; +} + +export async function createSettledMockAgent( + workspace: SeededWorkspace, + title: string, +): Promise { + const agent = await workspace.client.createAgent({ + provider: "mock", + model: "ten-second-stream", + modeId: "load-test", + cwd: workspace.repoPath, + workspaceId: workspace.workspaceId, + title, + }); + await workspace.client.waitForAgentUpsert( + agent.id, + (snapshot) => snapshot.status === "idle", + 30_000, + ); + return { + id: agent.id, + title, + cwd: workspace.repoPath, + workspaceId: workspace.workspaceId, + }; +} + +export async function emitSettledAssistantImage( + client: SeedDaemonClient, + agent: ArchiveTabAgent, + image: AssistantImageFixture, +): Promise { + await client.sendAgentMessage( + agent.id, + `Emit settled assistant image Markdown: ![${image.alt}](${image.relativePath})`, + ); + const result = await client.waitForFinish(agent.id, 30_000); + if (result.status !== "idle" || result.final?.lastError) { + throw new Error( + `Assistant image agent did not settle: ${result.final?.lastError ?? result.status}`, + ); + } +} + +export async function appendSettledTimelineTurns( + client: SeedDaemonClient, + agent: ArchiveTabAgent, + count: number, +): Promise { + for (let index = 0; index < count; index += 1) { + await client.sendAgentMessage( + agent.id, + `image-history-turn-${index}: emit 1 coalesced agent stream updates`, + ); + const result = await client.waitForFinish(agent.id, 30_000); + if (result.status !== "idle" || result.final?.lastError) { + throw new Error( + `Assistant image history turn did not settle: ${result.final?.lastError ?? result.status}`, + ); + } + } +} + +export async function expectAssistantImageRendered( + page: Page, + image: AssistantImageFixture, +): Promise { + const rendered = page.getByRole("img", { name: image.alt }).first(); + await expect(rendered).toBeVisible({ timeout: 30_000 }); + await expect + .poll(async () => + rendered.evaluate((element) => { + const imageElement = + element instanceof HTMLImageElement ? element : element.querySelector("img"); + return imageElement?.complete + ? { height: imageElement.naturalHeight, width: imageElement.naturalWidth } + : null; + }), + ) + .toEqual({ height: image.height, width: image.width }); +} + +export async function switchAwayAndBackWithoutImageInstability( + page: Page, + input: { + image: AssistantImageFixture; + imageAgent: ArchiveTabAgent; + otherAgent: ArchiveTabAgent; + }, +): Promise { + await beginVisibleImageStabilityObservation(page, input.image.alt, input.imageAgent.id); + await selectSettledAgentTab(page, input.otherAgent); + await selectSettledAgentTab(page, input.imageAgent); + await expectAssistantImageRendered(page, input.image); + await expectNoVisibleImageInstability(page); +} + +export async function openExistingImageAgentTabs( + page: Page, + input: { imageAgent: ArchiveTabAgent; otherAgent: ArchiveTabAgent }, +): Promise { + await openWorkspaceWithAgents(page, [input.otherAgent, input.imageAgent]); +} + +export async function openAssistantImageTimeline( + page: Page, + agent: ArchiveTabAgent, +): Promise { + await openAgentRoute(page, { workspaceId: agent.workspaceId, agentId: agent.id }); +} + +export async function expectAssistantImageNotMounted( + page: Page, + image: AssistantImageFixture, +): Promise { + await expect(page.getByRole("img", { name: image.alt })).toHaveCount(0); +} + +export async function userPagesUntilAssistantImageRenders( + page: Page, + image: AssistantImageFixture, +): Promise { + const rendered = page.getByRole("img", { name: image.alt }); + for (let attempt = 0; attempt < 10; attempt += 1) { + if ((await rendered.count()) > 0) { + await expectAssistantImageRendered(page, image); + return; + } + const previous = await rememberTimelineViewport(page); + await userScrollsTimelineToHistoryStart(page); + await expect + .poll(async () => (await rememberTimelineViewport(page)).scrollHeight) + .toBeGreaterThan(previous.scrollHeight); + } + await expectAssistantImageRendered(page, image); +} + +export async function remountAndRecoverAssistantImageFromHistory( + page: Page, + image: AssistantImageFixture, +): Promise { + await page.reload(); + await userPagesUntilAssistantImageRenders(page, image); +} + +export async function sendFollowUpAndExpectVisibleResponse( + page: Page, + input: { prompt: string; response: string }, +): Promise { + await submitMessage(page, input.prompt); + await expect(page.getByText(input.prompt, { exact: true })).toBeVisible({ timeout: 30_000 }); + await expect(page.getByText(input.response, { exact: true })).toBeVisible({ timeout: 30_000 }); +} + +async function selectSettledAgentTab(page: Page, agent: ArchiveTabAgent): Promise { + const tab = page.getByRole("button", { name: agent.title, exact: true }); + await tab.click(); + await expect(page).toHaveTitle(agent.title); + await expect(tab).toHaveAttribute("aria-selected", "true"); + await expect(page.locator('[data-testid="agent-chat-scroll"]:visible').first()).toBeVisible({ + timeout: 30_000, + }); +} + +async function beginVisibleImageStabilityObservation( + page: Page, + alt: string, + imageAgentId: string, +): Promise { + await page.evaluate( + ({ accessibleName, errorText, tabTestId }) => { + const record = { + errorSeen: false, + missingSeen: false, + inspect: () => undefined, + observer: null as MutationObserver | null, + }; + const isVisible = (element: Element) => { + const rect = element.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0; + }; + record.inspect = () => { + record.errorSeen ||= Array.from(document.querySelectorAll("*")).some( + (element) => + element.children.length === 0 && + element.textContent?.trim() === errorText && + isVisible(element), + ); + const imageTab = document.querySelector(`[data-testid="${tabTestId}"]`); + if (imageTab?.getAttribute("aria-selected") !== "true") return; + const imageVisible = Array.from(document.querySelectorAll('[role="img"]')).some( + (element) => element.getAttribute("aria-label") === accessibleName && isVisible(element), + ); + record.missingSeen ||= !imageVisible; + }; + record.observer = new MutationObserver(record.inspect); + record.observer.observe(document.body, { + attributes: true, + childList: true, + subtree: true, + characterData: true, + }); + record.inspect(); + ( + window as unknown as { + __paseoAssistantImageObservation?: typeof record; + } + ).__paseoAssistantImageObservation = record; + }, + { + accessibleName: alt, + errorText: IMAGE_PREVIEW_ERROR, + tabTestId: `workspace-tab-agent_${imageAgentId}`, + }, + ); +} + +async function expectNoVisibleImageInstability(page: Page): Promise { + const observation = await page.evaluate(() => { + const owner = window as unknown as { + __paseoAssistantImageObservation?: { + errorSeen: boolean; + missingSeen: boolean; + inspect(): void; + observer: MutationObserver | null; + }; + }; + const record = owner.__paseoAssistantImageObservation; + if (!record) throw new Error("Assistant image stability observation was not started"); + record.inspect(); + record.observer?.disconnect(); + delete owner.__paseoAssistantImageObservation; + return { errorSeen: record.errorSeen, missingSeen: record.missingSeen }; + }); + expect(observation).toEqual({ errorSeen: false, missingSeen: false }); +} + +function encodeDeterministicPng(width: number, height: number): Buffer { + const stride = width * 4 + 1; + const scanlines = Buffer.allocUnsafe(stride * height); + let state = 0x9e3779b9; + for (let row = 0; row < height; row += 1) { + const rowOffset = row * stride; + scanlines[rowOffset] = 0; + for (let offset = 1; offset < stride; offset += 1) { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + scanlines[rowOffset + offset] = state & 0xff; + } + } + + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = 8; + header[9] = 6; + return Buffer.concat([ + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), + pngChunk("IHDR", header), + pngChunk("IDAT", deflateSync(scanlines, { level: 0 })), + pngChunk("IEND", Buffer.alloc(0)), + ]); +} + +function pngChunk(type: string, data: Buffer): Buffer { + const typeBytes = Buffer.from(type, "ascii"); + const chunk = Buffer.allocUnsafe(12 + data.byteLength); + chunk.writeUInt32BE(data.byteLength, 0); + typeBytes.copy(chunk, 4); + data.copy(chunk, 8); + chunk.writeUInt32BE(crc32(Buffer.concat([typeBytes, data])), 8 + data.byteLength); + return chunk; +} + +function crc32(bytes: Buffer): number { + let crc = 0xffffffff; + for (const byte of bytes) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) { + crc = (crc >>> 1) ^ (crc & 1 ? 0xedb88320 : 0); + } + } + return (crc ^ 0xffffffff) >>> 0; +} diff --git a/packages/app/e2e/helpers/timeline-pagination.ts b/packages/app/e2e/helpers/timeline-pagination.ts index bf3023309..cc11dc82e 100644 --- a/packages/app/e2e/helpers/timeline-pagination.ts +++ b/packages/app/e2e/helpers/timeline-pagination.ts @@ -1,20 +1,51 @@ import { expect, type Page } from "@playwright/test"; import { buildAgentRoute, seedMockAgentWorkspace, type MockAgentWorkspace } from "./mock-agent"; import { + delayAgentBootstrapTailResponse, delayAgentOlderTimelineResponse, + holdDaemonHydration, + holdAgentOlderTimelinePages, type AgentTimelineResponseGate, + type BootstrapTimelineGate, + type OlderTimelinePagesGate, } from "./agent-timeline-gate"; +export { holdDaemonHydration }; + interface LongTimelineAgentOptions { turns: number; } interface LongTimelineAgent extends MockAgentWorkspace { + initialTailOldestPrompt: string; oldestPrompt: string; newestPrompt: string; } const PROMPT_PREFIX = "timeline-pagination-turn"; +const LIVE_BEFORE_HYDRATION_PROMPT = "timeline live before authoritative hydration"; +const HISTORY_START_THRESHOLD_PX = 96; + +interface TimelineViewportSnapshot { + scrollHeight: number; + scrollTop: number; +} + +interface TimelinePromptPositionSnapshot { + prompt: string; + top: number; +} + +interface OlderHistoryLoadingOperation { + animationStartTime: number | null; + marker: string; +} + +interface OlderHistoryPages { + expectRequestedPages(count: number): Promise; + expectSettledWithRequestedPages(count: number): Promise; + releasePage(pageNumber: number): void; +} function promptForTurn(index: number): string { return `${PROMPT_PREFIX}-${index}: emit 1 coalesced agent stream updates`; @@ -36,11 +67,41 @@ export async function seedLongMockAgentTimeline( return { ...agent, + initialTailOldestPrompt: promptForTurn(Math.max(0, options.turns - 20)), oldestPrompt: promptForTurn(0), newestPrompt: promptForTurn(options.turns - 1), }; } +export async function rememberTimelinePromptPosition( + page: Page, + prompt: string, +): Promise { + const timeline = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); + const item = timeline.getByText(prompt, { exact: true }); + await expect(item).toBeVisible(); + const box = await item.boundingBox(); + if (!box) { + throw new Error(`Expected a rendered timeline item for ${prompt}`); + } + return { prompt, top: box.y }; +} + +export async function expectTimelinePromptPositionPreserved( + page: Page, + before: TimelinePromptPositionSnapshot, +): Promise { + const timeline = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); + const item = timeline.getByText(before.prompt, { exact: true }); + await expect(item).toBeVisible(); + await expect + .poll(async () => { + const box = await item.boundingBox(); + return box ? Math.abs(box.y - before.top) : Number.POSITIVE_INFINITY; + }) + .toBeLessThanOrEqual(2); +} + export async function openAgentTimeline(page: Page, agent: LongTimelineAgent): Promise { await page.goto(buildAgentRoute(agent.workspaceId, agent.agentId)); await page.waitForURL( @@ -50,15 +111,17 @@ export async function openAgentTimeline(page: Page, agent: LongTimelineAgent): P } export async function expectTimelinePromptVisible(page: Page, prompt: string): Promise { - await expect(page.getByText(prompt, { exact: true })).toBeVisible({ timeout: 30_000 }); + const timeline = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); + await expect(timeline.getByText(prompt, { exact: true })).toBeVisible({ timeout: 30_000 }); } export async function expectTimelinePromptNotMounted(page: Page, prompt: string): Promise { - await expect(page.getByText(prompt, { exact: true })).toHaveCount(0); + const timeline = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); + await expect(timeline.getByText(prompt, { exact: true })).toHaveCount(0); } export async function makeLoadedTimelineFitViewport(page: Page): Promise { - await page.setViewportSize({ width: 1280, height: 8_000 }); + await page.setViewportSize({ width: 1280, height: 20_000 }); } export async function expectLoadedTimelineDoesNotScroll(page: Page): Promise { @@ -75,6 +138,33 @@ export async function expectLoadedTimelineDoesNotScroll(page: Page): Promise { + await expect + .poll(() => + page.evaluate((agentId) => { + const raw = localStorage.getItem("@paseo:replica-cache"); + if (!raw) return false; + const cache = JSON.parse(raw) as { + hosts?: Array<{ + timeline?: { + agentId?: string; + items?: unknown[]; + } | null; + }>; + }; + const timeline = cache.hosts?.find((host) => host.timeline?.agentId === agentId)?.timeline; + return timeline?.items?.length === 50; + }, agent.agentId), + ) + .toBe(true); + + await page.reload(); + await expectTimelinePromptVisible(page, agent.newestPrompt); +} + export async function holdNextOlderTimelinePage( page: Page, agent: LongTimelineAgent, @@ -90,6 +180,106 @@ export async function holdNextOlderTimelinePage( }; } +export async function holdOlderHistoryPages( + page: Page, + agent: LongTimelineAgent, +): Promise { + const gate: OlderTimelinePagesGate = await holdAgentOlderTimelinePages(page, agent.agentId); + return { + async expectRequestedPages(count) { + await gate.waitForRequestCount(count); + expect(gate.getRequestCount()).toBe(count); + }, + async expectSettledWithRequestedPages(count) { + await waitForTimelineGeometryToSettle(page); + expect(gate.getRequestCount()).toBe(count); + }, + releasePage(pageNumber) { + gate.releasePage(pageNumber); + }, + }; +} + +export async function rememberTimelineViewport(page: Page): Promise { + return readTimelineViewport(page); +} + +export async function expectTimelineViewportAnchoredAfterPrepend( + page: Page, + before: TimelineViewportSnapshot, +): Promise { + await expect + .poll(async () => (await readTimelineViewport(page)).scrollHeight) + .toBeGreaterThan(before.scrollHeight); + await waitForTimelineGeometryToSettle(page); + const after = await readTimelineViewport(page); + const contentGrowth = after.scrollHeight - before.scrollHeight; + const scrollAdjustment = after.scrollTop - before.scrollTop; + expect(Math.abs(contentGrowth - scrollAdjustment)).toBeLessThanOrEqual(2); +} + +export async function rememberOlderHistoryLoadingOperation( + page: Page, +): Promise { + const slot = page.getByTestId("load-older-history-spinner"); + await expect(slot).toBeVisible(); + const marker = `older-history-loading-${Date.now()}`; + await slot.evaluate((element, operationMarker) => { + const candidates = [element, ...Array.from(element.querySelectorAll("*"))]; + const animated = candidates.find( + (candidate) => getComputedStyle(candidate).animationName !== "none", + ); + if (!(animated instanceof HTMLElement)) { + throw new Error("Expected the older-history loader to contain an animated element"); + } + animated.dataset.olderHistoryLoadingOperation = operationMarker; + }, marker); + const animated = page.locator(`[data-older-history-loading-operation="${marker}"]`); + await expect + .poll(async () => { + const startTime = await animated.evaluate((element) => element.getAnimations()[0]?.startTime); + return typeof startTime === "number" ? startTime : null; + }) + .not.toBeNull(); + const animationStartTime = await animated.evaluate((element) => { + const startTime = element.getAnimations()[0]?.startTime; + return typeof startTime === "number" ? startTime : null; + }); + return { animationStartTime, marker }; +} + +export async function expectSameOlderHistoryLoadingOperation( + page: Page, + operation: OlderHistoryLoadingOperation, +): Promise { + const animated = page.locator(`[data-older-history-loading-operation="${operation.marker}"]`); + await expect(animated).toBeVisible(); + const animationStartTime = await animated.evaluate((element) => { + const startTime = element.getAnimations()[0]?.startTime; + return typeof startTime === "number" ? startTime : null; + }); + expect(animationStartTime).toBe(operation.animationStartTime); +} + +export async function expectTimelineAtHistoryStart(page: Page): Promise { + await expect + .poll(async () => (await readTimelineViewport(page)).scrollTop) + .toBeLessThanOrEqual(HISTORY_START_THRESHOLD_PX); +} + +export async function holdBootstrapTimelinePage( + page: Page, + agent: LongTimelineAgent, +): Promise { + return delayAgentBootstrapTailResponse(page, agent.agentId); +} + +export async function sendLiveTurnBeforeHydration(agent: LongTimelineAgent): Promise { + await agent.client.sendAgentMessage(agent.agentId, LIVE_BEFORE_HYDRATION_PROMPT); + await agent.client.waitForFinish(agent.agentId, 15_000); + return LIVE_BEFORE_HYDRATION_PROMPT; +} + export async function scrollTimelineToOldestLoadedEdge(page: Page): Promise { const scroll = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); await scroll.hover(); @@ -109,31 +299,95 @@ export async function scrollTimelineToOldestLoadedEdge(page: Page): Promise { +export async function userScrollsTimelineToHistoryStart(page: Page): Promise { const scroll = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); - const previousHeight = await scroll.evaluate((element) => { + await scroll.hover(); + for (let step = 0; step < 60; step += 1) { + if ((await readTimelineViewport(page)).scrollTop <= HISTORY_START_THRESHOLD_PX) { + break; + } + await page.mouse.wheel(0, -1_000); + await page.evaluate( + () => + new Promise((resolve) => { + requestAnimationFrame(() => resolve()); + }), + ); + } + await expect + .poll(async () => (await readTimelineViewport(page)).scrollTop) + .toBeLessThanOrEqual(HISTORY_START_THRESHOLD_PX); +} + +export async function scrollTimelineToNewestLoadedEdge(page: Page): Promise { + const scroll = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); + await scroll.evaluate((element) => { if (!(element instanceof HTMLElement)) { throw new Error("Agent chat scroll element is not an HTMLElement"); } - return element.scrollHeight; + element.scrollTop = element.scrollHeight; + element.dispatchEvent(new Event("scroll", { bubbles: true })); }); +} - await page.evaluate( - () => - new Promise((resolve) => { - requestAnimationFrame(() => requestAnimationFrame(() => resolve())); +export async function scrollTimelineUntilOlderHistoryIsReachable( + page: Page, + oldestPrompt: string, +): Promise { + const scroll = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); + const prompt = scroll.getByText(oldestPrompt, { exact: true }); + for (let attempt = 0; attempt < 10; attempt += 1) { + if ((await prompt.count()) > 0) { + await expect(prompt).toBeVisible(); + return; + } + const previousHeight = await readTimelineViewport(page); + await userScrollsTimelineToHistoryStart(page); + await expect + .poll(async () => (await readTimelineViewport(page)).scrollHeight) + .toBeGreaterThan(previousHeight.scrollHeight); + await waitForTimelineGeometryToSettle(page); + } + await expect(prompt).toBeVisible(); +} + +async function readTimelineViewport(page: Page): Promise { + const scroll = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); + return scroll.evaluate((element) => { + if (!(element instanceof HTMLElement)) { + throw new Error("Agent chat scroll element is not an HTMLElement"); + } + return { scrollHeight: element.scrollHeight, scrollTop: element.scrollTop }; + }); +} + +async function waitForTimelineGeometryToSettle(page: Page): Promise { + const scroll = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); + await scroll.evaluate( + (element) => + new Promise((resolve, reject) => { + if (!(element instanceof HTMLElement)) { + reject(new Error("Agent chat scroll element is not an HTMLElement")); + return; + } + const startedAt = performance.now(); + let stableFrames = 0; + let previous = `${element.scrollTop}:${element.scrollHeight}`; + const sample = () => { + const current = `${element.scrollTop}:${element.scrollHeight}`; + stableFrames = current === previous ? stableFrames + 1 : 0; + previous = current; + if (stableFrames >= 4) { + resolve(); + return; + } + if (performance.now() - startedAt > 5_000) { + reject(new Error("Timeline geometry did not settle")); + return; + } + requestAnimationFrame(sample); + }; + requestAnimationFrame(sample); }), ); - await scrollTimelineToOldestLoadedEdge(page); - await expect - .poll(async () => - scroll.evaluate((element) => { - if (!(element instanceof HTMLElement)) { - throw new Error("Agent chat scroll element is not an HTMLElement"); - } - return element.scrollHeight; - }), - ) - .toBeGreaterThan(previousHeight); - await scrollTimelineToOldestLoadedEdge(page); } diff --git a/packages/app/e2e/rewind-menu.ui-contract.spec.ts b/packages/app/e2e/rewind-menu.ui-contract.spec.ts index 7c12f3f5b..af5d75a0f 100644 --- a/packages/app/e2e/rewind-menu.ui-contract.spec.ts +++ b/packages/app/e2e/rewind-menu.ui-contract.spec.ts @@ -2,6 +2,7 @@ import type { Locator } from "@playwright/test"; import { expect, test, type Page } from "./fixtures"; import { openAgentRoute, seedMockAgentWorkspace } from "./helpers/mock-agent"; import { installDaemonWebSocketGate } from "./helpers/daemon-websocket-gate"; +import { scrollChatAwayFromBottom } from "./helpers/agent-bottom-anchor"; import { composerLocator, expectComposerDraft, @@ -191,6 +192,10 @@ test.describe("Rewind sheet", () => { await expect(page.getByText("Cycle 1", { exact: true })).toBeVisible(); await expectUserMessageCount(page, 2); + await scrollChatAwayFromBottom(page, { + deltaY: -900, + minDistanceFromBottom: 300, + }); await userMessage(page, firstPrompt).hover(); await page.getByTestId("rewind-menu-trigger").first().click(); const rewindSheet = page.getByTestId("rewind-menu-content"); diff --git a/packages/app/src/agent-stream/history-start-pagination.test.ts b/packages/app/src/agent-stream/history-start-pagination.test.ts index 2801f479f..af45b48f7 100644 --- a/packages/app/src/agent-stream/history-start-pagination.test.ts +++ b/packages/app/src/agent-stream/history-start-pagination.test.ts @@ -1,11 +1,15 @@ import { describe, expect, it } from "vitest"; import { + abandonHistoryStartPaginationRequest, createHistoryStartPaginationState, evaluateHistoryStartPagination, + isHistoryStartLoadingOperation, rearmHistoryStartPagination, + settleHistoryStartPagination, + type HistoryStartPaginationInput, } from "./history-start-pagination"; -const visibleHistoryStart = { +const visibleHistoryStart: HistoryStartPaginationInput = { distanceFromHistoryStart: 0, hasOlderHistory: true, isLoadingOlderHistory: false, @@ -14,47 +18,140 @@ const visibleHistoryStart = { }; describe("history start pagination", () => { - it("loads once for each authoritative history cursor", () => { - const initial = createHistoryStartPaginationState(); - const first = evaluateHistoryStartPagination(initial, visibleHistoryStart); - const duplicate = evaluateHistoryStartPagination(first.state, visibleHistoryStart); - const nextPage = evaluateHistoryStartPagination(first.state, { + it("waits for anchored page geometry before authorizing another page", () => { + const first = evaluateHistoryStartPagination( + createHistoryStartPaginationState(), + visibleHistoryStart, + ); + const inFlight = evaluateHistoryStartPagination(first.state, { + ...visibleHistoryStart, + isLoadingOlderHistory: true, + }); + const pageApplied = evaluateHistoryStartPagination(inFlight.state, { + ...visibleHistoryStart, + isLoadingOlderHistory: true, + progressKey: "epoch-1:10", + }); + + expect([first.shouldLoad, inFlight.shouldLoad, pageApplied.shouldLoad]).toEqual([ + true, + false, + false, + ]); + expect(pageApplied.state).toEqual({ status: "settling", loadedProgressKey: "epoch-1:10" }); + }); + + it("loads one page each time anchored geometry leaves and returns to history start", () => { + const first = evaluateHistoryStartPagination( + createHistoryStartPaginationState(), + visibleHistoryStart, + ); + const pageApplied = evaluateHistoryStartPagination(first.state, { + ...visibleHistoryStart, + progressKey: "epoch-1:10", + }); + const settledAway = settleHistoryStartPagination(pageApplied.state, { + ...visibleHistoryStart, + distanceFromHistoryStart: 300, + progressKey: "epoch-1:10", + }); + const returned = evaluateHistoryStartPagination(settledAway.state, { ...visibleHistoryStart, progressKey: "epoch-1:10", }); - expect([first.shouldLoad, duplicate.shouldLoad, nextPage.shouldLoad]).toEqual([ - true, - false, - true, - ]); + expect([ + first.shouldLoad, + pageApplied.shouldLoad, + settledAway.shouldLoad, + returned.shouldLoad, + ]).toEqual([true, false, false, true]); }); - it("allows the same revision again after the user leaves the history edge", () => { + it("continues the same loading operation when anchored geometry remains at history start", () => { const first = evaluateHistoryStartPagination( createHistoryStartPaginationState(), visibleHistoryStart, ); - const away = evaluateHistoryStartPagination(first.state, { + const pageApplied = evaluateHistoryStartPagination(first.state, { ...visibleHistoryStart, - distanceFromHistoryStart: 200, + progressKey: "epoch-1:10", + }); + const continued = settleHistoryStartPagination(pageApplied.state, { + ...visibleHistoryStart, + progressKey: "epoch-1:10", }); - const returned = evaluateHistoryStartPagination(away.state, visibleHistoryStart); - expect([first.shouldLoad, away.shouldLoad, returned.shouldLoad]).toEqual([true, false, true]); + expect(continued.shouldLoad).toBe(true); + expect(isHistoryStartLoadingOperation(first.state)).toBe(true); + expect(isHistoryStartLoadingOperation(pageApplied.state)).toBe(true); + expect(isHistoryStartLoadingOperation(continued.state)).toBe(true); }); - it("re-arms the same cursor when the user makes another upward edge gesture", () => { + it("latches a request that finishes without cursor progress", () => { const first = evaluateHistoryStartPagination( createHistoryStartPaginationState(), visibleHistoryStart, ); + const inFlight = evaluateHistoryStartPagination(first.state, { + ...visibleHistoryStart, + isLoadingOlderHistory: true, + }); + const finished = evaluateHistoryStartPagination(inFlight.state, visibleHistoryStart); + const duplicate = evaluateHistoryStartPagination(finished.state, visibleHistoryStart); + const away = evaluateHistoryStartPagination(duplicate.state, { + ...visibleHistoryStart, + distanceFromHistoryStart: 300, + }); + + expect(finished.state).toEqual({ status: "latched" }); + expect([finished.shouldLoad, duplicate.shouldLoad, away.shouldLoad]).toEqual([ + false, + false, + false, + ]); + expect(away.state).toEqual({ status: "ready" }); + }); + + it("allows another user attempt after a request finishes without progress", () => { + const first = evaluateHistoryStartPagination( + createHistoryStartPaginationState(), + visibleHistoryStart, + ); + const inFlight = evaluateHistoryStartPagination(first.state, { + ...visibleHistoryStart, + isLoadingOlderHistory: true, + }); + const failed = evaluateHistoryStartPagination(inFlight.state, visibleHistoryStart); const retried = evaluateHistoryStartPagination( - rearmHistoryStartPagination(first.state), + rearmHistoryStartPagination(failed.state), visibleHistoryStart, ); - expect([first.shouldLoad, retried.shouldLoad]).toEqual([true, true]); + expect(failed.state).toEqual({ status: "latched" }); + expect(retried.shouldLoad).toBe(true); + }); + + it("latches an attempt that becomes invalid before entering flight", () => { + const requested = evaluateHistoryStartPagination( + createHistoryStartPaginationState(), + visibleHistoryStart, + ); + + expect(abandonHistoryStartPaginationRequest(requested.state, "epoch-1:20")).toEqual({ + status: "latched", + }); + }); + + it("does not mistake repeated edge observations for a finished request", () => { + const first = evaluateHistoryStartPagination( + createHistoryStartPaginationState(), + visibleHistoryStart, + ); + const repeated = evaluateHistoryStartPagination(first.state, visibleHistoryStart); + + expect(repeated.state).toBe(first.state); + expect(repeated.shouldLoad).toBe(false); }); it("waits while history loading is unavailable or already active", () => { diff --git a/packages/app/src/agent-stream/history-start-pagination.ts b/packages/app/src/agent-stream/history-start-pagination.ts index 180ea5cf0..317753b1a 100644 --- a/packages/app/src/agent-stream/history-start-pagination.ts +++ b/packages/app/src/agent-stream/history-start-pagination.ts @@ -1,33 +1,91 @@ export const HISTORY_START_THRESHOLD_PX = 96; -export interface HistoryStartPaginationState { - requestedProgressKey: string | null; +export type HistoryStartPaginationState = + | { status: "ready" } + | { status: "loading"; requestedProgressKey: string; requestObserved: boolean } + | { status: "settling"; loadedProgressKey: string } + | { status: "latched" }; + +export interface HistoryStartPaginationInput { + distanceFromHistoryStart: number; + hasOlderHistory: boolean; + isLoadingOlderHistory: boolean; + isReady: boolean; + progressKey: string | null; +} + +export interface HistoryStartPaginationTransition { + state: HistoryStartPaginationState; + shouldLoad: boolean; } export function createHistoryStartPaginationState(): HistoryStartPaginationState { - return { requestedProgressKey: null }; + return { status: "ready" }; +} + +export function isHistoryStartLoadingOperation(state: HistoryStartPaginationState): boolean { + return state.status === "loading" || state.status === "settling"; } export function rearmHistoryStartPagination( - _state: HistoryStartPaginationState, + state: HistoryStartPaginationState, ): HistoryStartPaginationState { - return createHistoryStartPaginationState(); + return state.status === "latched" ? { status: "ready" } : state; +} + +export function abandonHistoryStartPaginationRequest( + state: HistoryStartPaginationState, + requestedProgressKey: string, +): HistoryStartPaginationState { + if ( + state.status !== "loading" || + state.requestObserved || + state.requestedProgressKey !== requestedProgressKey + ) { + return state; + } + return { status: "latched" }; } export function evaluateHistoryStartPagination( state: HistoryStartPaginationState, - input: { - distanceFromHistoryStart: number; - hasOlderHistory: boolean; - isLoadingOlderHistory: boolean; - isReady: boolean; - progressKey: string | null; - }, -): { state: HistoryStartPaginationState; shouldLoad: boolean } { - if (input.distanceFromHistoryStart > HISTORY_START_THRESHOLD_PX) { - return { state: createHistoryStartPaginationState(), shouldLoad: false }; + input: HistoryStartPaginationInput, +): HistoryStartPaginationTransition { + if (state.status === "loading") { + if (input.progressKey !== null && input.progressKey !== state.requestedProgressKey) { + return { + state: { status: "settling", loadedProgressKey: input.progressKey }, + shouldLoad: false, + }; + } + if (input.isLoadingOlderHistory && !state.requestObserved) { + return { state: { ...state, requestObserved: true }, shouldLoad: false }; + } + if (!input.isLoadingOlderHistory && !input.hasOlderHistory) { + return { state: { status: "latched" }, shouldLoad: false }; + } + if ( + state.requestObserved && + !input.isLoadingOlderHistory && + input.progressKey === state.requestedProgressKey + ) { + return { state: { status: "latched" }, shouldLoad: false }; + } + return { state, shouldLoad: false }; + } + + if (state.status === "settling") { + return { state, shouldLoad: false }; + } + + const isAtHistoryStart = input.distanceFromHistoryStart <= HISTORY_START_THRESHOLD_PX; + if (!isAtHistoryStart) { + return state.status === "ready" + ? { state, shouldLoad: false } + : { state: { status: "ready" }, shouldLoad: false }; } if ( + state.status === "latched" || !input.isReady || !input.hasOlderHistory || input.isLoadingOlderHistory || @@ -35,11 +93,42 @@ export function evaluateHistoryStartPagination( ) { return { state, shouldLoad: false }; } - if (state.requestedProgressKey === input.progressKey) { - return { state, shouldLoad: false }; - } return { - state: { requestedProgressKey: input.progressKey }, + state: { + status: "loading", + requestedProgressKey: input.progressKey, + requestObserved: false, + }, + shouldLoad: true, + }; +} + +export function settleHistoryStartPagination( + state: HistoryStartPaginationState, + input: HistoryStartPaginationInput, +): HistoryStartPaginationTransition { + if (state.status !== "settling") { + return { state, shouldLoad: false }; + } + const isAtHistoryStart = input.distanceFromHistoryStart <= HISTORY_START_THRESHOLD_PX; + if ( + !isAtHistoryStart || + !input.isReady || + !input.hasOlderHistory || + input.isLoadingOlderHistory || + input.progressKey === null + ) { + return { + state: isAtHistoryStart ? { status: "latched" } : { status: "ready" }, + shouldLoad: false, + }; + } + return { + state: { + status: "loading", + requestedProgressKey: input.progressKey, + requestObserved: false, + }, shouldLoad: true, }; } diff --git a/packages/app/src/agent-stream/history-start-settle-scheduler.test.ts b/packages/app/src/agent-stream/history-start-settle-scheduler.test.ts new file mode 100644 index 000000000..2a8894299 --- /dev/null +++ b/packages/app/src/agent-stream/history-start-settle-scheduler.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it, vi } from "vitest"; +import { createHistoryStartSettleScheduler } from "./history-start-settle-scheduler"; + +describe("history start settle scheduler", () => { + it("does not restart its countdown when geometry keeps changing", () => { + const frames = new Map void>(); + let nextFrameId = 1; + const onFrame = vi.fn(); + const onSettle = vi.fn(); + const scheduler = createHistoryStartSettleScheduler({ + settleFrames: 2, + requestFrame(callback) { + const id = nextFrameId; + nextFrameId += 1; + frames.set(id, callback); + return id; + }, + cancelFrame: (id) => frames.delete(id), + isSettling: () => true, + isLoading: () => false, + onFrame, + onSettle, + }); + const runNextFrame = () => { + const next = frames.entries().next().value as [number, () => void] | undefined; + if (!next) { + throw new Error("Expected a scheduled settlement frame"); + } + frames.delete(next[0]); + next[1](); + }; + + scheduler.schedule(); + scheduler.schedule(); + runNextFrame(); + scheduler.schedule(); + runNextFrame(); + scheduler.schedule(); + runNextFrame(); + + expect(onSettle).toHaveBeenCalledTimes(1); + expect(onFrame).toHaveBeenCalledTimes(3); + expect(frames.size).toBe(0); + }); +}); diff --git a/packages/app/src/agent-stream/history-start-settle-scheduler.ts b/packages/app/src/agent-stream/history-start-settle-scheduler.ts new file mode 100644 index 000000000..3049ab8ed --- /dev/null +++ b/packages/app/src/agent-stream/history-start-settle-scheduler.ts @@ -0,0 +1,52 @@ +export interface HistoryStartSettleScheduler { + schedule(): void; + cancel(): void; +} + +export function createHistoryStartSettleScheduler(input: { + settleFrames: number; + requestFrame: (callback: () => void) => number; + cancelFrame: (id: number) => void; + isSettling: () => boolean; + isLoading: () => boolean; + onFrame?: () => void; + onSettle: () => void; +}): HistoryStartSettleScheduler { + let frameId: number | null = null; + let remainingFrames = 0; + + const tick = () => { + input.onFrame?.(); + if (!input.isSettling()) { + frameId = null; + remainingFrames = 0; + return; + } + if (input.isLoading() || remainingFrames > 0) { + if (!input.isLoading()) { + remainingFrames -= 1; + } + frameId = input.requestFrame(tick); + return; + } + frameId = null; + input.onSettle(); + }; + + return { + schedule() { + if (frameId !== null) { + return; + } + remainingFrames = input.settleFrames; + frameId = input.requestFrame(tick); + }, + cancel() { + if (frameId !== null) { + input.cancelFrame(frameId); + } + frameId = null; + remainingFrames = 0; + }, + }; +} diff --git a/packages/app/src/agent-stream/strategy-native.tsx b/packages/app/src/agent-stream/strategy-native.tsx index 2a38fa26d..cb802918c 100644 --- a/packages/app/src/agent-stream/strategy-native.tsx +++ b/packages/app/src/agent-stream/strategy-native.tsx @@ -32,10 +32,19 @@ import { resolveBottomAnchorTransportBehavior, } from "./strategy"; import { + abandonHistoryStartPaginationRequest, createHistoryStartPaginationState, evaluateHistoryStartPagination, + isHistoryStartLoadingOperation, rearmHistoryStartPagination, + settleHistoryStartPagination, + type HistoryStartPaginationInput, + type HistoryStartPaginationTransition, } from "./history-start-pagination"; +import { + createHistoryStartSettleScheduler, + type HistoryStartSettleScheduler, +} from "./history-start-settle-scheduler"; const DEFAULT_MAINTAIN_VISIBLE_CONTENT_POSITION = Object.freeze({ minIndexForVisible: 0, @@ -53,6 +62,8 @@ const historyStartSlotStyle: ViewStyle = { paddingTop: 4, paddingBottom: 8, }; +const HISTORY_START_SETTLE_FRAMES = 2; + interface HistoryRowDisplayVariants { regular?: StreamItem; compact?: StreamItem; @@ -116,7 +127,11 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat const [isNativeViewportSettling, setIsNativeViewportSettling] = useState(false); const nativeViewportSettlingFrameIdRef = useRef(null); const historyStartReadyRef = useRef(false); - const historyStartPaginationStateRef = useRef(createHistoryStartPaginationState()); + const [historyStartPaginationState, setHistoryStartPaginationState] = useState( + createHistoryStartPaginationState, + ); + const historyStartPaginationStateRef = useRef(historyStartPaginationState); + const historyStartSettleSchedulerRef = useRef(null); const historyItems = useMemo(() => { if (segments.historyVirtualized.length === 0) { @@ -145,22 +160,74 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat ), [displayStateHistoryRows, historyRowRevision?.contentById], ); - const evaluateHistoryStart = useStableEvent(() => { + const getHistoryStartPaginationInput = useStableEvent((): HistoryStartPaginationInput => { const metrics = streamViewportMetricsRef.current; const hasMeasuredViewport = metrics.viewportMeasuredForKey === metrics.containerKey && metrics.contentMeasuredForKey === metrics.containerKey; - const result = evaluateHistoryStartPagination(historyStartPaginationStateRef.current, { + return { distanceFromHistoryStart: metrics.contentHeight - metrics.viewportHeight - metrics.offsetY, hasOlderHistory, isLoadingOlderHistory, isReady: historyStartReadyRef.current && hasMeasuredViewport, progressKey: olderHistoryProgressKey, - }); - historyStartPaginationStateRef.current = result.state; - if (result.shouldLoad) { - onNearHistoryStart(); + }; + }); + const applyHistoryStartPaginationTransition = useStableEvent( + (transition: HistoryStartPaginationTransition) => { + const previousState = historyStartPaginationStateRef.current; + historyStartPaginationStateRef.current = transition.state; + if (transition.state !== previousState) { + setHistoryStartPaginationState(transition.state); + } + if (transition.shouldLoad) { + const requestedProgressKey = olderHistoryProgressKey; + if (requestedProgressKey === null) { + return; + } + void (async () => { + const started = await onNearHistoryStart(); + if (started === true) { + return; + } + applyHistoryStartPaginationTransition({ + state: abandonHistoryStartPaginationRequest( + historyStartPaginationStateRef.current, + requestedProgressKey, + ), + shouldLoad: false, + }); + })(); + } + }, + ); + const evaluateHistoryStart = useStableEvent(() => { + const transition = evaluateHistoryStartPagination( + historyStartPaginationStateRef.current, + getHistoryStartPaginationInput(), + ); + applyHistoryStartPaginationTransition(transition); + }); + const scheduleHistoryStartSettle = useStableEvent(() => { + let scheduler = historyStartSettleSchedulerRef.current; + if (!scheduler) { + scheduler = createHistoryStartSettleScheduler({ + settleFrames: HISTORY_START_SETTLE_FRAMES, + requestFrame: requestAnimationFrame, + cancelFrame: cancelAnimationFrame, + isSettling: () => historyStartPaginationStateRef.current.status === "settling", + isLoading: () => getHistoryStartPaginationInput().isLoadingOlderHistory, + onSettle: () => { + const transition = settleHistoryStartPagination( + historyStartPaginationStateRef.current, + getHistoryStartPaginationInput(), + ); + applyHistoryStartPaginationTransition(transition); + }, + }); + historyStartSettleSchedulerRef.current = scheduler; } + scheduler.schedule(); }); const clearNativeViewportSettling = useCallback(() => { @@ -261,7 +328,9 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat clearNativeViewportSettling(); setIsNativeViewportSettling(false); historyStartReadyRef.current = false; - historyStartPaginationStateRef.current = createHistoryStartPaginationState(); + const initialHistoryStartState = createHistoryStartPaginationState(); + historyStartPaginationStateRef.current = initialHistoryStartState; + setHistoryStartPaginationState(initialHistoryStartState); const frame = requestAnimationFrame(() => { historyStartReadyRef.current = true; evaluateHistoryStart(); @@ -269,6 +338,8 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat return () => { cancelAnimationFrame(frame); clearPendingUserScrollEnd(); + historyStartSettleSchedulerRef.current?.cancel(); + historyStartSettleSchedulerRef.current = null; }; }, [agentId, clearNativeViewportSettling, clearPendingUserScrollEnd, evaluateHistoryStart]); @@ -369,16 +440,16 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat }); const handleScrollBeginDrag = useStableEvent((event: NativeSyntheticEvent) => { - if (!isLoadingOlderHistory) { - historyStartPaginationStateRef.current = rearmHistoryStartPagination( - historyStartPaginationStateRef.current, - ); - } clearPendingUserScrollEnd(); isUserScrollActiveRef.current = true; scrollKeyboardDismiss.onScrollBeginDrag(event); bottomAnchorController.beginUserScroll(); - evaluateHistoryStart(); + const rearmed = rearmHistoryStartPagination(historyStartPaginationStateRef.current); + if (rearmed !== historyStartPaginationStateRef.current) { + historyStartPaginationStateRef.current = rearmed; + setHistoryStartPaginationState(rearmed); + evaluateHistoryStart(); + } }); // Defer drag end so momentum can take ownership, but capture the terminal @@ -454,11 +525,23 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat contentHeight: nextContentHeight, }); evaluateHistoryStart(); + if (historyStartPaginationStateRef.current.status === "settling") { + scheduleHistoryStartSettle(); + } }); useEffect(() => { evaluateHistoryStart(); - }, [evaluateHistoryStart, hasOlderHistory, isLoadingOlderHistory, olderHistoryProgressKey]); + if (historyStartPaginationStateRef.current.status === "settling") { + scheduleHistoryStartSettle(); + } + }, [ + evaluateHistoryStart, + hasOlderHistory, + isLoadingOlderHistory, + olderHistoryProgressKey, + scheduleHistoryStartSettle, + ]); const renderItem = useStableEvent( ({ item, index }: ListRenderItemInfo): ReactElement | null => { @@ -499,20 +582,21 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat ]); const historyFooterContent = useMemo(() => { - if (!hasOlderHistory && !isLoadingOlderHistory) { + const isLoadingOperation = isHistoryStartLoadingOperation(historyStartPaginationState); + if (!hasOlderHistory && !isLoadingOperation) { return null; } return ( - {isLoadingOlderHistory ? ( + {isLoadingOperation ? ( ) : null} ); - }, [hasOlderHistory, isLoadingOlderHistory]); + }, [hasOlderHistory, historyStartPaginationState]); // RN's FlatList strictMode keeps its internal renderItem wrapper stable when // data or the live header changes, preserving the row identities above. diff --git a/packages/app/src/agent-stream/strategy-web.test.tsx b/packages/app/src/agent-stream/strategy-web.test.tsx index c125293cf..c9035fc68 100644 --- a/packages/app/src/agent-stream/strategy-web.test.tsx +++ b/packages/app/src/agent-stream/strategy-web.test.tsx @@ -130,7 +130,7 @@ describe("createWebStreamStrategy", () => { routeBottomAnchorRequest: null, isAuthoritativeHistoryReady: true, onNearBottomChange: vi.fn(), - onNearHistoryStart: vi.fn(), + onNearHistoryStart: vi.fn().mockReturnValue(true), isLoadingOlderHistory: false, hasOlderHistory: false, olderHistoryProgressKey: null, @@ -174,7 +174,7 @@ describe("createWebStreamStrategy", () => { routeBottomAnchorRequest: null, isAuthoritativeHistoryReady: true, onNearBottomChange: vi.fn(), - onNearHistoryStart: vi.fn(), + onNearHistoryStart: vi.fn().mockReturnValue(true), isLoadingOlderHistory: false, hasOlderHistory: false, olderHistoryProgressKey: null, @@ -230,7 +230,7 @@ describe("createWebStreamStrategy", () => { routeBottomAnchorRequest: null, isAuthoritativeHistoryReady: true, onNearBottomChange: vi.fn(), - onNearHistoryStart: vi.fn(), + onNearHistoryStart: vi.fn().mockReturnValue(true), isLoadingOlderHistory: false, hasOlderHistory: false, olderHistoryProgressKey: null, @@ -285,76 +285,6 @@ describe("createWebStreamStrategy", () => { expect(scrollTo).not.toHaveBeenCalled(); }); - it("fires near-history-start when the user scrolls near the top", async () => { - const strategy = createWebStreamStrategy({ isMobileBreakpoint: true }); - const viewportRef = React.createRef(); - const onNearHistoryStart = vi.fn(); - container = document.createElement("div"); - document.body.appendChild(container); - root = createRoot(container); - - act(() => { - root?.render( - <> - {strategy.render({ - agentId: "agent", - segments: { - historyVirtualized: [], - historyMounted: [userMessage(1), userMessage(2)], - liveHead: [], - }, - boundary: { - hasVirtualizedHistory: false, - hasMountedHistory: true, - hasLiveHead: false, - }, - renderers: createRenderers(vi.fn()), - listEmptyComponent: null, - viewportRef, - routeBottomAnchorRequest: null, - isAuthoritativeHistoryReady: true, - onNearBottomChange: vi.fn(), - onNearHistoryStart, - isLoadingOlderHistory: false, - hasOlderHistory: true, - olderHistoryProgressKey: "epoch-1:20", - scrollEnabled: true, - listStyle: null, - baseListContentContainerStyle: null, - forwardListContentContainerStyle: null, - })} - , - ); - }); - - const scrollContainer = container.querySelector('[data-testid="agent-chat-scroll"]'); - if (!(scrollContainer instanceof HTMLElement)) { - throw new Error("Expected agent chat scroll container"); - } - Object.defineProperty(scrollContainer, "clientHeight", { configurable: true, value: 400 }); - Object.defineProperty(scrollContainer, "scrollHeight", { configurable: true, value: 1200 }); - Object.defineProperty(scrollContainer, "scrollTop", { configurable: true, value: 64 }); - - await act(async () => { - await new Promise((resolve) => requestAnimationFrame(resolve)); - }); - - expect(onNearHistoryStart).not.toHaveBeenCalled(); - - act(() => { - scrollContainer.dispatchEvent(new WheelEvent("wheel", { deltaY: -1 })); - scrollContainer?.dispatchEvent(new Event("scroll")); - }); - - expect(onNearHistoryStart).toHaveBeenCalledTimes(1); - - act(() => { - scrollContainer.dispatchEvent(new WheelEvent("wheel", { deltaY: -1 })); - }); - - expect(onNearHistoryStart).toHaveBeenCalledTimes(2); - }); - it("waits for bottom anchoring before evaluating a delayed initial tail", async () => { HTMLElement.prototype.scrollTo = vi.fn(function ( this: HTMLElement, @@ -366,7 +296,7 @@ describe("createWebStreamStrategy", () => { }); const strategy = createWebStreamStrategy({ isMobileBreakpoint: true }); const viewportRef = React.createRef(); - const onNearHistoryStart = vi.fn(); + const onNearHistoryStart = vi.fn().mockReturnValue(true); const renderInput = { agentId: "agent", boundary: { @@ -473,7 +403,7 @@ describe("createWebStreamStrategy", () => { viewportRef, routeBottomAnchorRequest, onNearBottomChange: vi.fn(), - onNearHistoryStart: vi.fn(), + onNearHistoryStart: vi.fn().mockReturnValue(true), isLoadingOlderHistory: false, hasOlderHistory: false, olderHistoryProgressKey: null, @@ -581,7 +511,7 @@ describe("createWebStreamStrategy", () => { viewportRef, routeBottomAnchorRequest, onNearBottomChange: vi.fn(), - onNearHistoryStart: vi.fn(), + onNearHistoryStart: vi.fn().mockReturnValue(true), isLoadingOlderHistory: false, hasOlderHistory: false, olderHistoryProgressKey: null, @@ -678,7 +608,7 @@ describe("createWebStreamStrategy", () => { viewportRef, routeBottomAnchorRequest, onNearBottomChange: vi.fn(), - onNearHistoryStart: vi.fn(), + onNearHistoryStart: vi.fn().mockReturnValue(true), isLoadingOlderHistory: false, hasOlderHistory: false, olderHistoryProgressKey: null, @@ -777,7 +707,7 @@ describe("createWebStreamStrategy", () => { viewportRef, routeBottomAnchorRequest: null, onNearBottomChange: vi.fn(), - onNearHistoryStart: vi.fn(), + onNearHistoryStart: vi.fn().mockReturnValue(true), isLoadingOlderHistory: false, hasOlderHistory: false, olderHistoryProgressKey: null, diff --git a/packages/app/src/agent-stream/strategy-web.tsx b/packages/app/src/agent-stream/strategy-web.tsx index 7df586cef..7f31539e1 100644 --- a/packages/app/src/agent-stream/strategy-web.tsx +++ b/packages/app/src/agent-stream/strategy-web.tsx @@ -17,15 +17,30 @@ import { estimateStreamItemHeight } from "./web-virtualization"; import type { StreamRenderInput, StreamStrategy, StreamViewportHandle } from "./strategy"; import { createStreamStrategy } from "./strategy"; import { + abandonHistoryStartPaginationRequest, createHistoryStartPaginationState, evaluateHistoryStartPagination, + isHistoryStartLoadingOperation, rearmHistoryStartPagination, + settleHistoryStartPagination, + type HistoryStartPaginationInput, + type HistoryStartPaginationTransition, } from "./history-start-pagination"; +import { + createHistoryStartSettleScheduler, + type HistoryStartSettleScheduler, +} from "./history-start-settle-scheduler"; interface CreateWebStreamStrategyInput { isMobileBreakpoint: boolean; } +interface HistoryStartPrependAnchor { + progressKey: string; + rowId: string; + viewportOffset: number; +} + type ScrollBehaviorLike = "auto" | "smooth"; const WEB_BOTTOM_SETTLE_TIMEOUT_MS = 200; @@ -33,12 +48,22 @@ const USER_SCROLL_DELTA_EPSILON = 1; const BOTTOM_OVERSCROLL_TOLERANCE_PX = 2; const AUTO_SCROLL_BOTTOM_THRESHOLD_PX = 64; const AUTO_SCROLL_RESUME_THRESHOLD_PX = 1; +const HISTORY_START_SETTLE_FRAMES = 2; const ThemedLoadingSpinner = withUnistyles(LoadingSpinner); const foregroundMutedColorMapping = (theme: Theme) => ({ color: theme.colors.foregroundMuted, }); +function findHistoryRowElement(contentNode: HTMLElement, rowId: string): HTMLElement | null { + for (const element of contentNode.querySelectorAll("[data-history-row-id]")) { + if (element.dataset.historyRowId === rowId) { + return element; + } + } + return null; +} + const historyStartSlotStyle: CSSProperties = { display: "flex", alignItems: "center", @@ -146,7 +171,14 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool const pendingAutoScrollTimeoutRef = useRef(null); const pendingVirtualRowMeasureFramesRef = useRef(new Map()); const historyStartReadyRef = useRef(false); - const historyStartPaginationStateRef = useRef(createHistoryStartPaginationState()); + const [historyStartPaginationState, setHistoryStartPaginationState] = useState( + createHistoryStartPaginationState, + ); + const [isHistoryStartSlotReserved, setIsHistoryStartSlotReserved] = useState(hasOlderHistory); + const historyStartPaginationStateRef = useRef(historyStartPaginationState); + const historyStartPrependAnchorRef = useRef(null); + const historyStartPrependAnchorActiveRef = useRef(false); + const historyStartSettleSchedulerRef = useRef(null); const shouldUseVirtualizer = segments.historyVirtualized.length > 0; const { renderHistoryVirtualizedRow, @@ -176,6 +208,9 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool }); useEffect(() => { rowVirtualizer.shouldAdjustScrollPositionOnItemSizeChange = (_item, _delta, instance) => { + if (historyStartPrependAnchorActiveRef.current) { + return false; + } const viewportHeight = instance.scrollRect?.height ?? 0; const scrollOffset = instance.scrollOffset ?? 0; const remainingDistance = instance.getTotalSize() - (scrollOffset + viewportHeight); @@ -187,25 +222,162 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool }, [rowVirtualizer]); const virtualRows = rowVirtualizer.getVirtualItems(); const virtualTotalSize = rowVirtualizer.getTotalSize(); - const evaluateHistoryStart = useStableEvent(() => { + const getHistoryStartPaginationInput = useStableEvent((): HistoryStartPaginationInput | null => { const scrollContainer = scrollContainerRef.current; if (!scrollContainer) { - return; + return null; } const bottomAnchorSettled = !followOutputRef.current || isScrollContainerNearBottom(scrollContainer); - const result = evaluateHistoryStartPagination(historyStartPaginationStateRef.current, { + return { distanceFromHistoryStart: scrollContainer.scrollTop, hasOlderHistory, isLoadingOlderHistory, isReady: historyStartReadyRef.current && bottomAnchorSettled, progressKey: olderHistoryProgressKey, - }); - historyStartPaginationStateRef.current = result.state; - if (result.shouldLoad) { - onNearHistoryStart(); - } + }; }); + const applyHistoryStartPaginationTransition = useStableEvent( + (transition: HistoryStartPaginationTransition) => { + const previousState = historyStartPaginationStateRef.current; + historyStartPaginationStateRef.current = transition.state; + if (transition.state !== previousState) { + setHistoryStartPaginationState(transition.state); + } + if (!isHistoryStartLoadingOperation(transition.state)) { + historyStartPrependAnchorRef.current = null; + historyStartPrependAnchorActiveRef.current = false; + } + if (!transition.shouldLoad || olderHistoryProgressKey === null) { + return; + } + const scrollContainer = scrollContainerRef.current; + const contentNode = contentRef.current; + const anchorRow = segments.historyMounted.at(-1) ?? segments.historyVirtualized.at(-1); + const anchorElement = + contentNode && anchorRow ? findHistoryRowElement(contentNode, anchorRow.id) : null; + if (scrollContainer && anchorRow && anchorElement) { + historyStartPrependAnchorRef.current = { + progressKey: olderHistoryProgressKey, + rowId: anchorRow.id, + viewportOffset: + anchorElement.getBoundingClientRect().top - scrollContainer.getBoundingClientRect().top, + }; + } else { + historyStartPrependAnchorRef.current = null; + } + historyStartPrependAnchorActiveRef.current = false; + const requestedProgressKey = olderHistoryProgressKey; + void (async () => { + const started = await onNearHistoryStart(); + if (started === true) { + return; + } + applyHistoryStartPaginationTransition({ + state: abandonHistoryStartPaginationRequest( + historyStartPaginationStateRef.current, + requestedProgressKey, + ), + shouldLoad: false, + }); + })(); + }, + ); + const evaluateHistoryStart = useStableEvent(() => { + const input = getHistoryStartPaginationInput(); + if (!input) { + return; + } + const transition = evaluateHistoryStartPagination( + historyStartPaginationStateRef.current, + input, + ); + applyHistoryStartPaginationTransition(transition); + }); + const rearmHistoryStartFromUserIntent = useStableEvent(() => { + const rearmed = rearmHistoryStartPagination(historyStartPaginationStateRef.current); + if (rearmed === historyStartPaginationStateRef.current) { + return; + } + historyStartPaginationStateRef.current = rearmed; + setHistoryStartPaginationState(rearmed); + evaluateHistoryStart(); + }); + const applyHistoryStartPrependAnchor = useStableEvent(() => { + const scrollContainer = scrollContainerRef.current; + const contentNode = contentRef.current; + const anchor = historyStartPrependAnchorRef.current; + if ( + !scrollContainer || + !contentNode || + !anchor || + !historyStartPrependAnchorActiveRef.current + ) { + return; + } + const anchorElement = findHistoryRowElement(contentNode, anchor.rowId); + if (!anchorElement) { + return; + } + const viewportOffset = + anchorElement.getBoundingClientRect().top - scrollContainer.getBoundingClientRect().top; + scrollContainer.scrollTop += viewportOffset - anchor.viewportOffset; + lastKnownScrollTopRef.current = scrollContainer.scrollTop; + }); + const scheduleHistoryStartPrependSettle = useStableEvent(() => { + let scheduler = historyStartSettleSchedulerRef.current; + if (!scheduler) { + scheduler = createHistoryStartSettleScheduler({ + settleFrames: HISTORY_START_SETTLE_FRAMES, + requestFrame: (callback) => window.requestAnimationFrame(callback), + cancelFrame: (frame) => window.cancelAnimationFrame(frame), + isSettling: () => historyStartPaginationStateRef.current.status === "settling", + isLoading: () => { + const input = getHistoryStartPaginationInput(); + return ( + !input || + input.isLoadingOlderHistory || + pendingVirtualRowMeasureFramesRef.current.size > 0 + ); + }, + onFrame: applyHistoryStartPrependAnchor, + onSettle: () => { + const input = getHistoryStartPaginationInput(); + if (!input) { + return; + } + historyStartPrependAnchorActiveRef.current = false; + const transition = settleHistoryStartPagination( + historyStartPaginationStateRef.current, + input, + ); + historyStartPrependAnchorRef.current = null; + applyHistoryStartPaginationTransition(transition); + }, + }); + historyStartSettleSchedulerRef.current = scheduler; + } + scheduler.schedule(); + }); + + useLayoutEffect(() => { + const anchor = historyStartPrependAnchorRef.current; + if (!anchor || anchor.progressKey === olderHistoryProgressKey) { + return; + } + historyStartPrependAnchorActiveRef.current = true; + evaluateHistoryStart(); + applyHistoryStartPrependAnchor(); + scheduleHistoryStartPrependSettle(); + }, [ + applyHistoryStartPrependAnchor, + evaluateHistoryStart, + olderHistoryProgressKey, + scheduleHistoryStartPrependSettle, + segments.historyMounted, + segments.historyVirtualized, + virtualTotalSize, + ]); const measureVirtualizedRowElement = useCallback( (node: HTMLDivElement | null) => { @@ -335,7 +507,11 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool }, [cancelPendingStickToBottom, evaluateHistoryStart, updateScrollMetrics]); useEffect(() => { - historyStartPaginationStateRef.current = createHistoryStartPaginationState(); + const initialHistoryStartState = createHistoryStartPaginationState(); + historyStartPaginationStateRef.current = initialHistoryStartState; + setHistoryStartPaginationState(initialHistoryStartState); + historyStartPrependAnchorRef.current = null; + historyStartPrependAnchorActiveRef.current = false; const frame = window.requestAnimationFrame(() => { historyStartReadyRef.current = true; evaluateHistoryStart(); @@ -343,6 +519,8 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool return () => { window.cancelAnimationFrame(frame); historyStartReadyRef.current = false; + historyStartSettleSchedulerRef.current?.cancel(); + historyStartSettleSchedulerRef.current = null; }; }, [evaluateHistoryStart, props.agentId]); @@ -401,11 +579,15 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool useEffect(() => { updateScrollMetrics(); evaluateHistoryStart(); + if (historyStartPaginationStateRef.current.status === "settling") { + scheduleHistoryStartPrependSettle(); + } }, [ evaluateHistoryStart, hasOlderHistory, isLoadingOlderHistory, olderHistoryProgressKey, + scheduleHistoryStartPrependSettle, segments.historyMounted.length, segments.historyVirtualized.length, segments.liveHead.length, @@ -423,6 +605,12 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool updateScrollMetrics(); evaluateHistoryStart(); const observer = new ResizeObserver(() => { + if (historyStartPrependAnchorActiveRef.current) { + applyHistoryStartPrependAnchor(); + } + if (historyStartPaginationStateRef.current.status === "settling") { + scheduleHistoryStartPrependSettle(); + } updateScrollMetrics(); evaluateHistoryStart(); if (!followOutputRef.current) { @@ -437,7 +625,13 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool return () => { observer.disconnect(); }; - }, [evaluateHistoryStart, scheduleStickToBottom, updateScrollMetrics]); + }, [ + applyHistoryStartPrependAnchor, + evaluateHistoryStart, + scheduleHistoryStartPrependSettle, + scheduleStickToBottom, + updateScrollMetrics, + ]); useEffect(() => { const scrollContainer = scrollContainerRef.current; @@ -447,14 +641,9 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool const handleWheel = (event: WheelEvent) => { if (event.deltaY < 0) { - if (!isLoadingOlderHistory) { - historyStartPaginationStateRef.current = rearmHistoryStartPagination( - historyStartPaginationStateRef.current, - ); - } pendingUserScrollUpIntentRef.current = true; cancelPendingStickToBottom(); - evaluateHistoryStart(); + rearmHistoryStartFromUserIntent(); } }; const handlePointerDown = () => { @@ -477,14 +666,9 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool } const previousTouchY = lastTouchClientYRef.current; if (previousTouchY !== null && touch.clientY > previousTouchY + 1) { - if (!isLoadingOlderHistory) { - historyStartPaginationStateRef.current = rearmHistoryStartPagination( - historyStartPaginationStateRef.current, - ); - } pendingUserScrollUpIntentRef.current = true; cancelPendingStickToBottom(); - evaluateHistoryStart(); + rearmHistoryStartFromUserIntent(); } lastTouchClientYRef.current = touch.clientY; }; @@ -513,7 +697,7 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool scrollContainer.removeEventListener("touchend", handleTouchEnd); scrollContainer.removeEventListener("touchcancel", handleTouchEnd); }; - }, [cancelPendingStickToBottom, evaluateHistoryStart, handleDomScroll, isLoadingOlderHistory]); + }, [cancelPendingStickToBottom, handleDomScroll, rearmHistoryStartFromUserIntent]); useEffect(() => { const handle: StreamViewportHandle = { @@ -580,9 +764,9 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool ); const mountedHistoryRows = useMemo(() => { return segments.historyMounted.map((item, index) => ( - +
{renderHistoryMountedRow(item, index, segments.historyMounted)} - +
)); }, [renderHistoryMountedRow, segments.historyMounted]); const liveHeadRows = useMemo(() => { @@ -594,21 +778,27 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool const liveAuxiliary = useMemo(() => { return renderLiveAuxiliary(); }, [renderLiveAuxiliary]); + useEffect(() => { + if (hasOlderHistory || isHistoryStartLoadingOperation(historyStartPaginationState)) { + setIsHistoryStartSlotReserved(true); + } + }, [hasOlderHistory, historyStartPaginationState]); const historyStartSlot = useMemo(() => { - if (!hasOlderHistory && !isLoadingOlderHistory) { + const isLoadingOperation = isHistoryStartLoadingOperation(historyStartPaginationState); + if (!isHistoryStartSlotReserved && !hasOlderHistory && !isLoadingOperation) { return null; } return (
- {isLoadingOlderHistory ? ( + {isLoadingOperation ? ( ) : null}
); - }, [hasOlderHistory, isLoadingOlderHistory]); + }, [hasOlderHistory, historyStartPaginationState, isHistoryStartSlotReserved]); const shouldRenderEmpty = !boundary.hasMountedHistory && !boundary.hasVirtualizedHistory && @@ -635,6 +825,7 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool
diff --git a/packages/app/src/agent-stream/strategy.ts b/packages/app/src/agent-stream/strategy.ts index 9a467c578..15836f623 100644 --- a/packages/app/src/agent-stream/strategy.ts +++ b/packages/app/src/agent-stream/strategy.ts @@ -69,7 +69,7 @@ export interface StreamRenderInput { routeBottomAnchorRequest: BottomAnchorRouteRequest | null; isAuthoritativeHistoryReady: boolean; onNearBottomChange: (value: boolean) => void; - onNearHistoryStart: () => void; + onNearHistoryStart: () => boolean | Promise; isLoadingOlderHistory: boolean; hasOlderHistory: boolean; olderHistoryProgressKey: string | null; diff --git a/packages/app/src/agent-stream/view.tsx b/packages/app/src/agent-stream/view.tsx index 0bc079e78..63308e56b 100644 --- a/packages/app/src/agent-stream/view.tsx +++ b/packages/app/src/agent-stream/view.tsx @@ -77,6 +77,7 @@ import { type BottomAnchorLocalRequest, type BottomAnchorRouteRequest, } from "./bottom-anchor-controller"; +import { createAssistantImageOccurrenceKey } from "@/assistant-image/acquisition-cache"; import { AssistantFileLinkResolverProvider, normalizeInlinePathTarget, @@ -250,7 +251,7 @@ export interface AgentStreamViewProps { hasOlder: boolean; isLoadingOlder: boolean; progressKey: string | null; - onLoadOlder: () => void; + onLoadOlder: () => boolean | Promise; }; } @@ -694,6 +695,7 @@ const AgentStreamViewComponent = forwardRef ); }, - [client, handleInlinePathPress, resolvedServerId, toast, workspaceRoot], + [agentId, client, handleInlinePathPress, resolvedServerId, toast, workspaceRoot], ); const renderThoughtItem = useCallback( diff --git a/packages/app/src/assistant-image/acquisition-cache.test.ts b/packages/app/src/assistant-image/acquisition-cache.test.ts new file mode 100644 index 000000000..ce1961b63 --- /dev/null +++ b/packages/app/src/assistant-image/acquisition-cache.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, it } from "vitest"; +import { + collectRetainedAttachmentIds, + retainAttachmentForGarbageCollection, +} from "@/attachments/gc-retention"; +import { + createAssistantImageAcquisitionCache, + createAssistantImageFileAcquisitionKey, + createAssistantImageFilePreviewAttachmentId, + createAssistantImageOccurrenceKey, +} from "./acquisition-cache"; + +describe("assistant image acquisition cache", () => { + it("evicts a rejected acquisition so the next request can retry", async () => { + const cache = createAssistantImageAcquisitionCache({ capacity: 2 }); + let attempts = 0; + + await expect( + cache.acquire("image", async () => { + attempts += 1; + throw new Error("first attempt failed"); + }), + ).rejects.toThrow("first attempt failed"); + const recovered = await cache.acquire("image", async () => { + attempts += 1; + return "recovered"; + }); + + expect({ attempts, recovered, size: cache.size() }).toEqual({ + attempts: 2, + recovered: "recovered", + size: 1, + }); + }); + + it("bounds successful acquisitions and evicts the least recently used entry", async () => { + const cache = createAssistantImageAcquisitionCache({ capacity: 2 }); + const located: string[] = []; + const locate = async (key: string) => { + located.push(key); + return key; + }; + + await cache.acquire("a", async () => await locate("a")); + await cache.acquire("b", async () => await locate("b")); + await cache.acquire("a", async () => await locate("a-again")); + await cache.acquire("c", async () => await locate("c")); + await cache.acquire("b", async () => await locate("b-again")); + + expect({ located, size: cache.size() }).toEqual({ + located: ["a", "b", "c", "b-again"], + size: 2, + }); + }); + + it("reuses an acquired image when the current locator is unavailable", async () => { + const cache = createAssistantImageAcquisitionCache({ capacity: 2 }); + let unavailableCalls = 0; + + await cache.acquire("message:image", async () => "persisted attachment"); + const cached = await cache.acquire("message:image", async () => { + unavailableCalls += 1; + throw new Error("daemon disconnected"); + }); + + expect({ cached, unavailableCalls }).toEqual({ + cached: "persisted attachment", + unavailableCalls: 0, + }); + expect(cache.peek("message:image")).toBe("persisted attachment"); + }); + + it("scopes file acquisitions to the rendered message occurrence", () => { + const first = createAssistantImageFileAcquisitionKey({ + serverId: "server", + occurrenceKey: "message-1:image-1", + cwd: "/workspace", + path: "screenshot.png", + }); + const remount = createAssistantImageFileAcquisitionKey({ + serverId: "server", + occurrenceKey: "message-1:image-1", + cwd: "/workspace", + path: "screenshot.png", + }); + const laterMessage = createAssistantImageFileAcquisitionKey({ + serverId: "server", + occurrenceKey: "message-2:image-1", + cwd: "/workspace", + path: "screenshot.png", + }); + + expect(remount).toBe(first); + expect(laterMessage).not.toBe(first); + }); + + it("scopes persisted file previews to the rendered message occurrence", () => { + const first = createAssistantImageFilePreviewAttachmentId({ + serverId: "server-1", + occurrenceKey: "message-1:image-1", + mimeType: "image/png", + path: "/workspace/screenshot.png", + size: 512, + modifiedAt: "2026-07-27T12:00:00.000Z", + contentLength: 512, + }); + const second = createAssistantImageFilePreviewAttachmentId({ + serverId: "server-1", + occurrenceKey: "message-2:image-1", + mimeType: "image/png", + path: "/workspace/screenshot.png", + size: 512, + modifiedAt: "2026-07-27T12:00:00.000Z", + contentLength: 512, + }); + + expect(second).not.toBe(first); + }); + + it("scopes message occurrences to their agent", () => { + const first = createAssistantImageOccurrenceKey({ agentId: "agent-1", itemId: "message-1" }); + const second = createAssistantImageOccurrenceKey({ agentId: "agent-2", itemId: "message-1" }); + + expect(second).not.toBe(first); + }); + + it("retains successful values until their cache entry is evicted", async () => { + const retained: string[] = []; + const released: string[] = []; + const cache = createAssistantImageAcquisitionCache({ + capacity: 1, + onRetain(value) { + retained.push(value); + return () => released.push(value); + }, + }); + + await cache.acquire("first", async () => "attachment-1"); + expect({ retained, released }).toEqual({ retained: ["attachment-1"], released: [] }); + + await cache.acquire("second", async () => "attachment-2"); + expect({ retained, released }).toEqual({ + retained: ["attachment-1", "attachment-2"], + released: ["attachment-1"], + }); + }); + + it("does not evict a value while an active consumer retains it", async () => { + const released: string[] = []; + const cache = createAssistantImageAcquisitionCache({ + capacity: 1, + onRetain: (value) => () => released.push(value), + }); + + const first = cache.acquireRetained("first", async () => "attachment-1"); + await first.promise; + const second = cache.acquireRetained("second", async () => "attachment-2"); + await second.promise; + + expect({ released, size: cache.size() }).toEqual({ released: [], size: 2 }); + + first.release(); + expect({ released, size: cache.size() }).toEqual({ + released: ["attachment-1"], + size: 1, + }); + second.release(); + }); + + it("protects every actively consumed attachment from garbage collection past capacity", async () => { + const cache = createAssistantImageAcquisitionCache<{ id: string }>({ + capacity: 1, + onRetain: (attachment) => retainAttachmentForGarbageCollection(attachment.id), + }); + + const first = cache.acquireRetained("first", async () => ({ id: "mounted-image-1" })); + await first.promise; + const second = cache.acquireRetained("second", async () => ({ id: "mounted-image-2" })); + await second.promise; + + expect(collectRetainedAttachmentIds()).toEqual(new Set(["mounted-image-1", "mounted-image-2"])); + + first.release(); + expect(collectRetainedAttachmentIds()).toEqual(new Set(["mounted-image-2"])); + second.release(); + await expect( + cache.acquire("cleanup", async () => { + throw new Error("cleanup"); + }), + ).rejects.toThrow("cleanup"); + expect(collectRetainedAttachmentIds()).toEqual(new Set()); + }); +}); diff --git a/packages/app/src/assistant-image/acquisition-cache.ts b/packages/app/src/assistant-image/acquisition-cache.ts new file mode 100644 index 000000000..f76f594b8 --- /dev/null +++ b/packages/app/src/assistant-image/acquisition-cache.ts @@ -0,0 +1,159 @@ +import { createPreviewAttachmentId } from "@/attachments/utils"; + +export interface AssistantImageAcquisitionCache { + acquire(key: string, locate: () => Promise): Promise; + acquireRetained( + key: string, + locate: () => Promise, + ): { promise: Promise; value?: T; release: () => void }; + peek(key: string): T | undefined; + size(): number; +} + +export function createAssistantImageOccurrenceKey(input: { + agentId: string; + itemId: string; +}): string { + return `${input.agentId}:${input.itemId}`; +} + +export function createAssistantImageFilePreviewAttachmentId(input: { + serverId?: string; + occurrenceKey: string; + mimeType: string; + path: string; + size: number; + modifiedAt?: string | null; + contentLength: number; +}): string { + return createPreviewAttachmentId({ + mimeType: input.mimeType, + path: input.path, + size: input.size, + modifiedAt: input.modifiedAt, + contentLength: input.contentLength, + contentKey: `${input.serverId ?? "unknown-server"}:${input.occurrenceKey}`, + }); +} + +export function createAssistantImageFileAcquisitionKey(input: { + serverId?: string; + occurrenceKey: string; + cwd: string; + path: string; +}): string { + return `file:${input.serverId ?? "unknown-server"}:${input.occurrenceKey}:${input.cwd}:${input.path}`; +} + +export function createAssistantImageAcquisitionCache(input: { + capacity: number; + onRetain?: (value: T) => () => void; +}): AssistantImageAcquisitionCache { + if (!Number.isInteger(input.capacity) || input.capacity < 1) { + throw new Error("Assistant image acquisition cache capacity must be a positive integer."); + } + interface CacheEntry { + pending: Promise; + resolved: boolean; + value?: T; + release: (() => void) | null; + activeConsumers: number; + } + const entries = new Map(); + + const evict = (key: string, entry: CacheEntry) => { + if (entries.get(key) === entry) { + entries.delete(key); + } + entry.release?.(); + entry.release = null; + }; + + const enforceCapacity = () => { + while (entries.size > input.capacity) { + let evicted = false; + for (const [key, entry] of entries) { + if (entry.activeConsumers > 0) { + continue; + } + evict(key, entry); + evicted = true; + break; + } + if (!evicted) { + return; + } + } + }; + + const acquireEntry = (key: string, locate: () => Promise, retain: boolean): CacheEntry => { + const cached = entries.get(key); + if (cached) { + entries.delete(key); + entries.set(key, cached); + if (retain) { + cached.activeConsumers += 1; + } + return cached; + } + const pending = locate(); + const entry: CacheEntry = { + pending, + resolved: false, + release: null, + activeConsumers: retain ? 1 : 0, + }; + entries.set(key, entry); + enforceCapacity(); + void (async () => { + try { + const value = await pending; + const release = input.onRetain?.(value) ?? null; + if (entries.get(key) === entry) { + entry.value = value; + entry.resolved = true; + entry.release = release; + } else { + release?.(); + } + } catch { + evict(key, entry); + } + })(); + return entry; + }; + + return { + acquire(key, locate) { + return acquireEntry(key, locate, false).pending; + }, + acquireRetained(key, locate) { + const entry = acquireEntry(key, locate, true); + let released = false; + return { + promise: entry.pending, + ...(entry.resolved ? { value: entry.value } : {}), + release() { + if (released) { + return; + } + released = true; + entry.activeConsumers = Math.max(0, entry.activeConsumers - 1); + enforceCapacity(); + }, + }; + }, + peek(key) { + const entry = entries.get(key); + if (!entry?.resolved) { + return undefined; + } + entries.delete(key); + entries.set(key, entry); + return entry.value; + }, + size() { + return entries.size; + }, + }; +} diff --git a/packages/app/src/assistant-image/file-acquisition.test.ts b/packages/app/src/assistant-image/file-acquisition.test.ts new file mode 100644 index 000000000..552609e75 --- /dev/null +++ b/packages/app/src/assistant-image/file-acquisition.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import type { AttachmentMetadata } from "@/attachments/types"; +import { + createAssistantImageFileAcquisition, + type AssistantImageFileAcquisitionPort, +} from "./file-acquisition"; + +class MemoryFileAcquisitionPort implements AssistantImageFileAcquisitionPort { + readonly reads: Array<{ cwd: string; path: string }> = []; + + async readFile(cwd: string, path: string) { + this.reads.push({ cwd, path }); + return { + kind: "image" as const, + path, + mime: "image/png", + size: 4, + modifiedAt: "1", + bytes: new Uint8Array([1, 2, 3, 4]), + }; + } + + async persist(input: { id: string; mimeType: string; fileName: string | null }) { + return { + id: input.id, + mimeType: input.mimeType, + storageType: "web-indexeddb" as const, + storageKey: input.id, + fileName: input.fileName, + byteSize: 4, + createdAt: 1, + } satisfies AttachmentMetadata; + } +} + +describe("assistant image file acquisition", () => { + it("recreates the same acquisition with a live port after reconnect", async () => { + const common = { + resolution: { kind: "file_rpc" as const, cwd: "/workspace", path: "reconnect.png" }, + serverId: "server", + occurrenceKey: "agent:message:reconnect-image", + unavailableMessage: "Image unavailable", + }; + const disconnected = createAssistantImageFileAcquisition({ ...common, port: null }); + const connectedPort = new MemoryFileAcquisitionPort(); + const connected = createAssistantImageFileAcquisition({ ...common, port: connectedPort }); + + expect(disconnected?.key).toBe(connected?.key); + await expect(disconnected?.locate()).rejects.toThrow("Image unavailable"); + await expect(connected?.locate()).resolves.toMatchObject({ mimeType: "image/png" }); + expect(connectedPort.reads).toEqual([{ cwd: "/workspace", path: "reconnect.png" }]); + }); +}); diff --git a/packages/app/src/assistant-image/file-acquisition.ts b/packages/app/src/assistant-image/file-acquisition.ts new file mode 100644 index 000000000..f7a80e853 --- /dev/null +++ b/packages/app/src/assistant-image/file-acquisition.ts @@ -0,0 +1,67 @@ +import type { FileReadResult } from "@getpaseo/client/internal/daemon-client"; +import type { AttachmentMetadata } from "@/attachments/types"; +import { getFileNameFromPath } from "@/attachments/utils"; +import type { AssistantImageSourceResolution } from "@/utils/assistant-image-source"; +import { + createAssistantImageFileAcquisitionKey, + createAssistantImageFilePreviewAttachmentId, +} from "./acquisition-cache"; + +export interface AssistantImageFileAcquisitionPort { + readFile(cwd: string, path: string): Promise; + persist(input: { + id: string; + bytes: Uint8Array; + mimeType: string; + fileName: string | null; + }): Promise; +} + +export interface AssistantImageAcquisition { + key: string; + locate: () => Promise; +} + +export function createAssistantImageFileAcquisition(input: { + port: AssistantImageFileAcquisitionPort | null; + resolution: AssistantImageSourceResolution | null; + serverId?: string; + occurrenceKey: string; + unavailableMessage: string; +}): AssistantImageAcquisition | null { + if (input.resolution?.kind !== "file_rpc") { + return null; + } + const { port, resolution } = input; + return { + key: createAssistantImageFileAcquisitionKey({ + serverId: input.serverId, + occurrenceKey: input.occurrenceKey, + cwd: resolution.cwd, + path: resolution.path, + }), + locate: async () => { + if (!port) { + throw new Error(input.unavailableMessage); + } + const file = await port.readFile(resolution.cwd, resolution.path); + if (file.kind !== "image") { + throw new Error(input.unavailableMessage); + } + return await port.persist({ + id: createAssistantImageFilePreviewAttachmentId({ + serverId: input.serverId, + occurrenceKey: input.occurrenceKey, + mimeType: file.mime, + path: file.path || resolution.path, + size: file.size, + modifiedAt: file.modifiedAt, + contentLength: file.bytes.byteLength, + }), + bytes: file.bytes, + mimeType: file.mime, + fileName: getFileNameFromPath(file.path || resolution.path), + }); + }, + }; +} diff --git a/packages/app/src/assistant-image/lifecycle.test.ts b/packages/app/src/assistant-image/lifecycle.test.ts new file mode 100644 index 000000000..ed7729dd3 --- /dev/null +++ b/packages/app/src/assistant-image/lifecycle.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "vitest"; +import { createAssistantImageLifecycle, transitionAssistantImageLifecycle } from "./lifecycle"; + +describe("assistant image lifecycle", () => { + it("keeps preview URL recreation in the public loading state", () => { + const loading = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), { + type: "preview_created", + uri: "blob:first", + aspectRatio: null, + }); + const loaded = transitionAssistantImageLifecycle(loading, { + type: "image_loaded", + uri: "blob:first", + aspectRatio: 1.5, + }); + + const recreating = transitionAssistantImageLifecycle(loaded, { + type: "preview_released", + }); + + expect(recreating).toEqual({ status: "loading", uri: null, aspectRatio: null }); + }); + + it("publishes loaded only after the recreated URL loads", () => { + const recreating = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), { + type: "preview_created", + uri: "blob:recreated", + aspectRatio: 1.5, + }); + + expect(recreating).toEqual({ + status: "loading", + uri: "blob:recreated", + aspectRatio: 1.5, + }); + + const recreated = transitionAssistantImageLifecycle(recreating, { + type: "image_loaded", + uri: "blob:recreated", + aspectRatio: 0.75, + }); + + expect(recreated).toEqual({ + status: "loaded", + uri: "blob:recreated", + aspectRatio: 0.75, + }); + }); + + it("does not reset a loaded image when the same preview is reported again", () => { + const loading = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), { + type: "preview_created", + uri: "blob:current", + aspectRatio: null, + }); + const loaded = transitionAssistantImageLifecycle(loading, { + type: "image_loaded", + uri: "blob:current", + aspectRatio: 1.5, + }); + + const repeated = transitionAssistantImageLifecycle(loaded, { + type: "preview_created", + uri: "blob:current", + aspectRatio: 1.5, + }); + + expect(repeated).toBe(loaded); + }); + + it("publishes failed for a terminal image failure on the current URI", () => { + const loading = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), { + type: "preview_created", + uri: "blob:current", + aspectRatio: null, + }); + const failed = transitionAssistantImageLifecycle(loading, { + type: "failed", + uri: "blob:current", + message: "Unable to load image preview.", + }); + + expect(failed).toEqual({ + status: "failed", + message: "Unable to load image preview.", + }); + }); + + it("ignores a stale load callback from a replaced URI", () => { + const current = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), { + type: "preview_created", + uri: "blob:current", + aspectRatio: 1.25, + }); + + const afterStaleLoad = transitionAssistantImageLifecycle(current, { + type: "image_loaded", + uri: "blob:released", + aspectRatio: 2, + }); + + expect(afterStaleLoad).toEqual(current); + }); + + it("ignores a stale error callback from a replaced URI", () => { + const current = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), { + type: "preview_created", + uri: "blob:current", + aspectRatio: null, + }); + + const afterStaleError = transitionAssistantImageLifecycle(current, { + type: "failed", + uri: "blob:released", + message: "Image unavailable", + }); + + expect(afterStaleError).toEqual(current); + }); +}); diff --git a/packages/app/src/assistant-image/lifecycle.ts b/packages/app/src/assistant-image/lifecycle.ts new file mode 100644 index 000000000..bf0db18c9 --- /dev/null +++ b/packages/app/src/assistant-image/lifecycle.ts @@ -0,0 +1,46 @@ +export type AssistantImageLifecycle = + | { status: "loading"; uri: string | null; aspectRatio: number | null } + | { status: "loaded"; uri: string; aspectRatio: number } + | { status: "failed"; message: string }; + +export type AssistantImageLifecycleEvent = + | { type: "preview_created"; uri: string; aspectRatio: number | null } + | { type: "preview_released" } + | { type: "image_loaded"; uri: string; aspectRatio: number } + | { type: "failed"; uri: string; message: string }; + +export function createAssistantImageLifecycle(): AssistantImageLifecycle { + return { status: "loading", uri: null, aspectRatio: null }; +} + +export function transitionAssistantImageLifecycle( + state: AssistantImageLifecycle, + event: AssistantImageLifecycleEvent, +): AssistantImageLifecycle { + if (event.type === "image_loaded") { + if (state.status !== "loading" || state.uri !== event.uri) { + return state; + } + return { + status: "loaded", + uri: event.uri, + aspectRatio: event.aspectRatio, + }; + } + if (event.type === "failed") { + if (state.status === "failed" || state.uri !== event.uri) { + return state; + } + return { status: "failed", message: event.message }; + } + if (event.type === "preview_created") { + if (state.status === "loaded" && state.uri === event.uri) { + return state; + } + return { status: "loading", uri: event.uri, aspectRatio: event.aspectRatio }; + } + if (event.type === "preview_released") { + return { status: "loading", uri: null, aspectRatio: null }; + } + return state; +} diff --git a/packages/app/src/assistant-image/load-dimensions.test.ts b/packages/app/src/assistant-image/load-dimensions.test.ts new file mode 100644 index 000000000..a60f9b1b6 --- /dev/null +++ b/packages/app/src/assistant-image/load-dimensions.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { + type AssistantImageRenderedDimensionsReader, + resolveAssistantImageLoadDimensions, +} from "./load-dimensions"; + +class MemoryRenderedDimensionsReader implements AssistantImageRenderedDimensionsReader { + constructor(private readonly dimensions: { width: number; height: number } | null) {} + + read(): { width: number; height: number } | null { + return this.dimensions; + } +} + +describe("assistant image load dimensions", () => { + it("prefers native source dimensions", () => { + const dimensions = resolveAssistantImageLoadDimensions({ + source: { width: 640, height: 320 }, + target: { naturalWidth: 800, naturalHeight: 400 }, + renderedImage: null, + renderedDimensions: new MemoryRenderedDimensionsReader({ width: 900, height: 600 }), + }); + + expect(dimensions).toEqual({ width: 640, height: 320 }); + }); + + it("uses browser event dimensions when native dimensions are absent", () => { + const dimensions = resolveAssistantImageLoadDimensions({ + source: { width: 0, height: 0 }, + target: { naturalWidth: 800, naturalHeight: 400 }, + renderedImage: null, + renderedDimensions: new MemoryRenderedDimensionsReader({ width: 900, height: 600 }), + }); + + expect(dimensions).toEqual({ width: 800, height: 400 }); + }); + + it("falls back to the rendered image adapter", () => { + const renderedImage = { id: "rendered-image" }; + const dimensions = resolveAssistantImageLoadDimensions({ + target: null, + renderedImage, + renderedDimensions: new MemoryRenderedDimensionsReader({ width: 900, height: 600 }), + }); + + expect(dimensions).toEqual({ width: 900, height: 600 }); + }); +}); diff --git a/packages/app/src/assistant-image/load-dimensions.ts b/packages/app/src/assistant-image/load-dimensions.ts new file mode 100644 index 000000000..069814ced --- /dev/null +++ b/packages/app/src/assistant-image/load-dimensions.ts @@ -0,0 +1,45 @@ +export interface AssistantImageDimensions { + width: number; + height: number; +} + +interface AssistantImageDimensionCandidate { + width?: unknown; + height?: unknown; +} + +export interface AssistantImageRenderedDimensionsReader { + read(renderedImage: unknown): AssistantImageDimensions | null; +} + +function readPositiveDimensions( + candidate: AssistantImageDimensionCandidate | null | undefined, +): AssistantImageDimensions | null { + if ( + typeof candidate?.width !== "number" || + typeof candidate.height !== "number" || + candidate.width <= 0 || + candidate.height <= 0 + ) { + return null; + } + return { width: candidate.width, height: candidate.height }; +} + +export function resolveAssistantImageLoadDimensions(input: { + source?: AssistantImageDimensionCandidate | null; + target?: { naturalWidth?: unknown; naturalHeight?: unknown } | null; + renderedImage: unknown; + renderedDimensions: AssistantImageRenderedDimensionsReader; +}): AssistantImageDimensions | null { + const source = readPositiveDimensions(input.source); + if (source) { + return source; + } + + const target = readPositiveDimensions({ + width: input.target?.naturalWidth, + height: input.target?.naturalHeight, + }); + return target ?? input.renderedDimensions.read(input.renderedImage); +} diff --git a/packages/app/src/assistant-image/retry.test.ts b/packages/app/src/assistant-image/retry.test.ts new file mode 100644 index 000000000..9777663be --- /dev/null +++ b/packages/app/src/assistant-image/retry.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { runAssistantImageOperationWithRetry } from "./retry"; + +describe("assistant image retry", () => { + it("recovers from transient failures while the image remains mounted", async () => { + const waits: number[] = []; + let attempts = 0; + + const result = await runAssistantImageOperationWithRetry({ + operation: async () => { + attempts += 1; + if (attempts < 3) { + throw new Error("transient"); + } + return "loaded"; + }, + delaysMs: [10, 20, 30], + wait: async (delayMs) => { + waits.push(delayMs); + }, + }); + + expect({ attempts, waits, result }).toEqual({ + attempts: 3, + waits: [10, 20], + result: "loaded", + }); + }); + + it("stops retrying after the image is released", async () => { + let stopped = false; + let attempts = 0; + + await expect( + runAssistantImageOperationWithRetry({ + operation: async () => { + attempts += 1; + throw new Error("transient"); + }, + delaysMs: [10, 20], + shouldStop: () => stopped, + wait: async () => { + stopped = true; + }, + }), + ).rejects.toThrow("transient"); + expect(attempts).toBe(1); + }); +}); diff --git a/packages/app/src/assistant-image/retry.ts b/packages/app/src/assistant-image/retry.ts new file mode 100644 index 000000000..4a2cf5c58 --- /dev/null +++ b/packages/app/src/assistant-image/retry.ts @@ -0,0 +1,31 @@ +export const ASSISTANT_IMAGE_RETRY_DELAYS_MS = [100, 400, 1_200] as const; + +export async function runAssistantImageOperationWithRetry(input: { + operation: () => Promise; + delaysMs?: readonly number[]; + shouldStop?: () => boolean; + wait?: (delayMs: number) => Promise; +}): Promise { + const delays = input.delaysMs ?? ASSISTANT_IMAGE_RETRY_DELAYS_MS; + const shouldStop = input.shouldStop ?? (() => false); + const wait = + input.wait ?? + (async (delayMs: number) => { + await new Promise((resolve) => setTimeout(resolve, delayMs)); + }); + + for (let attempt = 0; ; attempt += 1) { + try { + return await input.operation(); + } catch (error) { + const delay = delays[attempt]; + if (delay === undefined || shouldStop()) { + throw error; + } + await wait(delay); + if (shouldStop()) { + throw error; + } + } + } +} diff --git a/packages/app/src/assistant-image/use-assistant-image.ts b/packages/app/src/assistant-image/use-assistant-image.ts new file mode 100644 index 000000000..0c0818e8f --- /dev/null +++ b/packages/app/src/assistant-image/use-assistant-image.ts @@ -0,0 +1,544 @@ +import { + useCallback, + useEffect, + useLayoutEffect, + useMemo, + useReducer, + useRef, + useState, +} from "react"; +import type { ImageLoadEvent } from "react-native"; +import { useTranslation } from "react-i18next"; +import type { DaemonClient } from "@getpaseo/client/internal/daemon-client"; +import type { AttachmentMetadata } from "@/attachments/types"; +import { isWeb } from "@/constants/platform"; +import { useStableEvent } from "@/hooks/use-stable-event"; +import { retainAttachmentForGarbageCollection } from "@/attachments/gc-retention"; +import { + persistAttachmentFromBytes, + persistAttachmentFromDataUrl, + releaseAttachmentPreviewUrl, + resolveAttachmentPreviewUrl, +} from "@/attachments/service"; +import { createPreviewAttachmentId, parseImageDataUrl } from "@/attachments/utils"; +import { + getAssistantImageMetadata, + setAssistantImageMetadata, +} from "@/utils/assistant-image-metadata"; +import { resolveAssistantImageSource } from "@/utils/assistant-image-source"; +import { createAssistantImageAcquisitionCache } from "./acquisition-cache"; +import { + createAssistantImageFileAcquisition, + type AssistantImageAcquisition, + type AssistantImageFileAcquisitionPort, +} from "./file-acquisition"; +import { + createAssistantImageLifecycle, + transitionAssistantImageLifecycle, + type AssistantImageLifecycle, + type AssistantImageLifecycleEvent, +} from "./lifecycle"; +import { + type AssistantImageRenderedDimensionsReader, + resolveAssistantImageLoadDimensions, +} from "./load-dimensions"; +import { runAssistantImageOperationWithRetry } from "./retry"; + +interface AssistantImageRenderBinding { + uri: string; + onRef: (instance: unknown) => void; + onLoad: (event: ImageLoadEvent) => void; + onError: () => void; +} + +const renderedDimensions: AssistantImageRenderedDimensionsReader = { + read(renderedImage) { + if (!isWeb || !(renderedImage instanceof HTMLElement)) { + return null; + } + const image = renderedImage.querySelector("img"); + return image && image.naturalWidth > 0 && image.naturalHeight > 0 + ? { width: image.naturalWidth, height: image.naturalHeight } + : null; + }, +}; + +export type AssistantImageResult = + | { + status: "loading"; + binding: AssistantImageRenderBinding | null; + aspectRatio: number | null; + } + | { + status: "loaded"; + binding: AssistantImageRenderBinding; + aspectRatio: number; + } + | { status: "failed"; message: string }; + +interface UseAssistantImageInput { + source: string; + occurrenceKey: string; + client?: DaemonClient | null; + workspaceRoot?: string; + serverId?: string; +} + +type PreviewUrlState = + | { status: "waiting" } + | { status: "loading" } + | { status: "loaded"; uri: string } + | { status: "failed"; error: unknown }; + +type AttachmentAcquisitionState = + | { status: "waiting" } + | { status: "loading" } + | { status: "loaded"; attachment: AttachmentMetadata } + | { status: "failed"; error: unknown }; + +interface DataImage { + mimeType: string; + base64: string; + cacheKey: string; +} + +const attachmentAcquisitionCache = createAssistantImageAcquisitionCache({ + capacity: 500, + onRetain: (attachment) => retainAttachmentForGarbageCollection(attachment.id), +}); + +interface CachedPreviewUrl { + attachment: AttachmentMetadata; + uri: string; +} + +const previewUrlCache = createAssistantImageAcquisitionCache({ + capacity: 500, + onRetain: + ({ attachment, uri }) => + () => { + void releaseAttachmentPreviewUrl({ attachment, url: uri }); + }, +}); + +const LOADED_IMAGE_CACHE_CAPACITY = 500; +const loadedImageCache = new Map(); + +function getLoadedImageAspectRatio(uri: string): number | null { + const aspectRatio = loadedImageCache.get(uri); + if (aspectRatio === undefined) { + return null; + } + loadedImageCache.delete(uri); + loadedImageCache.set(uri, aspectRatio); + return aspectRatio; +} + +function rememberLoadedImage(uri: string, aspectRatio: number): void { + loadedImageCache.delete(uri); + loadedImageCache.set(uri, aspectRatio); + if (loadedImageCache.size <= LOADED_IMAGE_CACHE_CAPACITY) { + return; + } + const leastRecentlyUsedUri = loadedImageCache.keys().next().value; + if (leastRecentlyUsedUri !== undefined) { + loadedImageCache.delete(leastRecentlyUsedUri); + } +} + +function useAttachmentAcquisition( + acquisition: AssistantImageAcquisition | null, +): AttachmentAcquisitionState { + const acquisitionKey = acquisition?.key ?? null; + const [entry, setEntry] = useState<{ + key: string | null; + state: AttachmentAcquisitionState; + }>(() => { + const cached = acquisitionKey ? attachmentAcquisitionCache.peek(acquisitionKey) : undefined; + return { + key: acquisitionKey, + state: cached ? { status: "loaded", attachment: cached } : { status: "waiting" }, + }; + }); + + useEffect(() => { + let disposed = false; + let releaseCurrent: (() => void) | null = null; + if (!acquisition || !acquisitionKey) { + setEntry({ key: null, state: { status: "waiting" } }); + return; + } + + const acquireCurrent = () => { + releaseCurrent?.(); + const retained = attachmentAcquisitionCache.acquireRetained( + acquisitionKey, + acquisition.locate, + ); + releaseCurrent = retained.release; + return retained; + }; + const initial = acquireCurrent(); + if (initial.value) { + setEntry({ key: acquisitionKey, state: { status: "loaded", attachment: initial.value } }); + return () => { + disposed = true; + releaseCurrent?.(); + }; + } + + setEntry({ key: acquisitionKey, state: { status: "loading" } }); + void (async () => { + let firstAttempt: ReturnType | null = initial; + try { + const attachment = await runAssistantImageOperationWithRetry({ + operation: async () => { + const retained = firstAttempt ?? acquireCurrent(); + firstAttempt = null; + return await retained.promise; + }, + shouldStop: () => disposed, + }); + if (!disposed) { + setEntry({ key: acquisitionKey, state: { status: "loaded", attachment } }); + } + } catch (error) { + if (!disposed) { + setEntry({ key: acquisitionKey, state: { status: "failed", error } }); + } + } + })(); + return () => { + disposed = true; + releaseCurrent?.(); + }; + }, [acquisition, acquisitionKey]); + + if (!acquisitionKey) { + return { status: "waiting" }; + } + const cached = attachmentAcquisitionCache.peek(acquisitionKey); + if (cached) { + return { status: "loaded", attachment: cached }; + } + return entry.key === acquisitionKey ? entry.state : { status: "waiting" }; +} + +function createDataImageAcquisition(input: { + source: string; + dataImage: DataImage | null; +}): AssistantImageAcquisition | null { + if (!input.dataImage) { + return null; + } + const { dataImage, source } = input; + return { + key: dataImage.cacheKey, + locate: async () => + await persistAttachmentFromDataUrl({ + id: createPreviewAttachmentId({ + mimeType: dataImage.mimeType, + contentLength: dataImage.base64.length, + contentKey: dataImage.cacheKey, + }), + dataUrl: source, + mimeType: dataImage.mimeType, + }), + }; +} + +function usePreviewUrl(attachment: AttachmentMetadata | null | undefined): PreviewUrlState { + const id = attachment?.id; + const storageType = attachment?.storageType; + const storageKey = attachment?.storageKey; + const mimeType = attachment?.mimeType; + const previewKey = + id && storageType && storageKey && mimeType + ? `${id}:${storageType}:${storageKey}:${mimeType}` + : null; + const [entry, setEntry] = useState<{ key: string | null; state: PreviewUrlState }>(() => { + return { + key: previewKey, + state: { status: "waiting" }, + }; + }); + const getCurrentAttachment = useStableEvent(() => attachment ?? null); + + useLayoutEffect(() => { + let disposed = false; + let releaseCurrent: (() => void) | null = null; + const current = getCurrentAttachment(); + + if (!current || !previewKey) { + setEntry({ key: null, state: { status: "waiting" } }); + return; + } + + const acquireCurrent = () => { + releaseCurrent?.(); + const retained = previewUrlCache.acquireRetained(previewKey, async () => ({ + attachment: current, + uri: await resolveAttachmentPreviewUrl(current), + })); + releaseCurrent = retained.release; + return retained; + }; + const initial = acquireCurrent(); + if (initial.value) { + setEntry({ key: previewKey, state: { status: "loaded", uri: initial.value.uri } }); + return () => { + disposed = true; + releaseCurrent?.(); + }; + } + + setEntry({ key: previewKey, state: { status: "loading" } }); + void (async () => { + let firstAttempt: ReturnType | null = initial; + try { + const preview = await runAssistantImageOperationWithRetry({ + operation: async () => { + const retained = firstAttempt ?? acquireCurrent(); + firstAttempt = null; + return await retained.promise; + }, + shouldStop: () => disposed, + }); + if (!disposed) { + setEntry({ key: previewKey, state: { status: "loaded", uri: preview.uri } }); + } + } catch (error) { + if (!disposed) { + setEntry({ key: previewKey, state: { status: "failed", error } }); + } + } + })(); + + return () => { + disposed = true; + releaseCurrent?.(); + }; + }, [getCurrentAttachment, previewKey]); + + if (!previewKey) { + return { status: "waiting" }; + } + return entry.key === previewKey ? entry.state : { status: "waiting" }; +} + +function lifecycleReducer( + state: AssistantImageLifecycle, + event: AssistantImageLifecycleEvent, +): AssistantImageLifecycle { + return transitionAssistantImageLifecycle(state, event); +} + +function errorMessage(error: unknown, fallback: string): string { + return error instanceof Error ? error.message : fallback; +} + +function getAcquisitionFailure(input: { + hasResolution: boolean; + isFileSource: boolean; + isDataImage: boolean; + fileAttachment: AttachmentAcquisitionState; + dataImageAttachment: AttachmentAcquisitionState; + preview: PreviewUrlState; + hasDirectUri: boolean; + fallbackMessage: string; +}): AssistantImageResult | null { + if (!input.hasResolution) { + return { status: "failed", message: input.fallbackMessage }; + } + if (input.isFileSource && input.fileAttachment.status === "failed") { + return { + status: "failed", + message: errorMessage(input.fileAttachment.error, input.fallbackMessage), + }; + } + if (input.isDataImage && input.dataImageAttachment.status === "failed") { + return { + status: "failed", + message: errorMessage(input.dataImageAttachment.error, input.fallbackMessage), + }; + } + if (!input.hasDirectUri && input.preview.status === "failed") { + return { + status: "failed", + message: errorMessage(input.preview.error, input.fallbackMessage), + }; + } + return null; +} + +export function useAssistantImage({ + source, + occurrenceKey, + client, + workspaceRoot, + serverId, +}: UseAssistantImageInput): AssistantImageResult { + const { t } = useTranslation(); + const resolution = useMemo( + () => resolveAssistantImageSource({ source, workspaceRoot }), + [source, workspaceRoot], + ); + const dataImage = useMemo(() => parseImageDataUrl(source), [source]); + const fileAcquisition = useMemo(() => { + const port: AssistantImageFileAcquisitionPort | null = client + ? { + readFile: async (cwd, path) => await client.readFile(cwd, path), + persist: persistAttachmentFromBytes, + } + : null; + return createAssistantImageFileAcquisition({ + port, + resolution, + serverId, + occurrenceKey, + unavailableMessage: t("message.attachments.imagePreviewUnavailable"), + }); + }, [client, occurrenceKey, resolution, serverId, t]); + const dataImageAcquisition = useMemo( + () => createDataImageAcquisition({ source, dataImage }), + [dataImage, source], + ); + const fileAttachment = useAttachmentAcquisition(fileAcquisition); + const dataImageAttachment = useAttachmentAcquisition(dataImageAcquisition); + const filePreview = usePreviewUrl( + fileAttachment.status === "loaded" ? fileAttachment.attachment : null, + ); + const dataImagePreview = usePreviewUrl( + dataImageAttachment.status === "loaded" ? dataImageAttachment.attachment : null, + ); + const directUri = resolution?.kind === "direct" && !dataImage ? resolution.uri : null; + const preview = dataImage ? dataImagePreview : filePreview; + const previewUri = preview.status === "loaded" ? preview.uri : null; + const uri = directUri ?? previewUri; + const cachedMetadata = useMemo( + () => getAssistantImageMetadata({ source, workspaceRoot, serverId }), + [serverId, source, workspaceRoot], + ); + const [lifecycle, dispatchLifecycle] = useReducer( + lifecycleReducer, + uri, + (initialUri): AssistantImageLifecycle => { + if (initialUri) { + const aspectRatio = getLoadedImageAspectRatio(initialUri); + if (aspectRatio !== null) { + return { status: "loaded", uri: initialUri, aspectRatio }; + } + } + return createAssistantImageLifecycle(); + }, + ); + const dispatch = useCallback((event: AssistantImageLifecycleEvent) => { + if (event.type === "image_loaded") { + rememberLoadedImage(event.uri, event.aspectRatio); + } else if (event.type === "failed" && event.uri) { + loadedImageCache.delete(event.uri); + } + dispatchLifecycle(event); + }, []); + const renderedImageRef = useRef(null); + const handleImageRef = useCallback((instance: unknown) => { + renderedImageRef.current = instance; + }, []); + + useEffect(() => { + if (!uri) { + dispatch({ type: "preview_released" }); + return; + } + + dispatch({ + type: "preview_created", + uri, + aspectRatio: cachedMetadata?.aspectRatio ?? null, + }); + }, [cachedMetadata, dispatch, uri]); + + const handleImageError = useCallback(() => { + if (uri) { + dispatch({ + type: "failed", + uri, + message: t("message.attachments.imageUnavailable"), + }); + } + }, [dispatch, t, uri]); + const handleImageLoad = useCallback( + (event: ImageLoadEvent) => { + if (!uri) { + return; + } + const nativeEvent = event.nativeEvent as ImageLoadEvent["nativeEvent"] & { + target?: { naturalWidth?: unknown; naturalHeight?: unknown }; + }; + const dimensions = resolveAssistantImageLoadDimensions({ + source: nativeEvent.source, + target: nativeEvent.target, + renderedImage: renderedImageRef.current, + renderedDimensions, + }); + const metadata = dimensions + ? setAssistantImageMetadata({ source, workspaceRoot, serverId }, dimensions) + : null; + const aspectRatio = metadata?.aspectRatio ?? cachedMetadata?.aspectRatio ?? null; + if (!aspectRatio) { + dispatch({ + type: "failed", + uri, + message: t("message.attachments.imageUnavailable"), + }); + return; + } + dispatch({ type: "image_loaded", uri, aspectRatio }); + }, + [cachedMetadata, dispatch, serverId, source, t, uri, workspaceRoot], + ); + + const acquisitionFailure = getAcquisitionFailure({ + hasResolution: resolution !== null, + isFileSource: resolution?.kind === "file_rpc", + isDataImage: dataImage !== null, + fileAttachment, + dataImageAttachment, + preview, + hasDirectUri: directUri !== null, + fallbackMessage: t("message.attachments.imagePreviewLoadFailed"), + }); + if (acquisitionFailure) { + return acquisitionFailure; + } + const hasCurrentLifecycleUri = lifecycle.status !== "failed" && lifecycle.uri === uri; + let binding: AssistantImageRenderBinding | null = null; + if (hasCurrentLifecycleUri && lifecycle.uri) { + binding = { + uri: lifecycle.uri, + onRef: handleImageRef, + onLoad: handleImageLoad, + onError: handleImageError, + }; + } + if (lifecycle.status === "loaded" && lifecycle.uri === uri) { + return { + status: "loaded", + binding: { + uri: lifecycle.uri, + onRef: handleImageRef, + onLoad: handleImageLoad, + onError: handleImageError, + }, + aspectRatio: lifecycle.aspectRatio, + }; + } + if (lifecycle.status === "failed") { + return lifecycle; + } + return { + status: "loading", + binding, + aspectRatio: hasCurrentLifecycleUri ? lifecycle.aspectRatio : null, + }; +} diff --git a/packages/app/src/attachments/gc-retention.test.ts b/packages/app/src/attachments/gc-retention.test.ts new file mode 100644 index 000000000..4b3992681 --- /dev/null +++ b/packages/app/src/attachments/gc-retention.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from "vitest"; +import { collectRetainedAttachmentIds, retainAttachmentForGarbageCollection } from "./gc-retention"; + +describe("attachment garbage-collection retention", () => { + it("keeps an attachment referenced until its final owner releases it", () => { + const releaseFirst = retainAttachmentForGarbageCollection("preview-1"); + const releaseSecond = retainAttachmentForGarbageCollection("preview-1"); + + expect(collectRetainedAttachmentIds()).toContain("preview-1"); + releaseFirst(); + expect(collectRetainedAttachmentIds()).toContain("preview-1"); + releaseSecond(); + expect(collectRetainedAttachmentIds()).not.toContain("preview-1"); + }); +}); diff --git a/packages/app/src/attachments/gc-retention.ts b/packages/app/src/attachments/gc-retention.ts new file mode 100644 index 000000000..4473350d9 --- /dev/null +++ b/packages/app/src/attachments/gc-retention.ts @@ -0,0 +1,22 @@ +const retentionCounts = new Map(); + +export function retainAttachmentForGarbageCollection(attachmentId: string): () => void { + retentionCounts.set(attachmentId, (retentionCounts.get(attachmentId) ?? 0) + 1); + let released = false; + return () => { + if (released) { + return; + } + released = true; + const nextCount = (retentionCounts.get(attachmentId) ?? 1) - 1; + if (nextCount <= 0) { + retentionCounts.delete(attachmentId); + return; + } + retentionCounts.set(attachmentId, nextCount); + }; +} + +export function collectRetainedAttachmentIds(): ReadonlySet { + return new Set(retentionCounts.keys()); +} diff --git a/packages/app/src/attachments/service.test.ts b/packages/app/src/attachments/service.test.ts index 080204ed0..05e13034f 100644 --- a/packages/app/src/attachments/service.test.ts +++ b/packages/app/src/attachments/service.test.ts @@ -1,7 +1,11 @@ import { afterEach, describe, expect, it } from "vitest"; import type { AttachmentMetadata, AttachmentStore, SaveAttachmentInput } from "@/attachments/types"; import { __setAttachmentStoreForTests } from "./store"; -import { encodeAttachmentsForSend, persistAttachmentFromBytes } from "./service"; +import { + encodeAttachmentsForSend, + garbageCollectAttachments, + persistAttachmentFromBytes, +} from "./service"; function createAttachment(input: Partial = {}): AttachmentMetadata { return { @@ -94,4 +98,46 @@ describe("attachment service", () => { { data: "att_send:base64", mimeType: "image/jpeg" }, ]); }); + + it("does not collect an attachment persisted while garbage collection is starting", async () => { + let releaseSave: () => void = () => undefined; + let reportSaveStarted: () => void = () => undefined; + const saveStarted = new Promise((resolve) => { + reportSaveStarted = resolve; + }); + const saveGate = new Promise((resolve) => { + releaseSave = resolve; + }); + const garbageCollections: string[][] = []; + const store: AttachmentStore = { + ...createRecordingStore(), + async save(input) { + reportSaveStarted(); + await saveGate; + return createAttachment({ id: input.id }); + }, + async garbageCollect({ referencedIds }) { + garbageCollections.push([...referencedIds]); + }, + }; + __setAttachmentStoreForTests(store); + + const persist = persistAttachmentFromBytes({ + id: "assistant-preview", + bytes: new Uint8Array([1, 2, 3]), + mimeType: "image/png", + }); + await saveStarted; + const collect = garbageCollectAttachments({ referencedIds: new Set() }); + + try { + await Promise.resolve(); + expect(garbageCollections).toEqual([]); + } finally { + releaseSave(); + await Promise.all([persist, collect]); + } + + expect(garbageCollections).toEqual([["assistant-preview"]]); + }); }); diff --git a/packages/app/src/attachments/service.ts b/packages/app/src/attachments/service.ts index 15cff1ee2..d878b1986 100644 --- a/packages/app/src/attachments/service.ts +++ b/packages/app/src/attachments/service.ts @@ -1,5 +1,40 @@ -import type { AttachmentMetadata } from "@/attachments/types"; +import { collectRetainedAttachmentIds } from "@/attachments/gc-retention"; import { getAttachmentStore } from "@/attachments/store"; +import type { AttachmentMetadata, SaveAttachmentInput } from "@/attachments/types"; + +const activePersistence = new Set>(); +const persistedDuringGarbageCollection = new Set(); +let pendingGarbageCollections = 0; +let garbageCollectionTail: Promise = Promise.resolve(); +let persistenceBarrier: Promise | null = null; +let releasePersistenceBarrier: (() => void) | null = null; + +async function waitForPersistenceBarrier(): Promise { + const barrier = persistenceBarrier; + if (!barrier) { + return; + } + await barrier; + await waitForPersistenceBarrier(); +} + +async function persistAttachment(input: SaveAttachmentInput): Promise { + await waitForPersistenceBarrier(); + const pending = (async () => { + const store = await getAttachmentStore(); + const attachment = await store.save(input); + if (pendingGarbageCollections > 0) { + persistedDuringGarbageCollection.add(attachment.id); + } + return attachment; + })(); + activePersistence.add(pending); + try { + return await pending; + } finally { + activePersistence.delete(pending); + } +} export async function persistAttachmentFromBlob(input: { blob: Blob; @@ -7,8 +42,7 @@ export async function persistAttachmentFromBlob(input: { fileName?: string | null; id?: string; }): Promise { - const store = await getAttachmentStore(); - return await store.save({ + return await persistAttachment({ id: input.id, mimeType: input.mimeType, fileName: input.fileName, @@ -22,8 +56,7 @@ export async function persistAttachmentFromDataUrl(input: { fileName?: string | null; id?: string; }): Promise { - const store = await getAttachmentStore(); - return await store.save({ + return await persistAttachment({ id: input.id, mimeType: input.mimeType, fileName: input.fileName, @@ -37,8 +70,7 @@ export async function persistAttachmentFromBytes(input: { fileName?: string | null; id?: string; }): Promise { - const store = await getAttachmentStore(); - return await store.save({ + return await persistAttachment({ id: input.id, mimeType: input.mimeType, fileName: input.fileName, @@ -52,8 +84,7 @@ export async function persistAttachmentFromFileUri(input: { fileName?: string | null; id?: string; }): Promise { - const store = await getAttachmentStore(); - return await store.save({ + return await persistAttachment({ id: input.id, mimeType: input.mimeType, fileName: input.fileName, @@ -133,6 +164,41 @@ export async function deleteAttachments( export async function garbageCollectAttachments(input: { referencedIds: ReadonlySet; }): Promise { - const store = await getAttachmentStore(); - await store.garbageCollect({ referencedIds: input.referencedIds }); + pendingGarbageCollections += 1; + if (!persistenceBarrier) { + persistenceBarrier = new Promise((resolve) => { + releasePersistenceBarrier = resolve; + }); + } + + const previousGarbageCollection = garbageCollectionTail; + const currentGarbageCollection = (async () => { + await previousGarbageCollection; + while (activePersistence.size > 0) { + await Promise.allSettled(activePersistence); + } + const referencedIds = new Set(input.referencedIds); + for (const id of collectRetainedAttachmentIds()) { + referencedIds.add(id); + } + for (const id of persistedDuringGarbageCollection) { + referencedIds.add(id); + } + const store = await getAttachmentStore(); + await store.garbageCollect({ referencedIds }); + })(); + garbageCollectionTail = currentGarbageCollection.catch(() => undefined); + + try { + await currentGarbageCollection; + } finally { + pendingGarbageCollections -= 1; + if (pendingGarbageCollections === 0) { + persistedDuringGarbageCollection.clear(); + const release = releasePersistenceBarrier; + releasePersistenceBarrier = null; + persistenceBarrier = null; + release?.(); + } + } } diff --git a/packages/app/src/attachments/utils.test.ts b/packages/app/src/attachments/utils.test.ts index 319f247d2..d6fce078f 100644 --- a/packages/app/src/attachments/utils.test.ts +++ b/packages/app/src/attachments/utils.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { createImageSourceCacheKey, + createPreviewAttachmentId, fileUriToPath, localFileSourceToPath, parseDataUrl, @@ -86,4 +87,29 @@ describe("parseImageDataUrl", () => { it("ignores SVG data URLs", () => { expect(parseImageDataUrl("data:image/svg+xml;base64,PHN2ZyAvPg==")).toBeNull(); }); + + it("distinguishes image data that differs only in the middle", () => { + const prefix = "a".repeat(64); + const suffix = "z".repeat(64); + const first = `data:image/png;base64,${prefix}${"b".repeat(256)}${suffix}`; + const second = `data:image/png;base64,${prefix}${"c".repeat(256)}${suffix}`; + + expect(createImageSourceCacheKey(first)).not.toBe(createImageSourceCacheKey(second)); + }); + + it("gives equal-length preview content distinct attachment identities", () => { + expect( + createPreviewAttachmentId({ + mimeType: "image/png", + contentLength: 512, + contentKey: "first-content", + }), + ).not.toBe( + createPreviewAttachmentId({ + mimeType: "image/png", + contentLength: 512, + contentKey: "second-content", + }), + ); + }); }); diff --git a/packages/app/src/attachments/utils.ts b/packages/app/src/attachments/utils.ts index bb1903f9f..5fa033435 100644 --- a/packages/app/src/attachments/utils.ts +++ b/packages/app/src/attachments/utils.ts @@ -56,7 +56,7 @@ export function parseImageDataUrl( if (!isRasterImageMimeType(parsed.mimeType)) { return null; } - const fingerprint = `${parsed.mimeType}\0${parsed.base64.length}\0${parsed.base64.slice(0, 64)}\0${parsed.base64.slice(-64)}`; + const fingerprint = `${parsed.mimeType}\0${parsed.base64}`; return { ...parsed, cacheKey: `data-image:${parsed.mimeType}:${parsed.base64.length}:${hashString(fingerprint)}`, @@ -87,12 +87,15 @@ export function createPreviewAttachmentId(input: { size?: number | null; modifiedAt?: string | null; contentLength?: number | null; + contentKey?: string | null; }): string { const path = input.path?.trim() ?? ""; const size = Number.isFinite(input.size) ? String(input.size) : ""; const modifiedAt = input.modifiedAt?.trim() ?? ""; const contentLength = Number.isFinite(input.contentLength) ? String(input.contentLength) : ""; - const hash = hashString(`${input.mimeType}\0${path}\0${size}\0${modifiedAt}\0${contentLength}`); + const contentKey = input.contentKey?.trim() ?? ""; + const identity = `${input.mimeType}\0${path}\0${size}\0${modifiedAt}\0${contentLength}`; + const hash = hashString(contentKey ? `${identity}\0${contentKey}` : identity); return `preview_${size || contentLength || "unknown"}_${hash}`; } diff --git a/packages/app/src/components/message.tsx b/packages/app/src/components/message.tsx index e1870bd72..8a85a8645 100644 --- a/packages/app/src/components/message.tsx +++ b/packages/app/src/components/message.tsx @@ -29,7 +29,6 @@ import { } from "react"; import type { ComponentType, ReactNode } from "react"; import { MarkdownIt, type ASTNode, type RenderRules } from "react-native-markdown-display"; -import { useQuery } from "@tanstack/react-query"; import MaskedView from "@react-native-masked-view/masked-view"; import { Circle, @@ -75,19 +74,7 @@ import { splitMarkdownBlocks } from "@/utils/split-markdown-blocks"; import { formatDuration, formatMessageTimestamp } from "@/utils/time"; import { writeMarkdownToRichClipboard } from "@/utils/rich-clipboard"; import { getDefaultMarkdownClipboardEnvironment } from "@/utils/rich-clipboard-default-environment"; -import { - getAssistantImageLoadStateFromMetadata, - getAssistantImageMetadata, - setAssistantImageMetadata, - type AssistantImageLoadState, -} from "@/utils/assistant-image-metadata"; import { setAssistantMarkdownBlockHeight } from "@/utils/assistant-message-height-estimate"; -import { resolveAssistantImageSource } from "@/utils/assistant-image-source"; -import { - createPreviewAttachmentId, - getFileNameFromPath, - parseImageDataUrl, -} from "@/attachments/utils"; import { getAgentAttachmentPillContent } from "@/attachments/attachment-pill-content"; import { PlanCard } from "./plan-card"; import { useToolCallSheet } from "./tool-call-sheet"; @@ -102,8 +89,7 @@ import { useAssistantLinkPress, } from "@/assistant-file-links"; import { getCompactionMarkerLabel } from "./message-compaction-label"; -import { useAttachmentPreviewUrl } from "@/attachments/use-attachment-preview-url"; -import { persistAttachmentFromBytes, persistAttachmentFromDataUrl } from "@/attachments/service"; +import { useAssistantImage } from "@/assistant-image/use-assistant-image"; import { AttachmentFrame, AttachmentLabel, @@ -737,6 +723,7 @@ export const LiveElapsed = memo(function LiveElapsed({ }); interface AssistantMessageProps { + occurrenceKey: string; message: string; timestamp: number; workspaceRoot?: string; @@ -764,11 +751,21 @@ export const assistantMessageStylesheet = StyleSheet.create((theme) => ({ imageSurface: { width: "100%", overflow: "hidden", + position: "relative", }, image: { width: "100%", height: "100%", }, + imageLoadingOverlay: { + position: "absolute", + top: 0, + right: 0, + bottom: 0, + left: 0, + alignItems: "center", + justifyContent: "center", + }, imageState: { alignItems: "center", justifyContent: "center", @@ -785,125 +782,9 @@ export const assistantMessageStylesheet = StyleSheet.create((theme) => ({ const ASSISTANT_IMAGE_MIN_HEIGHT = 160; -const AssistantMarkdownResolvedImage = memo(function AssistantMarkdownResolvedImage({ - uri, - alt, - containerStyle, - source, - workspaceRoot, - serverId, -}: { - uri: string; - alt?: string; - containerStyle?: StyleProp; - source: string; - workspaceRoot?: string; - serverId?: string; -}) { - const cachedMetadata = useMemo( - () => getAssistantImageMetadata({ source, workspaceRoot, serverId }), - [serverId, source, workspaceRoot], - ); - const [loadState, setLoadState] = useState(() => - getAssistantImageLoadStateFromMetadata(cachedMetadata), - ); - - useEffect(() => { - if (cachedMetadata) { - setLoadState(getAssistantImageLoadStateFromMetadata(cachedMetadata)); - return () => {}; - } - - setLoadState({ status: "loading" }); - let cancelled = false; - - Image.getSize( - uri, - (width, height) => { - if (cancelled) { - return; - } - if (width > 0 && height > 0) { - const metadata = setAssistantImageMetadata( - { source, workspaceRoot, serverId }, - { width, height }, - ); - setLoadState({ - status: "ready", - aspectRatio: metadata?.aspectRatio ?? width / height, - }); - } - }, - () => { - if (cancelled) { - return; - } - setLoadState({ status: "error" }); - }, - ); - - return () => { - cancelled = true; - }; - }, [cachedMetadata, serverId, source, uri, workspaceRoot]); - - const handleImageError = useCallback(() => { - setLoadState({ status: "error" }); - }, []); - const { t } = useTranslation(); - const surfaceStyle = useMemo>( - () => [ - assistantMessageStylesheet.imageSurface, - loadState.status === "ready" - ? { aspectRatio: loadState.aspectRatio } - : { height: ASSISTANT_IMAGE_MIN_HEIGHT }, - ], - [loadState], - ); - const frameStyle = useMemo>( - () => [assistantMessageStylesheet.imageFrame, containerStyle], - [containerStyle], - ); - const stateSurfaceStyle = useMemo>( - () => [surfaceStyle, assistantMessageStylesheet.imageState], - [surfaceStyle], - ); - const imageSource = useMemo(() => ({ uri }), [uri]); - - if (loadState.status !== "ready") { - return ( - - - {loadState.status === "loading" ? ( - - ) : null} - {loadState.status === "error" ? ( - - {t("message.attachments.imageUnavailable")} - - ) : null} - - - ); - } - - return ( - - - - - - ); -}); - function AssistantMarkdownImage({ source, + occurrenceKey, alt, hasLeadingContent, client, @@ -911,18 +792,13 @@ function AssistantMarkdownImage({ serverId, }: { source: string; + occurrenceKey: string; alt?: string; hasLeadingContent: boolean; client?: DaemonClient | null; workspaceRoot?: string; serverId?: string; }) { - const { t } = useTranslation(); - const resolution = useMemo( - () => resolveAssistantImageSource({ source, workspaceRoot }), - [source, workspaceRoot], - ); - const dataImage = useMemo(() => parseImageDataUrl(source), [source]); const containerStyle = useMemo>( () => ({ marginTop: hasLeadingContent ? 16 : 0, @@ -930,64 +806,31 @@ function AssistantMarkdownImage({ }), [hasLeadingContent], ); - - const query = useQuery({ - queryKey: [ - "assistantMarkdownImage", - serverId ?? "unknown-server", - resolution?.kind === "file_rpc" ? resolution.cwd : null, - resolution?.kind === "file_rpc" ? resolution.path : null, - ], - enabled: Boolean(client && resolution?.kind === "file_rpc"), - staleTime: 30_000, - queryFn: async () => { - if (!client || !resolution || resolution.kind !== "file_rpc") { - return null; - } - - const file = await client.readFile(resolution.cwd, resolution.path); - if (file.kind !== "image") { - throw new Error(t("message.attachments.imagePreviewUnavailable")); - } - - return await persistAttachmentFromBytes({ - id: createPreviewAttachmentId({ - mimeType: file.mime, - path: file.path || resolution.path, - size: file.size, - modifiedAt: file.modifiedAt, - contentLength: file.bytes.byteLength, - }), - bytes: file.bytes, - mimeType: file.mime, - fileName: getFileNameFromPath(file.path || resolution.path), - }); - }, + const image = useAssistantImage({ + source, + occurrenceKey, + client, + workspaceRoot, + serverId, }); - const dataImageQuery = useQuery({ - queryKey: ["assistantMarkdownDataImage", dataImage?.cacheKey ?? null], - enabled: dataImage !== null, - staleTime: 30_000, - queryFn: async () => { - if (!dataImage) { - return null; - } - - return await persistAttachmentFromDataUrl({ - id: createPreviewAttachmentId({ - mimeType: dataImage.mimeType, - contentLength: dataImage.base64.length, - }), - dataUrl: source, - mimeType: dataImage.mimeType, - }); - }, - }); - - const fileAssetUri = useAttachmentPreviewUrl(query.data); - const dataImageAssetUri = useAttachmentPreviewUrl(dataImageQuery.data); - const directUri = resolution?.kind === "direct" && !dataImage ? resolution.uri : null; - const resolvedUri = directUri ?? dataImageAssetUri ?? fileAssetUri ?? null; + const binding = image.status === "failed" ? null : image.binding; + const aspectRatio = image.status === "failed" ? null : image.aspectRatio; + const imageUri = binding?.uri ?? ""; + const imageSource = useMemo(() => ({ uri: imageUri }), [imageUri]); + const frameStyle = useMemo>( + () => [assistantMessageStylesheet.imageFrame, containerStyle], + [containerStyle], + ); + const imageSizeStyle = useMemo(() => { + if (aspectRatio) { + return { aspectRatio }; + } + return { height: ASSISTANT_IMAGE_MIN_HEIGHT }; + }, [aspectRatio]); + const surfaceStyle = useMemo>( + () => [assistantMessageStylesheet.imageSurface, imageSizeStyle], + [imageSizeStyle], + ); const stateFrameStyle = useMemo>( () => [ @@ -999,20 +842,15 @@ function AssistantMarkdownImage({ [containerStyle], ); - if (resolvedUri) { + if (image.status === "failed") { return ( - + + {image.message} + ); } - if (query.isLoading || dataImageQuery.isLoading) { + if (!binding) { return ( @@ -1020,29 +858,27 @@ function AssistantMarkdownImage({ ); } - const errorText = resolveAssistantImageErrorText( - query.error, - dataImageQuery.error, - t("message.attachments.imagePreviewLoadFailed"), - ); - return ( - - {errorText} + + + + {image.status === "loading" ? ( + + + + ) : null} + ); } -function resolveAssistantImageErrorText( - fileError: unknown, - dataError: unknown, - fallbackText: string, -): string { - if (fileError instanceof Error) return fileError.message; - if (dataError instanceof Error) return dataError.message; - return fallbackText; -} - function getInlineCodeAutoLinkUrl( markdownParser: ReturnType, content: string, @@ -1583,6 +1419,7 @@ function MarkdownListView({ baseStyle, spacing, children }: MarkdownListViewProp } export const AssistantMessage = memo(function AssistantMessage({ + occurrenceKey, message, timestamp: _timestamp, workspaceRoot, @@ -1917,6 +1754,7 @@ export const AssistantMessage = memo(function AssistantMessage({ splitMarkdownBlocks(message), [message]); const keyedBlocks = useMemo( diff --git a/packages/app/src/components/ui/loading-spinner.tsx b/packages/app/src/components/ui/loading-spinner.tsx index f73fd037b..abcbc64e8 100644 --- a/packages/app/src/components/ui/loading-spinner.tsx +++ b/packages/app/src/components/ui/loading-spinner.tsx @@ -1,3 +1,4 @@ +import React from "react"; import { ActivityIndicator, type ActivityIndicatorProps } from "react-native"; interface LoadingSpinnerProps { diff --git a/packages/app/src/contexts/session-context.tsx b/packages/app/src/contexts/session-context.tsx index 8a627d614..de18f551f 100644 --- a/packages/app/src/contexts/session-context.tsx +++ b/packages/app/src/contexts/session-context.tsx @@ -40,7 +40,12 @@ import type { AgentPermissionResponse } from "@getpaseo/protocol/agent-types"; import { getHostRuntimeStore, useHostRuntimeIsConnected } from "@/runtime/host-runtime"; import { useVoiceAudioEngineOptional, useVoiceRuntimeOptional } from "@/contexts/voice-context"; import type { AudioPlaybackSource } from "@/voice/audio-engine-types"; -import { useSessionStore, type MessageEntry, type SessionState } from "@/stores/session-store"; +import { + selectAgentTimelineState, + useSessionStore, + type MessageEntry, + type SessionState, +} from "@/stores/session-store"; import { useWorkspaceSetupStore } from "@/stores/workspace-setup-store"; import { sendOsNotification } from "@/utils/os-notifications"; import { getIsAppActivelyVisible, getIsAppVisible } from "@/utils/app-visibility"; @@ -192,11 +197,6 @@ type WorkspaceSetupProgressPayload = Extract< type SessionStoreActions = ReturnType; type SetInitializingAgents = SessionStoreActions["setInitializingAgents"]; -type SetAgentStreamState = SessionStoreActions["setAgentStreamState"]; -type SetAgentTimelineCursor = SessionStoreActions["setAgentTimelineCursor"]; -type MarkAgentHistorySynchronized = SessionStoreActions["markAgentHistorySynchronized"]; -type SetAgentAuthoritativeHistoryApplied = - SessionStoreActions["setAgentAuthoritativeHistoryApplied"]; function clearAgentInitializingFlag( setInitializingAgents: SetInitializingAgents, @@ -229,65 +229,6 @@ function handleTimelineError(input: { } } -function applyTimelineStreamPatches(input: { - result: ProcessTimelineResponseOutput; - agentId: string; - serverId: string; - currentTail: StreamItem[]; - currentHead: StreamItem[]; - setAgentStreamState: SetAgentStreamState; - setAgentTimelineCursor: SetAgentTimelineCursor; -}): void { - const { - result, - agentId, - serverId, - currentTail, - currentHead, - setAgentStreamState, - setAgentTimelineCursor, - } = input; - - if ( - result.tail !== currentTail || - result.head !== currentHead || - result.acknowledgedClientMessageIds.length > 0 - ) { - setAgentStreamState(serverId, agentId, { - ...(result.tail !== currentTail ? { tail: result.tail } : {}), - ...(result.head !== currentHead ? { head: result.head } : {}), - ...(result.acknowledgedClientMessageIds.length > 0 - ? { acknowledgedClientMessageIds: result.acknowledgedClientMessageIds } - : {}), - }); - } - - if (result.cursorChanged) { - setAgentTimelineCursor(serverId, (prev) => { - const current = prev.get(agentId); - if (!result.cursor) { - if (!current) { - return prev; - } - const next = new Map(prev); - next.delete(agentId); - return next; - } - if ( - current && - current.epoch === result.cursor.epoch && - current.startSeq === result.cursor.startSeq && - current.endSeq === result.cursor.endSeq - ) { - return prev; - } - const next = new Map(prev); - next.set(agentId, result.cursor); - return next; - }); - } -} - function executeTimelineSideEffects(input: { sideEffects: TimelineReducerSideEffect[]; agentId: string; @@ -308,8 +249,6 @@ function finalizeTimelineApplication(input: { serverId: string; shouldMarkAuthoritativeHistoryApplied: boolean; setInitializingAgents: SetInitializingAgents; - setAgentAuthoritativeHistoryApplied: SetAgentAuthoritativeHistoryApplied; - markAgentHistorySynchronized: MarkAgentHistorySynchronized; }): void { const { result, @@ -318,17 +257,13 @@ function finalizeTimelineApplication(input: { serverId, shouldMarkAuthoritativeHistoryApplied, setInitializingAgents, - setAgentAuthoritativeHistoryApplied, - markAgentHistorySynchronized, } = input; if (result.clearInitializing) { clearAgentInitializingFlag(setInitializingAgents, serverId, agentId); } if (shouldMarkAuthoritativeHistoryApplied) { - setAgentAuthoritativeHistoryApplied(serverId, agentId, true); useCreateFlowStore.getState().clearByAgent({ serverId, agentId }); - markAgentHistorySynchronized(serverId, agentId); const session = useSessionStore.getState().sessions[serverId]; const agent = session?.agents.get(agentId) ?? session?.agentDetails.get(agentId); if (agent && agent.status !== "running") { @@ -414,14 +349,10 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider const setAgentStreamState = useSessionStore((state) => state.setAgentStreamState); const clearAgentStreamHead = useSessionStore((state) => state.clearAgentStreamHead); const setAgentTimelineCursor = useSessionStore((state) => state.setAgentTimelineCursor); - const setAgentTimelineHasOlder = useSessionStore((state) => state.setAgentTimelineHasOlder); const setInitializingAgents = useSessionStore((state) => state.setInitializingAgents); const bumpHistorySyncGeneration = useSessionStore((state) => state.bumpHistorySyncGeneration); - const markAgentHistorySynchronized = useSessionStore( - (state) => state.markAgentHistorySynchronized, - ); - const setAgentAuthoritativeHistoryApplied = useSessionStore( - (state) => state.setAgentAuthoritativeHistoryApplied, + const applyAgentTimelineResponseState = useSessionStore( + (state) => state.applyAgentTimelineResponseState, ); const setAgents = useSessionStore((state) => state.setAgents); const setWorkspaces = useSessionStore((state) => state.setWorkspaces); @@ -644,22 +575,15 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider const isInitializing = session?.initializingAgents.get(agentId) === true; const activeInitDeferred = getInitDeferred(initKey); const hasActiveInitDeferred = Boolean(activeInitDeferred); - const currentCursor = session?.agentTimelineCursor.get(agentId); - const currentTail = session?.agentStreamTail.get(agentId) ?? []; + const timeline = selectAgentTimelineState(session, agentId); + const currentCursor = + timeline.status === "synced" ? (timeline.range ?? undefined) : undefined; + const currentTail = timeline.status === "cold" ? [] : timeline.items; const currentHead = session?.agentStreamHead.get(agentId) ?? []; const sendingClientMessageIds = getSendingClientMessageIds( session?.messageSubmissions.get(agentId), ); - setAgentTimelineHasOlder(serverId, (prev) => { - if (prev.get(agentId) === payload.hasOlder) { - return prev; - } - const next = new Map(prev); - next.set(agentId, payload.hasOlder); - return next; - }); - // Call pure reducer const result = processTimelineResponse({ payload, @@ -670,6 +594,7 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider hasActiveInitDeferred, initRequestDirection: activeInitDeferred?.requestDirection ?? "tail", sendingClientMessageIds, + hasAuthoritativeBaseline: timeline.status === "synced", }); if (result.error) { @@ -683,14 +608,13 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider return; } - applyTimelineStreamPatches({ - result, - agentId, - serverId, - currentTail, - currentHead, - setAgentStreamState, - setAgentTimelineCursor, + applyAgentTimelineResponseState(serverId, agentId, { + items: result.tail, + head: result.head, + range: result.cursorChanged ? (result.cursor ?? null) : (currentCursor ?? null), + older: payload.hasOlder ? "available" : "none", + synchronized: shouldMarkAuthoritativeHistoryApplied, + acknowledgedClientMessageIds: result.acknowledgedClientMessageIds, }); executeTimelineSideEffects({ @@ -706,20 +630,9 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider serverId, shouldMarkAuthoritativeHistoryApplied, setInitializingAgents, - setAgentAuthoritativeHistoryApplied, - markAgentHistorySynchronized, }); }, - [ - markAgentHistorySynchronized, - recoverTimelineGap, - serverId, - setAgentAuthoritativeHistoryApplied, - setAgentStreamState, - setAgentTimelineCursor, - setAgentTimelineHasOlder, - setInitializingAgents, - ], + [applyAgentTimelineResponseState, recoverTimelineGap, serverId, setInitializingAgents], ); useEffect(() => { @@ -730,16 +643,20 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider const sync = createViewedTimelineSync({ initialDeliveryMode, setSubscription: (agentIds) => client.setAgentTimelineSubscription(agentIds), - readCursor: (agentId) => - useSessionStore.getState().sessions[serverId]?.agentTimelineCursor.get(agentId), + readCursor: (agentId) => { + const timeline = selectAgentTimelineState( + useSessionStore.getState().sessions[serverId], + agentId, + ); + return timeline.status === "synced" ? (timeline.range ?? undefined) : undefined; + }, hasAuthoritativeHistory: (agentId) => - useSessionStore - .getState() - .sessions[serverId]?.agentAuthoritativeHistoryApplied.get(agentId) === true, + selectAgentTimelineState(useSessionStore.getState().sessions[serverId], agentId).status === + "synced", fetchPage: async (agentId, request) => { const session = useSessionStore.getState().sessions[serverId]; const initKey = getInitKey(serverId, agentId); - const shouldInitialize = session?.agentAuthoritativeHistoryApplied.get(agentId) !== true; + const shouldInitialize = selectAgentTimelineState(session, agentId).status !== "synced"; if (shouldInitialize) { if (!getInitDeferred(initKey)) { const deferred = createInitDeferred(initKey, request.direction ?? "tail"); diff --git a/packages/app/src/hooks/use-agent-initialization.test.ts b/packages/app/src/hooks/use-agent-initialization.test.ts index 52fb58e1e..e78388a90 100644 --- a/packages/app/src/hooks/use-agent-initialization.test.ts +++ b/packages/app/src/hooks/use-agent-initialization.test.ts @@ -111,6 +111,47 @@ describe("ensureAgentIsInitialized", () => { expect(getInitDeferred(getInitKey(serverId, agentId))?.requestDirection).toBe("tail"); }); + it("requests a bounded projected tail after restoring painted replica items", () => { + const client = new FakeDaemonClient(); + const runtime = new FakeTimelineRuntime(); + useSessionStore.getState().restoreSessionReplica(serverId, { + agents: new Map(), + workspaces: new Map(), + emptyProjects: new Map(), + timeline: { + agentId, + items: [ + { + kind: "assistant_message", + id: "painted-item", + text: "Painted before hydration", + timestamp: new Date("2026-07-27T10:00:00.000Z"), + }, + ], + }, + }); + + void ensureAgentIsInitialized({ + serverId, + agentId, + client: client as never, + runtime, + setAgentInitializing: bindSetAgentInitializing(), + }); + + expect(runtime.requests).toEqual([ + { + serverId, + agentId, + request: { + direction: "tail", + limit: TIMELINE_FETCH_PAGE_SIZE, + projection: "projected", + }, + }, + ]); + }); + it("times out initialization after 65 seconds", async () => { vi.useFakeTimers(); const client = new FakeDaemonClient(); diff --git a/packages/app/src/hooks/use-agent-initialization.ts b/packages/app/src/hooks/use-agent-initialization.ts index ae80d352e..ade27abac 100644 --- a/packages/app/src/hooks/use-agent-initialization.ts +++ b/packages/app/src/hooks/use-agent-initialization.ts @@ -1,7 +1,7 @@ import { useCallback, useMemo } from "react"; import { useTranslation } from "react-i18next"; import type { DaemonClient } from "@getpaseo/client/internal/daemon-client"; -import { useSessionStore } from "@/stores/session-store"; +import { selectAgentTimelineState, useSessionStore } from "@/stores/session-store"; import { createInitDeferred, getInitDeferred, @@ -52,8 +52,9 @@ export function ensureAgentIsInitialized(input: EnsureAgentIsInitializedInput): } const session = useSessionStore.getState().sessions[serverId]; - const cursor = session?.agentTimelineCursor.get(agentId); - const hasAuthoritativeHistory = session?.agentAuthoritativeHistoryApplied.get(agentId) === true; + const timeline = selectAgentTimelineState(session, agentId); + const cursor = timeline.status === "synced" ? (timeline.range ?? undefined) : undefined; + const hasAuthoritativeHistory = timeline.status === "synced"; const timelineRequest = planInitialAgentTimelineSync({ cursor, hasAuthoritativeHistory }); const deferred = createInitDeferred(key, timelineRequest.direction); diff --git a/packages/app/src/hooks/use-load-older-agent-history.test.ts b/packages/app/src/hooks/use-load-older-agent-history.test.ts index 28efe17fc..292b54ab8 100644 --- a/packages/app/src/hooks/use-load-older-agent-history.test.ts +++ b/packages/app/src/hooks/use-load-older-agent-history.test.ts @@ -84,7 +84,7 @@ describe("loadOlderAgentHistory", () => { const client = createClient(); const inFlight = createInFlight(); - await loadOlderAgentHistory(agentId, { + const started = await loadOlderAgentHistory(agentId, { client, cursor: undefined, hasOlder: true, @@ -94,13 +94,14 @@ describe("loadOlderAgentHistory", () => { expect(client.calls).toEqual([]); expect(inFlight.values).toEqual([false]); + expect(started).toBe(false); }); it("no-ops when the daemon says no older history exists", async () => { const client = createClient(); const inFlight = createInFlight(); - await loadOlderAgentHistory(agentId, { + const started = await loadOlderAgentHistory(agentId, { client, cursor: someCursor, hasOlder: false, @@ -110,13 +111,14 @@ describe("loadOlderAgentHistory", () => { expect(client.calls).toEqual([]); expect(inFlight.values).toEqual([false]); + expect(started).toBe(false); }); it("no-ops when a request is already in flight", async () => { const client = createClient(); const inFlight = createInFlight(true); - await loadOlderAgentHistory(agentId, { + const started = await loadOlderAgentHistory(agentId, { client, cursor: someCursor, hasOlder: true, @@ -126,13 +128,14 @@ describe("loadOlderAgentHistory", () => { expect(client.calls).toEqual([]); expect(inFlight.values).toEqual([true]); + expect(started).toBe(true); }); it("requests the page before the current start cursor and clears in-flight on success", async () => { const client = createClient(); const inFlight = createInFlight(); - await loadOlderAgentHistory(agentId, { + const started = await loadOlderAgentHistory(agentId, { client, cursor: someCursor, hasOlder: true, @@ -152,6 +155,7 @@ describe("loadOlderAgentHistory", () => { }, ]); expect(inFlight.values).toEqual([false, true, false]); + expect(started).toBe(true); }); it("shows a panel toast, warns, and clears in-flight on failure", async () => { diff --git a/packages/app/src/hooks/use-load-older-agent-history.ts b/packages/app/src/hooks/use-load-older-agent-history.ts index 1bf8b9557..2bc3f5c9d 100644 --- a/packages/app/src/hooks/use-load-older-agent-history.ts +++ b/packages/app/src/hooks/use-load-older-agent-history.ts @@ -2,7 +2,11 @@ import { useCallback } from "react"; import { useTranslation } from "react-i18next"; import type { ToastApi } from "@/components/toast-host"; import { i18n } from "@/i18n/i18next"; -import { useSessionStore, type AgentTimelineCursorState } from "@/stores/session-store"; +import { + selectAgentTimelineState, + useSessionStore, + type AgentTimelineCursorState, +} from "@/stores/session-store"; import { planTimelineOlderFetch } from "@/timeline/timeline-sync-plan"; import { getHostRuntimeStore } from "@/runtime/host-runtime"; @@ -36,11 +40,14 @@ export interface LoadOlderAgentHistoryDeps { export async function loadOlderAgentHistory( agentId: string, deps: LoadOlderAgentHistoryDeps, -): Promise { +): Promise { const { client, cursor, hasOlder, isLoadingOlder, setInFlight, toast, logger, failedMessage } = deps; - if (!client || !cursor || !hasOlder || isLoadingOlder) { - return; + if (isLoadingOlder) { + return true; + } + if (!client || !cursor || !hasOlder) { + return false; } setInFlight(true); @@ -58,6 +65,7 @@ export async function loadOlderAgentHistory( } finally { setInFlight(false); } + return true; } export function useLoadOlderAgentHistory({ @@ -70,15 +78,17 @@ export function useLoadOlderAgentHistory({ toast?: ToastApi | null; }) { const { t } = useTranslation(); - const hasOlder = - useSessionStore((state) => state.sessions[serverId]?.agentTimelineHasOlder.get(agentId)) === - true; + const hasOlder = useSessionStore((state) => { + const timeline = selectAgentTimelineState(state.sessions[serverId], agentId); + return timeline.status === "synced" && timeline.older === "available"; + }); const isLoadingOlder = useSessionStore((state) => state.sessions[serverId]?.agentTimelineOlderFetchInFlight.get(agentId), ) === true; const progressKey = useSessionStore((state) => { - const cursor = state.sessions[serverId]?.agentTimelineCursor.get(agentId); + const timeline = selectAgentTimelineState(state.sessions[serverId], agentId); + const cursor = timeline.status === "synced" ? timeline.range : null; return cursor ? `${cursor.epoch}:${cursor.startSeq}` : null; }); const setOlderFetchInFlight = useSessionStore( @@ -99,17 +109,18 @@ export function useLoadOlderAgentHistory({ [agentId, serverId, setOlderFetchInFlight], ); - const loadOlder = useCallback(() => { + const loadOlder = useCallback(async (): Promise => { const session = useSessionStore.getState().sessions[serverId]; - void loadOlderAgentHistory(agentId, { + const timeline = selectAgentTimelineState(session, agentId); + return await loadOlderAgentHistory(agentId, { client: session?.client ? { fetchAgentTimeline: (timelineAgentId, request) => getHostRuntimeStore().fetchAgentTimeline(serverId, timelineAgentId, request), } : null, - cursor: session?.agentTimelineCursor.get(agentId), - hasOlder: session?.agentTimelineHasOlder.get(agentId) === true, + cursor: timeline.status === "synced" ? (timeline.range ?? undefined) : undefined, + hasOlder: timeline.status === "synced" && timeline.older === "available", isLoadingOlder: session?.agentTimelineOlderFetchInFlight.get(agentId) === true, setInFlight, toast, diff --git a/packages/app/src/panels/agent-panel.tsx b/packages/app/src/panels/agent-panel.tsx index 7057169f1..dbc2ab46f 100644 --- a/packages/app/src/panels/agent-panel.tsx +++ b/packages/app/src/panels/agent-panel.tsx @@ -72,7 +72,7 @@ import { WorkspaceDraftAgentTab } from "@/composer/draft/workspace-tab"; import { useCreateFlowStore } from "@/stores/create-flow-store"; import { buildDraftStoreKey, generateDraftId } from "@/stores/draft-keys"; import { usePanelStore } from "@/stores/panel-store"; -import { type Agent, useSessionStore } from "@/stores/session-store"; +import { selectAgentTimelineState, type Agent, useSessionStore } from "@/stores/session-store"; import { useWorkspaceLayoutStore } from "@/stores/workspace-layout-store"; import { buildWorkspaceTabPersistenceKey } from "@/workspace-tabs/model"; import type { Theme } from "@/styles/theme"; @@ -792,11 +792,12 @@ function ChatAgentContent({ const historySyncGeneration = useSessionStore( (state) => state.sessions[serverId]?.historySyncGeneration ?? 0, ); - const hasAppliedAuthoritativeHistory = useSessionStore((state) => + const replicaTimelineStatus = useSessionStore((state) => agentId - ? state.sessions[serverId]?.agentAuthoritativeHistoryApplied?.get(agentId) === true - : false, + ? selectAgentTimelineState(state.sessions[serverId], agentId).status + : ("cold" as const), ); + const hasAppliedAuthoritativeHistory = replicaTimelineStatus === "synced"; const agentHistorySyncGeneration = useSessionStore((state) => agentId ? (state.sessions[serverId]?.agentHistorySyncGeneration?.get(agentId) ?? -1) : -1, ); @@ -832,7 +833,8 @@ function ChatAgentContent({ kind: "idle", }); - const hasHydratedHistoryBefore = hasAppliedAuthoritativeHistory; + const hasHydratedHistoryBefore = + hasAppliedAuthoritativeHistory || replicaTimelineStatus === "painted"; const attentionController = useAgentAttentionClear({ agentId, diff --git a/packages/app/src/panels/provider-subagent-panel.tsx b/packages/app/src/panels/provider-subagent-panel.tsx index a5bac3b7b..12558a2ed 100644 --- a/packages/app/src/panels/provider-subagent-panel.tsx +++ b/packages/app/src/panels/provider-subagent-panel.tsx @@ -104,10 +104,12 @@ function ProviderSubagentPanel() { .catch(() => undefined); }, [client, serverId, supported, target.parentAgentId, target.subagentId]); - const loadOlder = useCallback(() => { - if (!client || !supported || isLoadingOlder || !timeline?.hasOlder || !timeline.epoch) return; + const loadOlder = useCallback((): boolean => { + if (!client || !supported || isLoadingOlder || !timeline?.hasOlder || !timeline.epoch) { + return false; + } const firstSeq = timeline.rows.size ? Math.min(...timeline.rows.keys()) : null; - if (firstSeq === null) return; + if (firstSeq === null) return false; setIsLoadingOlder(true); void client .fetchProviderSubagentTimeline(target.parentAgentId, target.subagentId, { @@ -121,6 +123,7 @@ function ProviderSubagentPanel() { }) .catch(() => undefined) .finally(() => setIsLoadingOlder(false)); + return true; }, [ client, isLoadingOlder, diff --git a/packages/app/src/runtime/replica-cache/index.test.ts b/packages/app/src/runtime/replica-cache/index.test.ts index 69800f03a..3ae4114d2 100644 --- a/packages/app/src/runtime/replica-cache/index.test.ts +++ b/packages/app/src/runtime/replica-cache/index.test.ts @@ -4,6 +4,7 @@ import { normalizeAgentSnapshot } from "@/utils/agent-snapshots"; import { normalizeEmptyProjectDescriptor, normalizeWorkspaceDescriptor, + selectAgentTimelineState, useSessionStore, } from "@/stores/session-store"; import type { StreamItem } from "@/types/stream"; @@ -166,13 +167,14 @@ describe("ReplicaCache", () => { expect(session?.agents.get("agent-1")?.updatedAt).toBeInstanceOf(Date); expect(session?.workspaces.get("workspace-1")?.statusEnteredAt).toBeInstanceOf(Date); expect(session?.agentStreamTail.get("agent-1")).toEqual([message("message-1", "Cached")]); - expect(session?.agentAuthoritativeHistoryApplied.get("agent-1")).toBe(true); - expect(session?.agentTimelineCursor.get("agent-1")).toEqual({ - epoch: "epoch-1", - startSeq: 1, - endSeq: 12, + expect(session?.agentAuthoritativeHistoryApplied).toEqual(new Map()); + expect(session?.agentTimelineCursor).toEqual(new Map()); + expect(session?.agentTimelineHasOlder).toEqual(new Map()); + expect(session?.agentHistorySyncGeneration).toEqual(new Map()); + expect(selectAgentTimelineState(session, "agent-1")).toEqual({ + status: "painted", + items: [message("message-1", "Cached")], }); - expect(session?.agentTimelineHasOlder.get("agent-1")).toBe(true); }); it("persists only the focused agent view with a short timeline tail", async () => { @@ -216,6 +218,13 @@ describe("ReplicaCache", () => { expect(Array.from(session?.emptyProjects.keys() ?? [])).toEqual([]); expect(Array.from(timelines?.keys() ?? [])).toEqual(["agent-2"]); expect(timelines?.get("agent-2")).toEqual(secondTimeline.slice(-50)); + + const persisted = JSON.parse(storage.values.get("@paseo:replica-cache") ?? "null") as { + version: number; + hosts: Array<{ timeline: Record | null }>; + }; + expect(persisted.version).toBe(2); + expect(Object.keys(persisted.hosts[0]?.timeline ?? {}).sort()).toEqual(["agentId", "items"]); }); it("evicts the least recently written host when the cache exceeds its byte budget", async () => { @@ -243,14 +252,38 @@ describe("ReplicaCache", () => { expect(Object.keys(useSessionStore.getState().sessions).sort()).toEqual(["host-a", "host-c"]); }); - it("drops malformed or unknown cache versions", async () => { + it("rejects version 1 cache data and overwrites it on flush", async () => { const storage = new MemoryStorage(); - storage.values.set("@paseo:replica-cache", JSON.stringify({ version: 999, hosts: [] })); + storage.values.set( + "@paseo:replica-cache", + JSON.stringify({ + version: 1, + hosts: [ + { + serverId: SERVER_ID, + agents: [], + workspaces: [], + emptyProjects: [], + timeline: { + agentId: "agent-1", + items: [], + cursor: { epoch: "poisoned", startSeq: 1, endSeq: 100 }, + hasOlder: false, + }, + }, + ], + }), + ); const cache = new ReplicaCache(storage); cache.setHosts([SERVER_ID]); await cache.restore(); + await cache.flush(); expect(useSessionStore.getState().sessions[SERVER_ID]).toBeUndefined(); + expect(JSON.parse(storage.values.get("@paseo:replica-cache") ?? "null")).toEqual({ + version: 2, + hosts: [], + }); }); }); diff --git a/packages/app/src/runtime/replica-cache/index.ts b/packages/app/src/runtime/replica-cache/index.ts index b9bfab5c0..2cad88aec 100644 --- a/packages/app/src/runtime/replica-cache/index.ts +++ b/packages/app/src/runtime/replica-cache/index.ts @@ -11,6 +11,7 @@ import { import { normalizeEmptyProjectDescriptor, normalizeWorkspaceDescriptor, + selectAgentTimelineState, useSessionStore, type Agent, type SessionReplica, @@ -37,14 +38,6 @@ const StoredAgentSchema = z.object({ const StoredTimelineSchema = z.object({ agentId: z.string(), items: z.unknown(), - cursor: z - .object({ - epoch: z.string(), - startSeq: z.number().int().nonnegative(), - endSeq: z.number().int().nonnegative(), - }) - .nullable(), - hasOlder: z.boolean(), }); const StoredHostSchema = z.object({ @@ -141,8 +134,6 @@ function deserializeTimeline(stored: StoredHost["timeline"]): SessionReplica["ti return { agentId: stored.agentId, items: decoded, - cursor: stored.cursor, - hasOlder: stored.hasOlder, }; } @@ -376,24 +367,24 @@ export class ReplicaCache { focusedAgentId ? session.messageSubmissions.get(focusedAgentId) : undefined, ), ); - const items = focusedAgentId - ? session.agentStreamTail - .get(focusedAgentId) - ?.filter( + const timelineState = focusedAgentId + ? selectAgentTimelineState(session, focusedAgentId) + : { status: "cold" as const }; + const items = + timelineState.status === "cold" + ? undefined + : timelineState.items.filter( (item) => item.kind !== "user_message" || item.messageId !== undefined || !item.clientMessageId || !localSubmissionIds.has(item.clientMessageId), - ) - : undefined; + ); const timeline = focusedAgent && items ? { agentId: focusedAgent.id, items: encodeDates(items.slice(-MAX_TIMELINE_ITEMS)), - cursor: session.agentTimelineCursor.get(focusedAgent.id) ?? null, - hasOlder: session.agentTimelineHasOlder.get(focusedAgent.id) ?? false, } : null; const stored: StoredHost = { diff --git a/packages/app/src/stores/draft-store/index.ts b/packages/app/src/stores/draft-store/index.ts index c558d2fa0..528517041 100644 --- a/packages/app/src/stores/draft-store/index.ts +++ b/packages/app/src/stores/draft-store/index.ts @@ -8,6 +8,7 @@ import { persistAttachmentFromDataUrl, persistAttachmentFromFileUri, } from "@/attachments/service"; +import { collectRetainedAttachmentIds } from "@/attachments/gc-retention"; import { useCreateFlowStore } from "@/stores/create-flow-store"; import { useSessionStore, type SessionState } from "@/stores/session-store"; import { useWorkspaceAttachmentsStore } from "@/attachments/workspace-attachments-store"; @@ -138,6 +139,9 @@ async function runAttachmentGc(): Promise { for (const id of useDraftStore.getState().collectActiveAttachmentIds()) { referencedIds.add(id); } + for (const id of collectRetainedAttachmentIds()) { + referencedIds.add(id); + } const pendingByDraftId = useCreateFlowStore.getState().pendingByDraftId; for (const pendingCreate of Object.values(pendingByDraftId)) { diff --git a/packages/app/src/stores/session-store.test.ts b/packages/app/src/stores/session-store.test.ts index 1588fd114..369097d26 100644 --- a/packages/app/src/stores/session-store.test.ts +++ b/packages/app/src/stores/session-store.test.ts @@ -5,9 +5,11 @@ import type { WorkspaceDescriptorPayload } from "@getpaseo/protocol/messages"; import { normalizeWorkspaceDescriptor, + selectAgentTimelineState, useSessionStore, type WorkspaceDescriptor, } from "./session-store"; +import type { StreamItem } from "../types/stream"; import { patchWorkspaceScripts } from "../contexts/session-workspace-scripts"; function createWorkspace( @@ -53,6 +55,54 @@ function getTestSessionReferences() { }; } +describe("agent timeline state", () => { + it("commits canonical items, range, and older availability as one synced state", () => { + initializeTestSession(); + const items: StreamItem[] = [ + { + kind: "assistant_message", + id: "canonical-row", + text: "canonical", + timestamp: new Date("2026-07-27T10:00:00.000Z"), + }, + ]; + + useSessionStore.getState().applyAgentTimelineResponseState("test-server", "agent-1", { + items, + head: [], + range: { epoch: "epoch-1", startSeq: 51, endSeq: 100 }, + older: "available", + synchronized: true, + acknowledgedClientMessageIds: [], + }); + + expect( + selectAgentTimelineState(useSessionStore.getState().sessions["test-server"], "agent-1"), + ).toEqual({ + status: "synced", + items, + range: { epoch: "epoch-1", startSeq: 51, endSeq: 100 }, + older: "available", + }); + }); + + it("represents an empty authoritative timeline without inventing a range", () => { + initializeTestSession(); + useSessionStore.getState().applyAgentTimelineResponseState("test-server", "agent-1", { + items: [], + head: [], + range: null, + older: "none", + synchronized: true, + acknowledgedClientMessageIds: [], + }); + + expect( + selectAgentTimelineState(useSessionStore.getState().sessions["test-server"], "agent-1"), + ).toEqual({ status: "synced", items: [], range: null, older: "none" }); + }); +}); + describe("normalizeWorkspaceDescriptor", () => { it("normalizes workspace scripts and invalid activity timestamps", () => { const scripts = [ diff --git a/packages/app/src/stores/session-store.ts b/packages/app/src/stores/session-store.ts index 3a5504310..1d59abdfe 100644 --- a/packages/app/src/stores/session-store.ts +++ b/packages/app/src/stores/session-store.ts @@ -336,8 +336,6 @@ export interface AgentTimelineCursorState { export interface SessionReplicaTimeline { agentId: string; items: StreamItem[]; - cursor: AgentTimelineCursorState | null; - hasOlder: boolean; } export interface SessionReplica { @@ -347,6 +345,33 @@ export interface SessionReplica { timeline: SessionReplicaTimeline | null; } +export type AgentTimelineState = + | { status: "cold" } + | { status: "painted"; items: StreamItem[] } + | { + status: "synced"; + items: StreamItem[]; + range: AgentTimelineCursorState | null; + older: "available" | "none"; + }; + +export function selectAgentTimelineState( + session: SessionState | undefined, + agentId: string, +): AgentTimelineState { + if (!session) return { status: "cold" }; + const items = session.agentStreamTail.get(agentId) ?? []; + if (session.agentAuthoritativeHistoryApplied.get(agentId) === true) { + return { + status: "synced", + items, + range: session.agentTimelineCursor.get(agentId) ?? null, + older: session.agentTimelineHasOlder.get(agentId) === true ? "available" : "none", + }; + } + return items.length > 0 ? { status: "painted", items } : { status: "cold" }; +} + export type WorkspaceRestoreStatus = "restoring" | "failed" | "needs-host-upgrade"; // Per-session state @@ -520,6 +545,18 @@ interface SessionStoreActions { agentId: string, applied: boolean, ) => void; + applyAgentTimelineResponseState: ( + serverId: string, + agentId: string, + state: { + items: StreamItem[]; + head: StreamItem[]; + range: AgentTimelineCursorState | null; + older: "available" | "none"; + synchronized: boolean; + acknowledgedClientMessageIds: string[]; + }, + ) => void; // Initializing agents setInitializingAgents: ( @@ -758,16 +795,8 @@ export const useSessionStore = create()( const session = createInitialSessionState(serverId, null); const timeline = replica.timeline; const agentStreamTail = new Map(); - const agentTimelineCursor = new Map(); - const agentTimelineHasOlder = new Map(); - const agentAuthoritativeHistoryApplied = new Map(); - const agentHistorySyncGeneration = new Map(); if (timeline) { agentStreamTail.set(timeline.agentId, timeline.items); - agentTimelineHasOlder.set(timeline.agentId, timeline.hasOlder); - agentAuthoritativeHistoryApplied.set(timeline.agentId, true); - agentHistorySyncGeneration.set(timeline.agentId, session.historySyncGeneration); - if (timeline.cursor) agentTimelineCursor.set(timeline.agentId, timeline.cursor); } const agentLastActivity = new Map(prev.agentLastActivity); for (const agent of replica.agents.values()) { @@ -784,10 +813,6 @@ export const useSessionStore = create()( workspaces: replica.workspaces, emptyProjects: replica.emptyProjects, agentStreamTail, - agentTimelineCursor, - agentTimelineHasOlder, - agentAuthoritativeHistoryApplied, - agentHistorySyncGeneration, }, }, agentLastActivity, @@ -1457,6 +1482,61 @@ export const useSessionStore = create()( }); }, + applyAgentTimelineResponseState: (serverId, agentId, state) => { + set((prev) => { + const session = prev.sessions[serverId]; + if (!session) return prev; + + const nextTail = new Map(session.agentStreamTail); + nextTail.set(agentId, state.items); + const nextHead = new Map(session.agentStreamHead); + if (state.head.length > 0) nextHead.set(agentId, state.head); + else nextHead.delete(agentId); + const nextCursor = new Map(session.agentTimelineCursor); + if (state.range) nextCursor.set(agentId, state.range); + else nextCursor.delete(agentId); + const nextHasOlder = new Map(session.agentTimelineHasOlder); + nextHasOlder.set(agentId, state.older === "available"); + const nextAuthoritative = new Map(session.agentAuthoritativeHistoryApplied); + const nextSyncGeneration = new Map(session.agentHistorySyncGeneration); + const currentSubmissions = session.messageSubmissions.get(agentId) ?? []; + const observedSubmissions = observeMessageSubmissionCanonical( + currentSubmissions, + state.acknowledgedClientMessageIds, + ); + let messageSubmissions = session.messageSubmissions; + if (observedSubmissions !== currentSubmissions) { + messageSubmissions = new Map(session.messageSubmissions); + if (observedSubmissions.length > 0) { + messageSubmissions.set(agentId, observedSubmissions); + } else { + messageSubmissions.delete(agentId); + } + } + if (state.synchronized) { + nextAuthoritative.set(agentId, true); + nextSyncGeneration.set(agentId, session.historySyncGeneration); + } + + return { + ...prev, + sessions: { + ...prev.sessions, + [serverId]: { + ...session, + agentStreamTail: nextTail, + agentStreamHead: nextHead, + agentTimelineCursor: nextCursor, + agentTimelineHasOlder: nextHasOlder, + agentAuthoritativeHistoryApplied: nextAuthoritative, + agentHistorySyncGeneration: nextSyncGeneration, + messageSubmissions, + }, + }, + }; + }); + }, + // Initializing agents setInitializingAgents: (serverId, state) => { set((prev) => { diff --git a/packages/app/src/timeline/session-stream-reducers.test.ts b/packages/app/src/timeline/session-stream-reducers.test.ts index 10eed0188..79c392a24 100644 --- a/packages/app/src/timeline/session-stream-reducers.test.ts +++ b/packages/app/src/timeline/session-stream-reducers.test.ts @@ -173,6 +173,7 @@ const baseTimelineInput: ProcessTimelineResponseInput = { hasActiveInitDeferred: false, initRequestDirection: "tail", sendingClientMessageIds: [], + hasAuthoritativeBaseline: true, }; const baseStreamInput: ProcessAgentStreamEventInput = { @@ -362,6 +363,442 @@ describe("processTimelineResponse", () => { expect(result.head).toEqual([liveThought, liveAssistant]); }); + it("keeps a newer live tool completion when bootstrap contains the running call", () => { + const callId = "toolu_live_completion"; + const liveCompletion = hydrateStreamState( + [ + { + event: { + type: "timeline", + provider: "claude", + item: makeToolCallTimelineEntry(2, callId, "completed", { + type: "read", + filePath: "/tmp/example.ts", + }).item, + } as AgentStreamEventPayload, + timestamp: new Date(2000), + timelineCursor: { epoch: "epoch-1", seq: 2 }, + }, + ], + { source: "canonical" }, + ); + + const result = processTimelineResponse({ + ...baseTimelineInput, + currentHead: liveCompletion, + hasAuthoritativeBaseline: false, + isInitializing: true, + hasActiveInitDeferred: true, + payload: { + ...baseTimelineInput.payload, + direction: "tail", + reset: false, + epoch: "epoch-1", + startCursor: { seq: 1 }, + endCursor: { seq: 1 }, + entries: [ + makeToolCallTimelineEntry(1, callId, "running", { + type: "unknown", + input: { file_path: "/tmp/example.ts" }, + output: null, + }), + ], + }, + }); + + expect(getAgentToolCalls([...result.tail, ...result.head])).toEqual([ + expect.objectContaining({ + timelineCursor: { epoch: "epoch-1", seq: 2 }, + payload: expect.objectContaining({ + data: expect.objectContaining({ callId, status: "completed" }), + }), + }), + ]); + }); + + it("keeps one newer todo state when bootstrap contains its older state", () => { + const liveTodo = hydrateStreamState( + [ + { + event: { + type: "timeline", + provider: "codex", + item: { + type: "todo", + items: [{ text: "Verify hydration", completed: true }], + }, + } as AgentStreamEventPayload, + timestamp: new Date(2000), + timelineCursor: { epoch: "epoch-1", seq: 2 }, + }, + ], + { source: "canonical" }, + ); + + const result = processTimelineResponse({ + ...baseTimelineInput, + currentHead: liveTodo, + hasAuthoritativeBaseline: false, + isInitializing: true, + hasActiveInitDeferred: true, + payload: { + ...baseTimelineInput.payload, + direction: "tail", + reset: false, + epoch: "epoch-1", + startCursor: { seq: 1 }, + endCursor: { seq: 1 }, + entries: [ + { + seqStart: 1, + seqEnd: 1, + provider: "codex", + item: { + type: "todo", + items: [{ text: "Verify hydration", completed: false }], + }, + timestamp: new Date(1000).toISOString(), + }, + ], + }, + }); + + expect([...result.tail, ...result.head].filter((item) => item.kind === "todo_list")).toEqual([ + expect.objectContaining({ + timelineCursor: { epoch: "epoch-1", seq: 2 }, + items: [{ text: "Verify hydration", completed: true }], + }), + ]); + }); + + it("keeps one completed compaction when bootstrap contains its loading state", () => { + const liveCompaction = hydrateStreamState( + [ + { + event: { + type: "timeline", + provider: "codex", + item: { type: "compaction", status: "completed", trigger: "auto", preTokens: 1200 }, + } as AgentStreamEventPayload, + timestamp: new Date(2000), + timelineCursor: { epoch: "epoch-1", seq: 2 }, + }, + ], + { source: "canonical" }, + ); + + const result = processTimelineResponse({ + ...baseTimelineInput, + currentHead: liveCompaction, + hasAuthoritativeBaseline: false, + isInitializing: true, + hasActiveInitDeferred: true, + payload: { + ...baseTimelineInput.payload, + direction: "tail", + reset: false, + epoch: "epoch-1", + startCursor: { seq: 1 }, + endCursor: { seq: 1 }, + entries: [ + { + seqStart: 1, + seqEnd: 1, + provider: "codex", + item: { type: "compaction", status: "loading", trigger: "auto" }, + timestamp: new Date(1000).toISOString(), + }, + ], + }, + }); + + expect([...result.tail, ...result.head].filter((item) => item.kind === "compaction")).toEqual([ + expect.objectContaining({ + timelineCursor: { epoch: "epoch-1", seq: 2 }, + status: "completed", + trigger: "auto", + preTokens: 1200, + }), + ]); + }); + + it("replaces a painted replica with an empty authoritative timeline", () => { + const result = processTimelineResponse({ + ...baseTimelineInput, + currentTail: [makeAssistantItem("painted replica")], + isInitializing: true, + hasActiveInitDeferred: true, + hasAuthoritativeBaseline: false, + payload: { + ...baseTimelineInput.payload, + direction: "tail", + startCursor: null, + endCursor: null, + entries: [], + hasOlder: false, + }, + }); + + expect(result.tail).toEqual([]); + expect(result.cursor).toBeNull(); + }); + + it("keeps a newer live assistant continuation when bootstrap ends on the same message", () => { + const result = processTimelineResponse({ + ...baseTimelineInput, + currentTail: [makeAssistantItem("painted replica")], + currentHead: [ + { + kind: "assistant_message", + id: "assistant-live", + messageId: "assistant-live", + text: " continuation", + timestamp: new Date(2001), + timelineCursor: { epoch: "epoch-1", seq: 101 }, + }, + ], + isInitializing: true, + hasActiveInitDeferred: true, + hasAuthoritativeBaseline: false, + payload: { + ...baseTimelineInput.payload, + direction: "tail", + startCursor: { seq: 61 }, + endCursor: { seq: 100 }, + entries: [ + { + ...makeTimelineEntry(100, "canonical prefix"), + item: { + type: "assistant_message", + text: "canonical prefix", + messageId: "assistant-live", + }, + }, + ], + }, + }); + + const assistants = [...result.tail, ...result.head].filter( + (item) => item.kind === "assistant_message", + ); + expect(assistants).toHaveLength(1); + expect(assistants[0]?.text).toBe("canonical prefix continuation"); + expect(assistants[0]?.timelineCursor).toEqual({ epoch: "epoch-1", seq: 101 }); + }); + + it("keeps assistant segments separate when a live tool sits between them", () => { + const messageId = "assistant-with-tool"; + const liveTool = hydrateStreamState( + [ + { + event: { + type: "timeline", + provider: "claude", + item: makeToolCallTimelineEntry(101, "tool-between-segments", "completed", { + type: "read", + filePath: "/tmp/example.ts", + }).item, + } as AgentStreamEventPayload, + timestamp: new Date(2101), + timelineCursor: { epoch: "epoch-1", seq: 101 }, + }, + { + event: makeAssistantTimelineEvent("after tool", messageId), + timestamp: new Date(2102), + timelineCursor: { epoch: "epoch-1", seq: 102 }, + }, + ], + { source: "canonical" }, + ); + + const result = processTimelineResponse({ + ...baseTimelineInput, + currentTail: [makeAssistantItem("painted replica")], + currentHead: liveTool, + isInitializing: true, + hasActiveInitDeferred: true, + hasAuthoritativeBaseline: false, + payload: { + ...baseTimelineInput.payload, + direction: "tail", + startCursor: { seq: 61 }, + endCursor: { seq: 100 }, + entries: [ + { + ...makeTimelineEntry(100, "before tool"), + item: { + type: "assistant_message", + text: "before tool", + messageId, + }, + }, + ], + }, + }); + + expect([...result.tail, ...result.head].map((item) => item.kind)).toEqual([ + "assistant_message", + "tool_call", + "assistant_message", + ]); + expect(getAssistantTexts([...result.tail, ...result.head])).toEqual([ + "before tool", + "after tool", + ]); + }); + + it("keeps a newer live assistant continuation without a provider message ID", () => { + const result = processTimelineResponse({ + ...baseTimelineInput, + currentTail: [makeAssistantItem("painted replica")], + currentHead: [ + { + kind: "assistant_message", + id: "assistant-live", + text: "canonical prefix continuation", + timestamp: new Date(2001), + timelineCursor: { epoch: "epoch-1", seq: 101 }, + }, + ], + isInitializing: true, + hasActiveInitDeferred: true, + hasAuthoritativeBaseline: false, + payload: { + ...baseTimelineInput.payload, + direction: "tail", + startCursor: { seq: 61 }, + endCursor: { seq: 100 }, + entries: [makeTimelineEntry(100, "canonical prefix")], + }, + }); + + const assistants = [...result.tail, ...result.head].filter( + (item) => item.kind === "assistant_message", + ); + expect(assistants).toHaveLength(1); + expect(assistants[0]?.text).toBe("canonical prefix continuation"); + expect(assistants[0]?.timelineCursor).toEqual({ epoch: "epoch-1", seq: 101 }); + }); + + it("keeps a canonical live user row newer than the bootstrap page", () => { + const live = processAgentStreamEvent({ + ...baseStreamInput, + event: { + type: "timeline", + provider: "claude", + item: { + type: "user_message", + text: "remote live prompt", + clientMessageId: "remote-client-message", + }, + }, + seq: 101, + epoch: "epoch-1", + currentTail: [makeAssistantItem("painted replica")], + hasAuthoritativeBaseline: false, + }); + expect(live.head[0]).toMatchObject({ + kind: "user_message", + timelineCursor: { epoch: "epoch-1", seq: 101 }, + }); + + const result = processTimelineResponse({ + ...baseTimelineInput, + currentTail: live.tail, + currentHead: live.head, + isInitializing: true, + hasActiveInitDeferred: true, + hasAuthoritativeBaseline: false, + payload: { + ...baseTimelineInput.payload, + direction: "tail", + startCursor: { seq: 61 }, + endCursor: { seq: 100 }, + entries: [makeTimelineEntry(100, "canonical answer")], + }, + }); + + expect(getUserTexts(result.head)).toEqual(["remote live prompt"]); + }); + + it("keeps a provider-acknowledged painted-tail prompt newer than the bootstrap page", () => { + const submitted = makeSubmittedUserMessage("submitted before bootstrap", "client-message-1"); + const live = processAgentStreamEvent({ + ...baseStreamInput, + event: { + type: "timeline", + provider: "claude", + item: { + type: "user_message", + text: "canonical presentation", + clientMessageId: "client-message-1", + messageId: "provider-message-1", + }, + }, + seq: 51, + epoch: "epoch-1", + currentTail: [makeAssistantItem("painted replica"), submitted], + currentCursor: undefined, + hasAuthoritativeBaseline: false, + }); + + const result = processTimelineResponse({ + ...baseTimelineInput, + currentTail: live.tail, + currentHead: live.head, + sendingClientMessageIds: [], + isInitializing: true, + hasActiveInitDeferred: true, + hasAuthoritativeBaseline: false, + payload: { + ...baseTimelineInput.payload, + direction: "tail", + startCursor: { seq: 11 }, + endCursor: { seq: 50 }, + entries: [makeTimelineEntry(50, "canonical answer")], + }, + }); + + expect(getUserTexts([...result.tail, ...result.head])).toEqual(["submitted before bootstrap"]); + expect(result.head[0]).toMatchObject({ + kind: "user_message", + clientMessageId: "client-message-1", + messageId: "provider-message-1", + timelineCursor: { epoch: "epoch-1", seq: 51 }, + }); + }); + + it("does not duplicate a live row already covered by the bootstrap page", () => { + const live = processAgentStreamEvent({ + ...baseStreamInput, + event: makeTimelineEvent("thinking", "reasoning"), + seq: 100, + epoch: "epoch-1", + currentTail: [makeAssistantItem("painted replica")], + hasAuthoritativeBaseline: false, + }); + + const result = processTimelineResponse({ + ...baseTimelineInput, + currentTail: live.tail, + currentHead: live.head, + isInitializing: true, + hasActiveInitDeferred: true, + hasAuthoritativeBaseline: false, + payload: { + ...baseTimelineInput.payload, + direction: "tail", + startCursor: { seq: 61 }, + endCursor: { seq: 100 }, + entries: [makeTimelineEntry(100, "thinking", "reasoning")], + }, + }); + + expect([...result.tail, ...result.head].filter((item) => item.kind === "thought")).toHaveLength( + 1, + ); + }); + it("uses the timeline entry timestamp as canonical", () => { const result = processTimelineResponse({ ...baseTimelineInput, @@ -1873,7 +2310,10 @@ describe("processTimelineResponse", () => { expect(getAssistantTexts(result.tail)).toEqual(["older chunk newer chunk"]); expect(result.tail[0]).toEqual( - expect.objectContaining({ timelineCursor: { epoch: "epoch-1", seq: 3 } }), + expect.objectContaining({ + id: "assistant-newer", + timelineCursor: { epoch: "epoch-1", seq: 3 }, + }), ); expect(result.cursor).toEqual({ epoch: "epoch-1", @@ -2417,6 +2857,61 @@ describe("processAgentStreamEvent", () => { endSeq: 1, }); }); + + it("renders a live row over painted items without creating an authoritative cursor", () => { + const result = processAgentStreamEvent({ + ...baseStreamInput, + event: makeTimelineEvent("live before bootstrap", "user_message"), + seq: 51, + epoch: "epoch-1", + currentTail: [makeAssistantItem("painted replica")], + currentCursor: undefined, + hasAuthoritativeBaseline: false, + }); + + expect(getAssistantTexts(result.tail)).toEqual(["painted replica"]); + expect(getUserTexts(result.head)).toEqual(["live before bootstrap"]); + expect(result.head[0]).toMatchObject({ + kind: "user_message", + timelineCursor: { epoch: "epoch-1", seq: 51 }, + }); + expect(result.cursorChanged).toBe(false); + expect(result.cursor).toBeNull(); + expect(result.sideEffects).toEqual([]); + }); + + it("reconciles a pre-bootstrap echo with its submitted row", () => { + const submitted = makeSubmittedUserMessage("submitted before bootstrap", "client-message-1"); + const result = processAgentStreamEvent({ + ...baseStreamInput, + event: { + type: "timeline", + provider: "claude", + item: { + type: "user_message", + text: "canonical presentation", + clientMessageId: "client-message-1", + messageId: "provider-message-1", + }, + }, + seq: 51, + epoch: "epoch-1", + currentTail: [makeAssistantItem("painted replica"), submitted], + currentCursor: undefined, + hasAuthoritativeBaseline: false, + }); + + const users = [...result.tail, ...result.head].filter((item) => item.kind === "user_message"); + expect(users).toEqual([ + expect.objectContaining({ + id: "client-message-1", + clientMessageId: "client-message-1", + messageId: "provider-message-1", + text: "submitted before bootstrap", + timelineCursor: { epoch: "epoch-1", seq: 51 }, + }), + ]); + }); }); describe("processAgentStreamEvents", () => { diff --git a/packages/app/src/timeline/session-stream-reducers.ts b/packages/app/src/timeline/session-stream-reducers.ts index 2690bfb50..1cd26517f 100644 --- a/packages/app/src/timeline/session-stream-reducers.ts +++ b/packages/app/src/timeline/session-stream-reducers.ts @@ -1,5 +1,5 @@ import type { AgentStreamEventPayload } from "@getpaseo/protocol/messages"; -import { useSessionStore } from "@/stores/session-store"; +import { selectAgentTimelineState, useSessionStore } from "@/stores/session-store"; import type { AssistantMessageItem, StreamItem } from "@/types/stream"; import { applyStreamEvent, @@ -89,6 +89,7 @@ export interface ProcessTimelineResponseInput { hasActiveInitDeferred: boolean; initRequestDirection: InitRequestDirection; sendingClientMessageIds: readonly string[]; + hasAuthoritativeBaseline: boolean; } export interface ProcessTimelineResponseOutput { @@ -233,6 +234,10 @@ function applyTimelineReplacePath(args: { previousHead: currentHead, sendingClientMessageIds, preserveLiveHead, + canonicalCoverage: { + epoch: payload.epoch, + endSeq: payload.endCursor?.seq ?? null, + }, }); const cursor: TimelineCursor | null = payload.startCursor && payload.endCursor @@ -255,7 +260,6 @@ function applyTimelineReplacePath(args: { acknowledgedClientMessageIds, }; } - interface IncrementalAcceptResult { acceptedUnits: TimelineUnit[]; cursor: TimelineCursor | undefined; @@ -406,16 +410,15 @@ function mergePrependedCanonicalTail(olderTail: StreamItem[], currentTail: Strea return [...olderTail, ...currentTail]; } - return [ - ...olderTail.slice(0, -1), - { - ...olderLast, - text: `${olderLast.text}${currentFirst.text}`, - timestamp: currentFirst.timestamp, - ...(currentFirst.timelineCursor ? { timelineCursor: currentFirst.timelineCursor } : {}), - }, - ...currentTail.slice(1), - ]; + const mergedAssistant: AssistantMessageItem = { + ...currentFirst, + text: `${olderLast.text}${currentFirst.text}`, + }; + if (mergedAssistant.messageId === undefined && olderLast.messageId !== undefined) { + mergedAssistant.messageId = olderLast.messageId; + } + + return [...olderTail.slice(0, -1), mergedAssistant, ...currentTail.slice(1)]; } function replaceLiveAssistantWithProjectedText(params: { @@ -820,6 +823,7 @@ export function processTimelineResponse( hasActiveInitDeferred, initRequestDirection, sendingClientMessageIds, + hasAuthoritativeBaseline, } = input; // ------------------------------------------------------------------ @@ -891,7 +895,8 @@ export function processTimelineResponse( currentTail, currentHead, sendingClientMessageIds, - preserveLiveHead: currentCursor?.epoch === payload.epoch, + preserveLiveHead: + currentCursor?.epoch === payload.epoch || (!payload.reset && !hasAuthoritativeBaseline), toHydratedEvents, }) : applyTimelineIncrementalPath({ @@ -955,6 +960,7 @@ export interface ProcessAgentStreamEventInput { currentTail: StreamItem[]; currentHead: StreamItem[]; currentCursor: TimelineCursor | undefined; + hasAuthoritativeBaseline?: boolean; timestamp: Date; } @@ -989,6 +995,7 @@ export interface ProcessAgentStreamEventsInput { currentTail: StreamItem[]; currentHead: StreamItem[]; currentCursor: TimelineCursor | undefined; + hasAuthoritativeBaseline?: boolean; } export type AgentStreamReducerSnapshot = Omit; @@ -1017,8 +1024,9 @@ function processTimelineSequencingGate(input: { seq: number | undefined; epoch: string | undefined; currentCursor: TimelineCursor | undefined; + hasAuthoritativeBaseline: boolean; }): TimelineSequencingGateResult { - const { event, seq, epoch, currentCursor } = input; + const { event, seq, epoch, currentCursor, hasAuthoritativeBaseline } = input; const base: TimelineSequencingGateResult = { shouldApplyStreamEvent: true, nextTimelineCursor: null, @@ -1029,6 +1037,9 @@ function processTimelineSequencingGate(input: { if (event.type !== "timeline" || typeof seq !== "number" || typeof epoch !== "string") { return base; } + if (!hasAuthoritativeBaseline) { + return base; + } const decision = classifySessionTimelineSeq({ cursor: currentCursor ? { epoch: currentCursor.epoch, endSeq: currentCursor.endSeq } : null, @@ -1085,9 +1096,24 @@ function processTimelineSequencingGate(input: { export function processAgentStreamEvent( input: ProcessAgentStreamEventInput, ): ProcessAgentStreamEventOutput { - const { event, seq, epoch, currentTail, currentHead, currentCursor, timestamp } = input; + const { + event, + seq, + epoch, + currentTail, + currentHead, + currentCursor, + timestamp, + hasAuthoritativeBaseline = true, + } = input; - const sequencing = processTimelineSequencingGate({ event, seq, epoch, currentCursor }); + const sequencing = processTimelineSequencingGate({ + event, + seq, + epoch, + currentCursor, + hasAuthoritativeBaseline, + }); const timelineCursor = event.type === "timeline" && seq !== undefined && epoch !== undefined ? { epoch, seq } @@ -1096,30 +1122,61 @@ export function processAgentStreamEvent( // ------------------------------------------------------------------ // Apply stream event to tail/head // ------------------------------------------------------------------ - const applied = sequencing.shouldApplyStreamEvent - ? applyStreamEvent({ - tail: sequencing.resetLiveTimeline ? [] : currentTail, - head: sequencing.resetLiveTimeline ? [] : currentHead, + let streamResult: ReturnType; + if (!sequencing.shouldApplyStreamEvent) { + streamResult = { + tail: currentTail, + head: currentHead, + changedTail: false, + changedHead: false, + }; + } else if (!hasAuthoritativeBaseline) { + if (event.type === "timeline" && event.item.type === "user_message") { + streamResult = applyStreamEvent({ + tail: currentTail, + head: currentHead, event, timestamp, source: "live", timelineCursor, - }) - : { + unmatchedUserMessageInsert: "head", + }); + } else { + const overlay = applyStreamEvent({ + tail: currentHead, + head: [], + event, + timestamp, + source: "live", + timelineCursor, + }); + streamResult = { tail: currentTail, - head: currentHead, + head: [...overlay.tail, ...overlay.head], changedTail: false, - changedHead: false, + changedHead: overlay.changedTail || overlay.changedHead, }; + } + } else { + streamResult = applyStreamEvent({ + tail: sequencing.resetLiveTimeline ? [] : currentTail, + head: sequencing.resetLiveTimeline ? [] : currentHead, + event, + timestamp, + source: "live", + timelineCursor, + }); + } + const { tail, head, changedTail, changedHead } = streamResult; return { - tail: applied.tail, - head: applied.head, - changedTail: applied.changedTail, - changedHead: applied.changedHead, + tail, + head, + changedTail, + changedHead, cursor: sequencing.nextTimelineCursor, cursorChanged: sequencing.cursorChanged, - acknowledgedClientMessageIds: applied.acknowledgedClientMessageIds ?? [], + acknowledgedClientMessageIds: streamResult.acknowledgedClientMessageIds ?? [], sideEffects: sequencing.sideEffects, }; } @@ -1144,6 +1201,7 @@ export function processAgentStreamEvents( currentTail: tail, currentHead: head, currentCursor: cursor, + hasAuthoritativeBaseline: input.hasAuthoritativeBaseline, timestamp: reducerEvent.timestamp, }); @@ -1281,10 +1339,12 @@ export function createSessionAgentStreamReducerQueue( return createAgentStreamReducerQueue({ getSnapshot: (agentId) => { const session = useSessionStore.getState().sessions[serverId]; + const timeline = selectAgentTimelineState(session, agentId); return { - currentTail: session?.agentStreamTail.get(agentId) ?? [], + currentTail: timeline.status === "cold" ? [] : timeline.items, currentHead: session?.agentStreamHead.get(agentId) ?? [], - currentCursor: session?.agentTimelineCursor.get(agentId), + currentCursor: timeline.status === "synced" ? (timeline.range ?? undefined) : undefined, + hasAuthoritativeBaseline: timeline.status === "synced", }; }, commit: (agentId, result, events) => { diff --git a/packages/app/src/types/stream.test.ts b/packages/app/src/types/stream.test.ts index 00d73b7f6..e3de58f42 100644 --- a/packages/app/src/types/stream.test.ts +++ b/packages/app/src/types/stream.test.ts @@ -1350,7 +1350,10 @@ describe("turn lifecycle events", () => { }, }, new Date("2025-01-01T15:03:11Z"), - { source: "canonical" }, + { + source: "canonical", + timelineCursor: { epoch: "epoch-1", seq: 42 }, + }, ); const userMessages = state.filter((item) => item.kind === "user_message"); @@ -1361,6 +1364,7 @@ describe("turn lifecycle events", () => { assert.strictEqual(userMessage.messageId, "provider-owned-canonical"); assert.strictEqual(userMessage.text, "Analyze this"); assert.strictEqual(userMessage.timestamp.getTime(), submittedTimestamp.getTime()); + assert.deepStrictEqual(userMessage.timelineCursor, { epoch: "epoch-1", seq: 42 }); assert.deepStrictEqual(userMessage.images, [image]); assert.deepStrictEqual(userMessage.attachments, [attachment]); }); diff --git a/packages/app/src/types/stream.ts b/packages/app/src/types/stream.ts index 8b66ab80b..45a712884 100644 --- a/packages/app/src/types/stream.ts +++ b/packages/app/src/types/stream.ts @@ -88,6 +88,7 @@ export interface UserMessageItem { id: string; clientMessageId?: string; messageId?: string; + timelineCursor?: TimelinePosition; text: string; timestamp: Date; images?: UserMessageImageAttachment[]; @@ -98,6 +99,7 @@ export interface UserMessageInput { id?: string; clientMessageId?: string; messageId?: string; + timelineCursor?: TimelinePosition; text: string; timestamp: Date; images?: UserMessageImageAttachment[]; @@ -114,6 +116,7 @@ export function createUserMessage(input: UserMessageInput): UserMessageItem { id, ...(input.clientMessageId ? { clientMessageId: input.clientMessageId } : {}), ...(input.messageId ? { messageId: input.messageId } : {}), + ...(input.timelineCursor ? { timelineCursor: input.timelineCursor } : {}), text: input.text, timestamp: input.timestamp, ...(input.images && input.images.length > 0 ? { images: input.images } : {}), @@ -236,11 +239,13 @@ function produceUserMessage( ...presentation, clientMessageId: incoming.clientMessageId ?? existing.clientMessageId, messageId: incoming.messageId ?? existing.messageId, + timelineCursor: incoming.timelineCursor ?? existing.timelineCursor, }); if ( existing.id === merged.id && existing.clientMessageId === merged.clientMessageId && existing.messageId === merged.messageId && + existing.timelineCursor === merged.timelineCursor && existing.text === merged.text && existing.timestamp === merged.timestamp && existing.images === merged.images && @@ -392,6 +397,7 @@ export interface CanonicalStreamReplacementInput { previousHead: StreamItem[]; sendingClientMessageIds: readonly string[]; preserveLiveHead: boolean; + canonicalCoverage: { epoch: string; endSeq: number | null }; } export interface CanonicalStreamReplacementResult { @@ -400,44 +406,173 @@ export interface CanonicalStreamReplacementResult { acknowledgedClientMessageIds: string[]; } +function isAfterCanonicalCoverage( + position: TimelinePosition, + coverage: CanonicalStreamReplacementInput["canonicalCoverage"], +): boolean { + return ( + position.epoch === coverage.epoch && + (coverage.endSeq === null || position.seq > coverage.endSeq) + ); +} + function removeUserMessageAt(items: UserMessageItem[], index: number): UserMessageItem[] { return [...items.slice(0, index), ...items.slice(index + 1)]; } +function mergeRetainedLifecycleItem(tail: StreamItem[], retained: StreamItem): StreamItem[] | null { + if (!retained.timelineCursor) { + return null; + } + if (isAgentToolCallItem(retained)) { + const tailIndex = findExistingAgentToolCallIndex(tail, retained.payload.data.callId); + const existing = tail[tailIndex]; + if (tailIndex < 0 || !existing || !isAgentToolCallItem(existing)) { + return null; + } + const next = [...tail]; + next[tailIndex] = mergeAgentToolCallItem( + existing, + retained.payload.data, + retained.timestamp, + retained.timelineCursor, + ); + return next; + } + if (retained.kind === "todo_list") { + const tailIndex = tail.length - 1; + const existing = tail[tailIndex]; + if (!existing || existing.kind !== "todo_list" || existing.provider !== retained.provider) { + return null; + } + const next = [...tail]; + next[tailIndex] = { + ...existing, + timelineCursor: retained.timelineCursor, + timestamp: retained.timestamp, + items: retained.items, + }; + return next; + } + if (retained.kind === "compaction" && retained.status === "completed") { + const tailIndex = tail.findIndex( + (item) => item.kind === "compaction" && item.status === "loading", + ); + const existing = tail[tailIndex]; + if (tailIndex < 0 || !existing || existing.kind !== "compaction") { + return null; + } + const next = [...tail]; + next[tailIndex] = { + ...existing, + timelineCursor: retained.timelineCursor, + status: "completed", + trigger: retained.trigger ?? existing.trigger, + preTokens: retained.preTokens ?? existing.preTokens, + }; + return next; + } + return null; +} + +function reconcileReplacementHeadAgainstTail( + tail: StreamItem[], + retainedHead: StreamItem[], +): { tail: StreamItem[]; head: StreamItem[] } { + let reconciledTail = tail; + const reconciledHeadIndexes = new Set(); + for (const [headIndex, item] of retainedHead.entries()) { + const nextTail = mergeRetainedLifecycleItem(reconciledTail, item); + if (!nextTail) { + continue; + } + reconciledTail = nextTail; + reconciledHeadIndexes.add(headIndex); + } + + const tailIds = new Set(reconciledTail.map((item) => item.id)); + return { + tail: reconciledTail, + head: retainedHead.filter( + (item, index) => + !reconciledHeadIndexes.has(index) && + (item.kind === "assistant_message" || !tailIds.has(item.id)), + ), + }; +} + function preserveReplacementHead( tail: StreamItem[], currentHead: StreamItem[], preserveLiveHead: boolean, sendingClientMessageIds: ReadonlySet, + canonicalCoverage: CanonicalStreamReplacementInput["canonicalCoverage"], ): CanonicalStreamReplacementResult { + const canonicalTailAssistant = tail.at(-1); const retainedHead = preserveLiveHead - ? currentHead + ? currentHead.filter( + (item) => + !item.timelineCursor || + isAfterCanonicalCoverage(item.timelineCursor, canonicalCoverage) || + (item.kind === "assistant_message" && + canonicalTailAssistant?.kind === "assistant_message" && + item.text.startsWith(canonicalTailAssistant.text)), + ) : currentHead.filter( (item) => item.kind === "user_message" && item.clientMessageId !== undefined && sendingClientMessageIds.has(item.clientMessageId), ); - const tailIds = new Set(tail.map((item) => item.id)); - const unreconciledHead = retainedHead.filter( - (item) => item.kind === "assistant_message" || !tailIds.has(item.id), - ); - const liveAssistantIndex = unreconciledHead.findLastIndex( - (item) => item.kind === "assistant_message", + const { tail: reconciledTail, head: unreconciledHead } = reconcileReplacementHeadAgainstTail( + tail, + retainedHead, ); + const liveAssistantIndex = unreconciledHead[0]?.kind === "assistant_message" ? 0 : -1; if (liveAssistantIndex < 0) { - return { tail, head: unreconciledHead, acknowledgedClientMessageIds: [] }; + return { tail: reconciledTail, head: unreconciledHead, acknowledgedClientMessageIds: [] }; } const liveAssistant = unreconciledHead[liveAssistantIndex]; - const tailAssistant = tail.at(-1); + const tailAssistant = reconciledTail.at(-1); if ( liveAssistant.kind !== "assistant_message" || !tailAssistant || - tailAssistant.kind !== "assistant_message" || - !liveAssistant.text.startsWith(tailAssistant.text) + tailAssistant.kind !== "assistant_message" ) { - return { tail, head: unreconciledHead, acknowledgedClientMessageIds: [] }; + return { tail: reconciledTail, head: unreconciledHead, acknowledgedClientMessageIds: [] }; + } + + const hasNewerCursor = + liveAssistant.timelineCursor !== undefined && + tailAssistant.timelineCursor !== undefined && + liveAssistant.timelineCursor.epoch === tailAssistant.timelineCursor.epoch && + liveAssistant.timelineCursor.seq > tailAssistant.timelineCursor.seq; + const hasMatchingProviderMessageId = + liveAssistant.messageId !== undefined && liveAssistant.messageId === tailAssistant.messageId; + const hasIdlessTextContinuation = + liveAssistant.messageId === undefined && + tailAssistant.messageId === undefined && + liveAssistant.text.startsWith(tailAssistant.text); + const isNewerContinuation = + hasNewerCursor && (hasMatchingProviderMessageId || hasIdlessTextContinuation); + if (isNewerContinuation) { + const text = liveAssistant.text.startsWith(tailAssistant.text) + ? liveAssistant.text + : `${tailAssistant.text}${liveAssistant.text}`; + const head = [ + ...unreconciledHead.slice(0, liveAssistantIndex), + { ...liveAssistant, text }, + ...unreconciledHead.slice(liveAssistantIndex + 1), + ]; + return { + tail: reconciledTail.slice(0, -1), + head, + acknowledgedClientMessageIds: [], + }; + } + if (!liveAssistant.text.startsWith(tailAssistant.text)) { + return { tail: reconciledTail, head: unreconciledHead, acknowledgedClientMessageIds: [] }; } const head = [ @@ -445,7 +580,11 @@ function preserveReplacementHead( { ...liveAssistant, text: tailAssistant.text }, ...unreconciledHead.slice(liveAssistantIndex + 1), ]; - return { tail: tail.slice(0, -1), head, acknowledgedClientMessageIds: [] }; + return { + tail: reconciledTail.slice(0, -1), + head, + acknowledgedClientMessageIds: [], + }; } export function replaceWithCanonicalStream( @@ -498,16 +637,25 @@ export function replaceWithCanonicalStream( nextTail.push(item); } + const retainedTailMessages: UserMessageItem[] = []; for (const local of unmatchedTailMessages) { - if (!local.clientMessageId || !sendingClientMessageIds.has(local.clientMessageId)) { - continue; + if (local.clientMessageId && sendingClientMessageIds.has(local.clientMessageId)) { + nextTail.push(local); + } else if ( + input.preserveLiveHead && + local.timelineCursor && + isAfterCanonicalCoverage(local.timelineCursor, input.canonicalCoverage) + ) { + retainedTailMessages.push(local); } - nextTail.push(local); } + nextHead = [...retainedTailMessages, ...nextHead]; nextHead = nextHead.filter((item) => { if (item.kind !== "user_message" || !item.clientMessageId) return true; - return sendingClientMessageIds.has(item.clientMessageId); + if (sendingClientMessageIds.has(item.clientMessageId)) return true; + if (!input.preserveLiveHead || !item.timelineCursor) return false; + return isAfterCanonicalCoverage(item.timelineCursor, input.canonicalCoverage); }); const replacement = preserveReplacementHead( @@ -515,6 +663,7 @@ export function replaceWithCanonicalStream( nextHead, input.preserveLiveHead, sendingClientMessageIds, + input.canonicalCoverage, ); return { ...replacement, @@ -543,6 +692,7 @@ export type ThoughtStatus = "loading" | "ready"; export interface ThoughtItem { kind: "thought"; id: string; + timelineCursor?: TimelinePosition; text: string; timestamp: Date; status: ThoughtStatus; @@ -577,6 +727,7 @@ export type ToolCallPayload = export interface ToolCallItem { kind: "tool_call"; id: string; + timelineCursor?: TimelinePosition; timestamp: Date; payload: ToolCallPayload; } @@ -594,6 +745,7 @@ type ActivityLogType = "system" | "info" | "success" | "error"; export interface ActivityLogItem { kind: "activity_log"; id: string; + timelineCursor?: TimelinePosition; timestamp: Date; activityType: ActivityLogType; message: string; @@ -603,6 +755,7 @@ export interface ActivityLogItem { export interface CompactionItem { kind: "compaction"; id: string; + timelineCursor?: TimelinePosition; timestamp: Date; status: "loading" | "completed"; trigger?: "auto" | "manual"; @@ -617,6 +770,7 @@ export interface TodoEntry { export interface TodoListItem { kind: "todo_list"; id: string; + timelineCursor?: TimelinePosition; timestamp: Date; provider: AgentProvider; items: TodoEntry[]; @@ -675,6 +829,7 @@ function appendUserMessage( _source: StreamUpdateSource, messageId?: string, clientMessageId?: string, + timelineCursor?: TimelinePosition, ): StreamItem[] { const { chunk, hasContent } = normalizeChunk(text); if (!hasContent) { @@ -686,6 +841,7 @@ function appendUserMessage( id: messageId ?? createUniqueTimelineId(state, "user", chunkSeed, timestamp), clientMessageId, messageId, + timelineCursor, text: chunk, timestamp, }); @@ -756,7 +912,12 @@ function appendAssistantMessage( return [...state, item]; } -function appendThought(state: StreamItem[], text: string, timestamp: Date): StreamItem[] { +function appendThought( + state: StreamItem[], + text: string, + timestamp: Date, + timelineCursor?: TimelinePosition, +): StreamItem[] { const { chunk, hasContent } = normalizeChunk(text); if (!chunk) { return state; @@ -766,6 +927,7 @@ function appendThought(state: StreamItem[], text: string, timestamp: Date): Stre if (last && last.kind === "thought") { const updated: ThoughtItem = { ...last, + ...(timelineCursor ? { timelineCursor } : {}), text: `${last.text}${chunk}`, timestamp, status: "loading", @@ -781,6 +943,7 @@ function appendThought(state: StreamItem[], text: string, timestamp: Date): Stre const item: ThoughtItem = { kind: "thought", id: createUniqueTimelineId(state, "thought", idSeed, timestamp), + ...(timelineCursor ? { timelineCursor } : {}), text: chunk, timestamp, status: "loading", @@ -916,6 +1079,7 @@ export function mergeAgentToolCallItem( existing: AgentToolCallItem, data: AgentToolCallData, timestamp: Date, + timelineCursor?: TimelinePosition, ): AgentToolCallItem { const mergedStatus = mergeAgentToolCallStatus(existing.payload.data.status, data.status); const mergedError = @@ -927,6 +1091,7 @@ export function mergeAgentToolCallItem( return { ...existing, + ...(timelineCursor ? { timelineCursor } : {}), timestamp, payload: { source: "agent", @@ -946,6 +1111,7 @@ function appendAgentToolCall( state: StreamItem[], data: AgentToolCallData, timestamp: Date, + timelineCursor?: TimelinePosition, ): StreamItem[] { const existingIndex = findExistingAgentToolCallIndex(state, data.callId); @@ -954,7 +1120,7 @@ function appendAgentToolCall( if (!existing || !isAgentToolCallItem(existing)) { return state; } - const merged = mergeAgentToolCallItem(existing, data, timestamp); + const merged = mergeAgentToolCallItem(existing, data, timestamp, timelineCursor); if ( merged.payload.data.provider === existing.payload.data.provider && @@ -963,7 +1129,8 @@ function appendAgentToolCall( merged.payload.data.status === existing.payload.data.status && merged.payload.data.error === existing.payload.data.error && merged.payload.data.detail === existing.payload.data.detail && - merged.payload.data.metadata === existing.payload.data.metadata + merged.payload.data.metadata === existing.payload.data.metadata && + merged.timelineCursor === existing.timelineCursor ) { return state; } @@ -976,6 +1143,7 @@ function appendAgentToolCall( const item: ToolCallItem = { kind: "tool_call", id: `agent_tool_${data.callId}`, + ...(timelineCursor ? { timelineCursor } : {}), timestamp, payload: { source: "agent", @@ -1004,6 +1172,7 @@ function appendTodoList( provider: AgentProvider, items: TodoEntry[], timestamp: Date, + timelineCursor?: TimelinePosition, ): StreamItem[] { const normalizedItems = items.map((item) => ({ text: item.text, @@ -1015,6 +1184,7 @@ function appendTodoList( const next = [...state]; const updated: TodoListItem = { ...lastItem, + ...(timelineCursor ? { timelineCursor } : {}), items: normalizedItems, timestamp, }; @@ -1028,6 +1198,7 @@ function appendTodoList( const entry: TodoListItem = { kind: "todo_list", id: entryId, + ...(timelineCursor ? { timelineCursor } : {}), timestamp, provider, items: normalizedItems, @@ -1044,6 +1215,7 @@ function reduceTimelineToolCall( { type: "tool_call" } >, timestamp: Date, + timelineCursor?: TimelinePosition, ): StreamItem[] { const normalizedToolName = item.name .trim() @@ -1066,6 +1238,7 @@ function reduceTimelineToolCall( event.provider, tasks.map((entry) => ({ text: entry.text, completed: entry.completed })), timestamp, + timelineCursor, ); } @@ -1076,6 +1249,7 @@ function reduceTimelineToolCall( event.provider, tasks.map((entry) => ({ text: entry.text, completed: entry.completed })), timestamp, + timelineCursor, ); } @@ -1091,6 +1265,7 @@ function reduceTimelineToolCall( metadata: item.metadata, }, timestamp, + timelineCursor, ); } @@ -1101,6 +1276,7 @@ function reduceTimelineCompaction( { type: "compaction" } >, timestamp: Date, + timelineCursor?: TimelinePosition, ): StreamItem[] { if (item.status === "completed") { const loadingIdx = state.findIndex((s) => s.kind === "compaction" && s.status === "loading"); @@ -1108,6 +1284,7 @@ function reduceTimelineCompaction( if (loadingIdx >= 0 && existing && existing.kind === "compaction") { const updated: CompactionItem = { ...existing, + ...(timelineCursor ? { timelineCursor } : {}), status: "completed", trigger: item.trigger ?? existing.trigger, preTokens: item.preTokens ?? existing.preTokens, @@ -1121,6 +1298,7 @@ function reduceTimelineCompaction( const compaction: CompactionItem = { kind: "compaction", id: createTimelineId("compaction", item.status, timestamp), + ...(timelineCursor ? { timelineCursor } : {}), timestamp, status: item.status, trigger: item.trigger, @@ -1148,6 +1326,7 @@ function reduceTimelineEvent( source, item.messageId, item.clientMessageId, + timelineCursor, ), ); case "assistant_message": @@ -1163,9 +1342,11 @@ function reduceTimelineEvent( ), ); case "reasoning": - return appendThought(state, item.text, timestamp); + return appendThought(state, item.text, timestamp, timelineCursor); case "tool_call": - return finalizeActiveThoughts(reduceTimelineToolCall(state, event, item, timestamp)); + return finalizeActiveThoughts( + reduceTimelineToolCall(state, event, item, timestamp, timelineCursor), + ); case "todo": { if (event.provider === "claude") { return finalizeActiveThoughts(state); @@ -1174,12 +1355,15 @@ function reduceTimelineEvent( text: todo.text, completed: todo.completed, })); - return finalizeActiveThoughts(appendTodoList(state, event.provider, items, timestamp)); + return finalizeActiveThoughts( + appendTodoList(state, event.provider, items, timestamp, timelineCursor), + ); } case "error": { const activity: ActivityLogItem = { kind: "activity_log", id: createTimelineId("error", item.message ?? "", timestamp), + ...(timelineCursor ? { timelineCursor } : {}), timestamp, activityType: "error", message: item.message ?? "Unknown error", @@ -1187,7 +1371,9 @@ function reduceTimelineEvent( return finalizeActiveThoughts(appendActivityLog(state, activity)); } case "compaction": - return finalizeActiveThoughts(reduceTimelineCompaction(state, item, timestamp)); + return finalizeActiveThoughts( + reduceTimelineCompaction(state, item, timestamp, timelineCursor), + ); default: return state; } @@ -1510,8 +1696,10 @@ function applyCanonicalUserMessageEvent(params: { head: StreamItem[]; event: AgentStreamEventPayload; timestamp: Date; + timelineCursor?: TimelinePosition; + unmatchedInsert?: "tail" | "head"; }): ApplyStreamEventResult | null { - const { tail, head, event, timestamp } = params; + const { tail, head, event, timestamp, timelineCursor, unmatchedInsert = "tail" } = params; if (event.type !== "timeline" || event.item.type !== "user_message") return null; const normalized = normalizeChunk(event.item.text); @@ -1523,9 +1711,29 @@ function applyCanonicalUserMessageEvent(params: { createUniqueTimelineId([...tail, ...head], "user", normalized.chunk.trim(), timestamp), messageId: event.item.messageId, clientMessageId: event.item.clientMessageId, + timelineCursor, text: normalized.chunk, timestamp, }); + if (unmatchedInsert === "head") { + const reconciled = upsertUserMessageAcrossStream({ + tail, + head, + message: canonical, + insert: normalized.hasContent ? "head" : "none", + presentation: "existing", + }); + return { + tail: reconciled.tail, + head: reconciled.head, + changedTail: reconciled.changedTail, + changedHead: reconciled.changedHead, + acknowledgedClientMessageIds: + reconciled.location?.matched && reconciled.location.message.clientMessageId + ? [reconciled.location.message.clientMessageId] + : [], + }; + } const reconciled = placeCanonicalUserMessageAtTail(flushedTail, canonical, normalized.hasContent); return { tail: reconciled.items, @@ -1557,9 +1765,17 @@ export function applyStreamEvent(params: { timestamp: Date; source?: StreamUpdateSource; timelineCursor?: TimelinePosition; + unmatchedUserMessageInsert?: "tail" | "head"; }): ApplyStreamEventResult { const { tail, head, event, timestamp } = params; - const canonicalUserResult = applyCanonicalUserMessageEvent({ tail, head, event, timestamp }); + const canonicalUserResult = applyCanonicalUserMessageEvent({ + tail, + head, + event, + timestamp, + timelineCursor: params.timelineCursor, + unmatchedInsert: params.unmatchedUserMessageInsert, + }); if (canonicalUserResult) return canonicalUserResult; const source = params.source ?? "live"; let nextTail = tail; diff --git a/packages/app/src/utils/assistant-image-metadata.test.ts b/packages/app/src/utils/assistant-image-metadata.test.ts index e6138dc71..d4b99655f 100644 --- a/packages/app/src/utils/assistant-image-metadata.test.ts +++ b/packages/app/src/utils/assistant-image-metadata.test.ts @@ -3,7 +3,6 @@ import { clearAssistantImageMetadataCache, estimateAssistantMessageHeightFromCache, extractAssistantImageSources, - getAssistantImageLoadStateFromMetadata, getAssistantImageMetadata, setAssistantImageMetadata, } from "./assistant-image-metadata"; @@ -42,23 +41,6 @@ describe("assistant image metadata", () => { }); }); - it("maps missing metadata to the image loading state", () => { - expect(getAssistantImageLoadStateFromMetadata(null)).toEqual({ status: "loading" }); - }); - - it("maps cached metadata to the image ready state", () => { - expect( - getAssistantImageLoadStateFromMetadata({ - width: 900, - height: 1600, - aspectRatio: 9 / 16, - }), - ).toEqual({ - status: "ready", - aspectRatio: 9 / 16, - }); - }); - it("estimates assistant message height from cached image metadata", () => { setAssistantImageMetadata( { diff --git a/packages/app/src/utils/assistant-image-metadata.ts b/packages/app/src/utils/assistant-image-metadata.ts index b11d26baf..9f6e159b0 100644 --- a/packages/app/src/utils/assistant-image-metadata.ts +++ b/packages/app/src/utils/assistant-image-metadata.ts @@ -8,11 +8,6 @@ export interface AssistantImageMetadata { aspectRatio: number; } -export type AssistantImageLoadState = - | { status: "loading" } - | { status: "ready"; aspectRatio: number } - | { status: "error" }; - const assistantImageMetadataCache = new Map(); const assistantImageParseCache = new Map(); const ASSISTANT_IMAGE_METADATA_CACHE_LIMIT = 500; @@ -131,15 +126,6 @@ export function getAssistantImageMetadata(input: { return null; } -export function getAssistantImageLoadStateFromMetadata( - metadata: AssistantImageMetadata | null, -): AssistantImageLoadState { - if (!metadata) { - return { status: "loading" }; - } - return { status: "ready", aspectRatio: metadata.aspectRatio }; -} - export function setAssistantImageMetadata( input: { source: string; diff --git a/packages/server/src/server/agent/providers/mock-load-test-agent.test.ts b/packages/server/src/server/agent/providers/mock-load-test-agent.test.ts index cacdda38e..c32d9a25d 100644 --- a/packages/server/src/server/agent/providers/mock-load-test-agent.test.ts +++ b/packages/server/src/server/agent/providers/mock-load-test-agent.test.ts @@ -284,6 +284,39 @@ describe("MockLoadTestAgentClient", () => { unsubscribe(); }); + test("emits a settled assistant Markdown image path selected by prompt", async () => { + vi.useFakeTimers(); + const client = new MockLoadTestAgentClient(); + const session = await client.createSession({ + provider: "mock", + cwd: process.cwd(), + model: "ten-second-stream", + }); + const events: AgentStreamEvent[] = []; + const unsubscribe = session.subscribe((event) => events.push(event)); + const markdown = "![Fixture image](screenshots/fixture.png)"; + + const resultPromise = session.run(`Emit settled assistant image Markdown: ${markdown}`); + await vi.advanceTimersByTimeAsync(0); + + expect( + events.flatMap((event): AgentTimelineItem[] => + event.type === "timeline" && event.item.type === "assistant_message" ? [event.item] : [], + ), + ).toEqual([ + expect.objectContaining({ + type: "assistant_message", + text: markdown, + }), + ]); + await expect(resultPromise).resolves.toMatchObject({ + sessionId: session.id, + finalText: markdown, + canceled: false, + }); + unsubscribe(); + }); + test("agent manager coalesces adjacent assistant tokens into fewer messages", async () => { vi.useFakeTimers(); const workdir = mkdtempSync(join(tmpdir(), "paseo-mock-load-test-")); diff --git a/packages/server/src/server/agent/providers/mock-load-test-agent.ts b/packages/server/src/server/agent/providers/mock-load-test-agent.ts index ffab16c79..76f9848d8 100644 --- a/packages/server/src/server/agent/providers/mock-load-test-agent.ts +++ b/packages/server/src/server/agent/providers/mock-load-test-agent.ts @@ -36,6 +36,8 @@ export const MOCK_LOAD_TEST_DEFAULT_MODEL_ID = "five-minute-stream"; const MOCK_LOAD_TEST_MODE_ID = "load-test"; const MOCK_LOAD_TEST_DURATION_MS = 5 * 60 * 1000; const MOCK_LOAD_TEST_INTERVAL_MS = 40; +const ONE_PIXEL_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Wl4Kj8AAAAASUVORK5CYII="; const CAPABILITIES: AgentCapabilityFlags = { supportsStreaming: true, @@ -156,6 +158,13 @@ function shouldEmitTurnFailure(prompt: AgentPromptInput): boolean { return /emit\s+(?:a\s+)?synthetic\s+turn\s+failure/i.test(promptToText(prompt)); } +function parseSettledAssistantImageMarkdown(prompt: AgentPromptInput): string | null { + const match = /^emit settled assistant image markdown:\s*(!\[[^\]\r\n]*\]\(.+\))\s*$/i.exec( + promptToText(prompt), + ); + return match?.[1] ?? null; +} + function parseMockQuestionPrompt(prompt: AgentPromptInput): MockQuestionPromptRequest | null { const text = promptToText(prompt); if (!/emit\s+(?:a\s+)?synthetic\s+questions?/i.test(text)) { @@ -669,10 +678,13 @@ export class MockLoadTestAgentSession implements AgentSession { const stress = parseAgentStreamStressPrompt(prompt); const questionPrompt = parseMockQuestionPrompt(prompt); const structuredBranchName = parseStructuredBranchNamePrompt(prompt); + const settledAssistantImageMarkdown = parseSettledAssistantImageMarkdown(prompt); if (shouldEmitTurnFailure(prompt)) { this.scheduleFailedTurn(turn); } else if (structuredBranchName) { - this.scheduleStructuredJsonTurn(turn, structuredBranchName); + this.scheduleSettledAssistantTurn(turn, JSON.stringify(structuredBranchName)); + } else if (settledAssistantImageMarkdown) { + this.scheduleSettledAssistantTurn(turn, settledAssistantImageMarkdown); } else if (shouldEmitPlanApprovalPrompt(prompt)) { this.schedulePlanApprovalTurn(turn); } else if (questionPrompt) { @@ -887,14 +899,14 @@ export class MockLoadTestAgentSession implements AgentSession { turn.timer.unref?.(); } - private scheduleStructuredJsonTurn(turn: ActiveTurn, result: Record): void { + private scheduleSettledAssistantTurn(turn: ActiveTurn, finalText: string): void { turn.timer = setTimeout(() => { - this.emitStructuredJsonTurn(turn, result); + this.emitSettledAssistantTurn(turn, finalText); }, 0); turn.timer.unref?.(); } - private emitStructuredJsonTurn(turn: ActiveTurn, result: Record): void { + private emitSettledAssistantTurn(turn: ActiveTurn, finalText: string): void { if (this.activeTurn !== turn) { return; } @@ -906,7 +918,6 @@ export class MockLoadTestAgentSession implements AgentSession { turnId: turn.turnId, }); - const finalText = JSON.stringify(result); this.emitTimeline(turn.turnId, { type: "assistant_message", text: finalText, @@ -1117,9 +1128,14 @@ export class MockLoadTestAgentSession implements AgentSession { }), ); } else { + const imageBytes = Buffer.from(ONE_PIXEL_PNG_BASE64, "base64"); + const imagePayload = Buffer.concat([ + imageBytes, + Buffer.alloc(Math.max(0, largePayload.bytes - imageBytes.length)), + ]).toString("base64"); this.emitTimeline(turn.turnId, { type: "assistant_message", - text: `data:image/png;base64,${payload}`, + text: `![Synthetic image](data:image/png;base64,${imagePayload})`, messageId: turn.assistantMessageId, }); }