From 7709c42b1358dda9717bb213a9a130bdbefc3cb9 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 27 Jul 2026 18:04:00 +0200 Subject: [PATCH] fix(app): keep timeline history and previews stable Treat persisted timeline replicas as display-only so authoritative pagination always comes from the daemon. Require renewed user intent between older-history pages unless the viewport genuinely remains at the history edge. Model assistant image acquisition as loading, loaded, or failed so recreating a preview URL cannot be rendered as an error. --- docs/timeline-sync.md | 17 + .../app/e2e/agent-tab-image-stability.spec.ts | 87 ++++ .../app/e2e/agent-timeline-pagination.spec.ts | 107 ++++- .../app/e2e/helpers/agent-timeline-gate.ts | 179 +++++++- .../app/e2e/helpers/timeline-pagination.ts | 262 ++++++++++- .../history-start-pagination.test.ts | 107 ++++- .../agent-stream/history-start-pagination.ts | 111 ++++- .../app/src/agent-stream/strategy-native.tsx | 104 ++++- .../src/agent-stream/strategy-web.test.tsx | 70 --- .../app/src/agent-stream/strategy-web.tsx | 190 ++++++-- .../assistant-image/acquisition-cache.test.ts | 46 ++ .../src/assistant-image/acquisition-cache.ts | 45 ++ .../app/src/assistant-image/lifecycle.test.ts | 99 +++++ packages/app/src/assistant-image/lifecycle.ts | 43 ++ .../assistant-image/use-assistant-image.ts | 407 ++++++++++++++++++ packages/app/src/components/message.tsx | 275 +++--------- packages/app/src/contexts/session-context.tsx | 138 ++---- .../hooks/use-agent-initialization.test.ts | 41 ++ .../app/src/hooks/use-agent-initialization.ts | 7 +- .../src/hooks/use-load-older-agent-history.ts | 21 +- packages/app/src/panels/agent-panel.tsx | 12 +- .../src/runtime/replica-cache/index.test.ts | 49 ++- .../app/src/runtime/replica-cache/index.ts | 27 +- packages/app/src/stores/session-store.test.ts | 48 +++ packages/app/src/stores/session-store.ts | 108 ++++- .../src/timeline/session-stream-reducers.ts | 150 +++++-- .../utils/assistant-image-metadata.test.ts | 18 - .../app/src/utils/assistant-image-metadata.ts | 14 - .../agent/providers/mock-load-test-agent.ts | 9 +- 29 files changed, 2135 insertions(+), 656 deletions(-) create mode 100644 packages/app/e2e/agent-tab-image-stability.spec.ts create mode 100644 packages/app/src/assistant-image/acquisition-cache.test.ts create mode 100644 packages/app/src/assistant-image/acquisition-cache.ts create mode 100644 packages/app/src/assistant-image/lifecycle.test.ts create mode 100644 packages/app/src/assistant-image/lifecycle.ts create mode 100644 packages/app/src/assistant-image/use-assistant-image.ts diff --git a/docs/timeline-sync.md b/docs/timeline-sync.md index b852e340a8..7c84824618 100644 --- a/docs/timeline-sync.md +++ b/docs/timeline-sync.md @@ -37,6 +37,12 @@ Initialization timeouts guard lack of catch-up progress, not the full multi-page The first load of an agent without a local cursor is different: it fetches a bounded latest tail page. Older history remains user-driven by scrolling upward. +Reaching the history-start threshold loads one older page and preserves the visible content anchor. +Cursor progress does not trigger another page. The user must leave and return to the threshold unless +the anchored page still leaves the viewport at history start, as with short or compacted content; in +that case pagination continues as one loading operation until the page fills the viewport or history +is exhausted. + ## Durable item anchors Provider message IDs are not guaranteed for every displayed item. Paseo-generated system errors are one example. Rendered item indices are not durable either because pagination and projection can merge source rows. @@ -61,6 +67,17 @@ recomposition while the runtime still owns the same directory snapshot and timel Removing the host from the registry is the destructive boundary: it stops the runtime and clears the session and host-scoped setup state together. +The durable replica cache is a display cache, not a synchronization checkpoint. Its timeline record +contains only the focused `agentId` and a truncated item tail. It never persists a cursor, epoch, +older-history availability, authority status, or sync generation because those facts would describe +the complete source dataset rather than the truncated display dataset. + +Restoring that cache produces a painted timeline: the items may render immediately, but the first +daemon timeline request is still `tail`. A successful tail response atomically establishes canonical +items, range, and older-history availability. Live rows received between cache paint and that tail +response stay in the separate live head, do not advance a cursor or trigger gap recovery, and are +reconciled with the authoritative tail and subsequent catch-up. + ## Selective and legacy delivery The app chooses one delivery policy from `server_info.features.selectiveAgentTimeline`: diff --git a/packages/app/e2e/agent-tab-image-stability.spec.ts b/packages/app/e2e/agent-tab-image-stability.spec.ts new file mode 100644 index 0000000000..e4e8acfa06 --- /dev/null +++ b/packages/app/e2e/agent-tab-image-stability.spec.ts @@ -0,0 +1,87 @@ +import type { Page } from "@playwright/test"; +import { expect, test as base } from "./fixtures"; +import { openWorkspaceWithAgents, type ArchiveTabAgent } from "./helpers/archive-tab"; +import { seedWorkspace, type SeedDaemonClient, type SeededWorkspace } from "./helpers/seed-client"; + +const IMAGE_ALT = "Synthetic image"; +const IMAGE_PREVIEW_ERROR = "Unable to load image preview."; +const IMAGE_PROMPT = "Emit 4096 byte image agent stream update"; + +const test = base.extend<{ imageWorkspace: SeededWorkspace }>({ + imageWorkspace: async ({ page: _page }, provide) => { + const workspace = await seedWorkspace({ repoPrefix: "agent-tab-image-stability-" }); + await provide(workspace); + await workspace.cleanup(); + }, +}); + +async function createMockAgent( + client: SeedDaemonClient, + input: { cwd: string; workspaceId: string; title: string }, +): Promise { + const agent = await client.createAgent({ + provider: "mock", + model: "ten-second-stream", + modeId: "load-test", + cwd: input.cwd, + workspaceId: input.workspaceId, + title: input.title, + }); + await client.waitForAgentUpsert(agent.id, (snapshot) => snapshot.status === "idle", 30_000); + return { id: agent.id, ...input }; +} + +async function observeImagePreviewErrors(page: Page): Promise { + await page.evaluate((errorText) => { + const root = document.documentElement; + root.dataset.agentTabImageErrorFlash = "false"; + const inspect = () => { + const errorVisible = Array.from(document.querySelectorAll("*")).some((element) => { + if (element.children.length > 0 || element.textContent?.trim() !== errorText) { + return false; + } + const rect = element.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0; + }); + if (errorVisible) { + root.dataset.agentTabImageErrorFlash = "true"; + } + }; + const observer = new MutationObserver(inspect); + observer.observe(document.body, { childList: true, subtree: true, characterData: true }); + window.setTimeout(() => observer.disconnect(), 2_000); + }, IMAGE_PREVIEW_ERROR); +} + +async function expectNoObservedImagePreviewError(page: Page): Promise { + await page.waitForTimeout(250); + expect(await page.locator("html").getAttribute("data-agent-tab-image-error-flash")).toBe("false"); +} + +test("switching between settled agent tabs keeps the image timeline valid", async ({ + imageWorkspace: workspace, + page, +}) => { + test.setTimeout(120_000); + const imageAgent = await createMockAgent(workspace.client, { + cwd: workspace.repoPath, + workspaceId: workspace.workspaceId, + title: `Image timeline ${Date.now()}`, + }); + const otherAgent = await createMockAgent(workspace.client, { + cwd: workspace.repoPath, + workspaceId: workspace.workspaceId, + title: `Other timeline ${Date.now()}`, + }); + await workspace.client.sendAgentMessage(imageAgent.id, IMAGE_PROMPT); + await workspace.client.waitForFinish(imageAgent.id, 15_000); + await openWorkspaceWithAgents(page, [otherAgent, imageAgent]); + await expect(page.getByRole("img", { name: IMAGE_ALT })).toBeVisible({ timeout: 30_000 }); + + await page.getByRole("button", { name: otherAgent.title, exact: true }).click(); + await observeImagePreviewErrors(page); + await page.getByRole("button", { name: imageAgent.title, exact: true }).click(); + + await expect(page.getByRole("img", { name: IMAGE_ALT })).toHaveCount(1, { timeout: 30_000 }); + await expectNoObservedImagePreviewError(page); +}); diff --git a/packages/app/e2e/agent-timeline-pagination.spec.ts b/packages/app/e2e/agent-timeline-pagination.spec.ts index 30c0bc3793..209e37063d 100644 --- a/packages/app/e2e/agent-timeline-pagination.spec.ts +++ b/packages/app/e2e/agent-timeline-pagination.spec.ts @@ -1,27 +1,97 @@ import { test } from "./fixtures"; import { - expectLoadedTimelineDoesNotScroll, + expectSameOlderHistoryLoadingOperation, + expectTimelineAtHistoryStart, expectTimelinePromptNotMounted, expectTimelinePromptVisible, - holdNextOlderTimelinePage, + holdBootstrapTimelinePage, + holdDaemonHydration, + holdOlderHistoryPages, makeLoadedTimelineFitViewport, openAgentTimeline, + rememberOlderHistoryLoadingOperation, + rememberTimelineViewport, + reloadAgentTimelineFromPersistedReplica, scrollTimelineUntilOlderHistoryIsReachable, + scrollTimelineToNewestLoadedEdge, seedLongMockAgentTimeline, + sendLiveTurnBeforeHydration, + expectTimelineViewportAnchoredAfterPrepend, + userScrollsTimelineToHistoryStart, } from "./helpers/timeline-pagination"; test.describe("Agent timeline pagination", () => { - test("loads older history when the user scrolls to the top of a long agent timeline", async ({ + test("loads one page each time the user returns to history start", async ({ page }) => { + test.setTimeout(120_000); + const agent = await seedLongMockAgentTimeline({ turns: 80 }); + try { + const history = await holdOlderHistoryPages(page, agent); + await openAgentTimeline(page, agent); + await expectTimelinePromptVisible(page, agent.newestPrompt); + await expectTimelinePromptNotMounted(page, agent.oldestPrompt); + + await userScrollsTimelineToHistoryStart(page); + await history.expectRequestedPages(1); + history.releasePage(1); + await history.expectSettledWithRequestedPages(1); + + await userScrollsTimelineToHistoryStart(page); + await history.expectRequestedPages(2); + } finally { + await agent.cleanup(); + } + }); + + test("keeps the visible timeline position anchored while prepending a page", async ({ page }) => { + test.setTimeout(120_000); + const agent = await seedLongMockAgentTimeline({ turns: 80 }); + try { + const history = await holdOlderHistoryPages(page, agent); + await openAgentTimeline(page, agent); + await userScrollsTimelineToHistoryStart(page); + await history.expectRequestedPages(1); + const viewport = await rememberTimelineViewport(page); + + history.releasePage(1); + await expectTimelineViewportAnchoredAfterPrepend(page, viewport); + } finally { + await agent.cleanup(); + } + }); + + test("continues one loading operation while older pages still leave history start exposed", async ({ page, }) => { test.setTimeout(120_000); const agent = await seedLongMockAgentTimeline({ turns: 80 }); try { + await makeLoadedTimelineFitViewport(page); + const history = await holdOlderHistoryPages(page, agent); await openAgentTimeline(page, agent); - await expectTimelinePromptVisible(page, agent.newestPrompt); - await expectTimelinePromptNotMounted(page, agent.oldestPrompt); + await history.expectRequestedPages(1); + const loading = await rememberOlderHistoryLoadingOperation(page); - await scrollTimelineUntilOlderHistoryIsReachable(page); + history.releasePage(1); + await history.expectRequestedPages(2); + await expectTimelineAtHistoryStart(page); + await expectSameOlderHistoryLoadingOperation(page, loading); + } finally { + await agent.cleanup(); + } + }); + + test("keeps complete loaded history reachable after reload", async ({ page }) => { + test.setTimeout(120_000); + const agent = await seedLongMockAgentTimeline({ turns: 80 }); + try { + await openAgentTimeline(page, agent); + await scrollTimelineUntilOlderHistoryIsReachable(page, agent.oldestPrompt); + await expectTimelinePromptVisible(page, agent.oldestPrompt); + + const hydration = await holdDaemonHydration(page); + await reloadAgentTimelineFromPersistedReplica(page, agent); + hydration.release(); + await scrollTimelineUntilOlderHistoryIsReachable(page, agent.oldestPrompt); await expectTimelinePromptVisible(page, agent.oldestPrompt); } finally { @@ -29,18 +99,27 @@ test.describe("Agent timeline pagination", () => { } }); - test("loads older history when the initial page does not fill the viewport", async ({ page }) => { + test("preserves a live row received before replica hydration", async ({ page }) => { test.setTimeout(120_000); - const agent = await seedLongMockAgentTimeline({ turns: 30 }); + const agent = await seedLongMockAgentTimeline({ turns: 80 }); try { - await makeLoadedTimelineFitViewport(page); - const olderPage = await holdNextOlderTimelinePage(page, agent); await openAgentTimeline(page, agent); - await expectTimelinePromptVisible(page, agent.newestPrompt); - await expectLoadedTimelineDoesNotScroll(page); - await olderPage.expectLoading(); - olderPage.release(); + await scrollTimelineUntilOlderHistoryIsReachable(page, agent.oldestPrompt); + + const hydration = await holdBootstrapTimelinePage(page, agent); + await reloadAgentTimelineFromPersistedReplica(page, agent); + await hydration.waitForDelayedResponse(); + const livePrompt = await sendLiveTurnBeforeHydration(agent); + await expectTimelinePromptVisible(page, livePrompt); + + hydration.release(); + await hydration.waitForDelayedCatchUp(); + await expectTimelinePromptVisible(page, livePrompt); + hydration.releaseCatchUp(); + await scrollTimelineUntilOlderHistoryIsReachable(page, agent.oldestPrompt); await expectTimelinePromptVisible(page, agent.oldestPrompt); + await scrollTimelineToNewestLoadedEdge(page); + await expectTimelinePromptVisible(page, livePrompt); } finally { await agent.cleanup(); } diff --git a/packages/app/e2e/helpers/agent-timeline-gate.ts b/packages/app/e2e/helpers/agent-timeline-gate.ts index 511d303db2..104f0a1595 100644 --- a/packages/app/e2e/helpers/agent-timeline-gate.ts +++ b/packages/app/e2e/helpers/agent-timeline-gate.ts @@ -15,6 +15,21 @@ export interface AgentTimelineResponseGate { waitForDelayedResponse(): Promise; } +export interface OlderTimelinePagesGate { + getRequestCount(): number; + releasePage(pageNumber: number): void; + waitForRequestCount(count: number): Promise; +} + +export interface DaemonHydrationGate { + release(): void; +} + +export interface BootstrapTimelineGate extends AgentTimelineResponseGate { + releaseCatchUp(): void; + waitForDelayedCatchUp(): Promise; +} + function parseWebSocketJson(message: WebSocketMessage): unknown { const rawMessage = typeof message === "string" ? message : message.toString("utf8"); try { @@ -45,6 +60,32 @@ function getPayload(message: Record): Record | : null; } +export async function holdDaemonHydration(page: Page): Promise { + let released = false; + const delayedForwards: Array<() => void> = []; + + await page.routeWebSocket(daemonWsRoutePattern(), (ws) => { + const server = ws.connectToServer(); + ws.onMessage((message) => server.send(message)); + server.onMessage((message) => { + if (released) { + ws.send(message); + return; + } + delayedForwards.push(() => ws.send(message)); + }); + }); + + return { + release() { + released = true; + for (const forward of delayedForwards.splice(0)) { + forward(); + } + }, + }; +} + export async function delayCreatedAgentInitialTailResponse( page: Page, ): Promise { @@ -127,6 +168,142 @@ export async function delayCreatedAgentInitialTailResponse( export async function delayAgentOlderTimelineResponse( page: Page, agentId: string, +): Promise { + return delayAgentTimelineResponse(page, agentId, "before"); +} + +export async function holdAgentOlderTimelinePages( + page: Page, + agentId: string, +): Promise { + let requestCount = 0; + let responseCount = 0; + const releasedPages = new Set(); + const delayedForwards = new Map void>>(); + const requestWaiters = new Map void>>(); + + const resolveRequestWaiters = () => { + for (const [count, resolvers] of requestWaiters) { + if (requestCount < count) continue; + requestWaiters.delete(count); + for (const resolve of resolvers) resolve(); + } + }; + + await page.routeWebSocket(daemonWsRoutePattern(), (ws) => { + const server = ws.connectToServer(); + ws.onMessage((message) => { + const sessionMessage = getSessionMessage(message); + if ( + sessionMessage?.type === "fetch_agent_timeline_request" && + sessionMessage.agentId === agentId && + sessionMessage.direction === "before" + ) { + requestCount += 1; + resolveRequestWaiters(); + } + server.send(message); + }); + server.onMessage((message) => { + const sessionMessage = getSessionMessage(message); + const payload = sessionMessage ? getPayload(sessionMessage) : null; + if ( + sessionMessage?.type === "fetch_agent_timeline_response" && + payload?.agentId === agentId && + payload.direction === "before" + ) { + responseCount += 1; + const pageNumber = responseCount; + if (releasedPages.has(pageNumber)) { + ws.send(message); + return; + } + const forwards = delayedForwards.get(pageNumber) ?? []; + forwards.push(() => ws.send(message)); + delayedForwards.set(pageNumber, forwards); + return; + } + ws.send(message); + }); + }); + + return { + getRequestCount: () => requestCount, + releasePage(pageNumber) { + releasedPages.add(pageNumber); + for (const forward of delayedForwards.get(pageNumber) ?? []) forward(); + delayedForwards.delete(pageNumber); + }, + waitForRequestCount(count) { + if (requestCount >= count) return Promise.resolve(); + return new Promise((resolve) => { + const resolvers = requestWaiters.get(count) ?? []; + resolvers.push(resolve); + requestWaiters.set(count, resolvers); + }); + }, + }; +} + +export async function delayAgentBootstrapTailResponse( + page: Page, + agentId: string, +): Promise { + let tailReleased = false; + let catchUpReleased = false; + const delayedTailForwards: Array<() => void> = []; + const delayedCatchUpForwards: Array<() => void> = []; + let resolveDelayedTail: (() => void) | null = null; + let resolveDelayedCatchUp: (() => void) | null = null; + const delayedTail = new Promise((resolve) => { + resolveDelayedTail = resolve; + }); + const delayedCatchUp = new Promise((resolve) => { + resolveDelayedCatchUp = resolve; + }); + + await page.routeWebSocket(daemonWsRoutePattern(), (ws) => { + const server = ws.connectToServer(); + ws.onMessage((message) => server.send(message)); + server.onMessage((message) => { + const sessionMessage = getSessionMessage(message); + const payload = sessionMessage ? getPayload(sessionMessage) : null; + const isTimelineResponse = + sessionMessage?.type === "fetch_agent_timeline_response" && payload?.agentId === agentId; + if (isTimelineResponse && payload.direction === "tail") { + resolveDelayedTail?.(); + if (tailReleased) ws.send(message); + else delayedTailForwards.push(() => ws.send(message)); + return; + } + if (isTimelineResponse && payload.direction === "after") { + resolveDelayedCatchUp?.(); + if (catchUpReleased) ws.send(message); + else delayedCatchUpForwards.push(() => ws.send(message)); + return; + } + ws.send(message); + }); + }); + + return { + release() { + tailReleased = true; + for (const forward of delayedTailForwards.splice(0)) forward(); + }, + releaseCatchUp() { + catchUpReleased = true; + for (const forward of delayedCatchUpForwards.splice(0)) forward(); + }, + waitForDelayedResponse: () => delayedTail, + waitForDelayedCatchUp: () => delayedCatchUp, + }; +} + +async function delayAgentTimelineResponse( + page: Page, + agentId: string, + direction: "before" | "tail", ): Promise { let releaseRequested = false; let delayedResponseSeen = false; @@ -148,7 +325,7 @@ export async function delayAgentOlderTimelineResponse( !delayedResponseSeen && sessionMessage?.type === "fetch_agent_timeline_response" && payload?.agentId === agentId && - payload.direction === "before" + payload.direction === direction ) { delayedResponseSeen = true; resolveDelayedResponse?.(); diff --git a/packages/app/e2e/helpers/timeline-pagination.ts b/packages/app/e2e/helpers/timeline-pagination.ts index bf30233091..b522ff6c58 100644 --- a/packages/app/e2e/helpers/timeline-pagination.ts +++ b/packages/app/e2e/helpers/timeline-pagination.ts @@ -1,10 +1,17 @@ import { expect, type Page } from "@playwright/test"; import { buildAgentRoute, seedMockAgentWorkspace, type MockAgentWorkspace } from "./mock-agent"; import { + delayAgentBootstrapTailResponse, delayAgentOlderTimelineResponse, + holdDaemonHydration, + holdAgentOlderTimelinePages, type AgentTimelineResponseGate, + type BootstrapTimelineGate, + type OlderTimelinePagesGate, } from "./agent-timeline-gate"; +export { holdDaemonHydration }; + interface LongTimelineAgentOptions { turns: number; } @@ -15,6 +22,24 @@ interface LongTimelineAgent extends MockAgentWorkspace { } const PROMPT_PREFIX = "timeline-pagination-turn"; +const LIVE_BEFORE_HYDRATION_PROMPT = "timeline live before authoritative hydration"; +const HISTORY_START_THRESHOLD_PX = 96; + +interface TimelineViewportSnapshot { + scrollHeight: number; + scrollTop: number; +} + +interface OlderHistoryLoadingOperation { + animationStartTime: number | null; + marker: string; +} + +interface OlderHistoryPages { + expectRequestedPages(count: number): Promise; + expectSettledWithRequestedPages(count: number): Promise; + releasePage(pageNumber: number): void; +} function promptForTurn(index: number): string { return `${PROMPT_PREFIX}-${index}: emit 1 coalesced agent stream updates`; @@ -50,15 +75,17 @@ export async function openAgentTimeline(page: Page, agent: LongTimelineAgent): P } export async function expectTimelinePromptVisible(page: Page, prompt: string): Promise { - await expect(page.getByText(prompt, { exact: true })).toBeVisible({ timeout: 30_000 }); + const timeline = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); + await expect(timeline.getByText(prompt, { exact: true })).toBeVisible({ timeout: 30_000 }); } export async function expectTimelinePromptNotMounted(page: Page, prompt: string): Promise { - await expect(page.getByText(prompt, { exact: true })).toHaveCount(0); + const timeline = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); + await expect(timeline.getByText(prompt, { exact: true })).toHaveCount(0); } export async function makeLoadedTimelineFitViewport(page: Page): Promise { - await page.setViewportSize({ width: 1280, height: 8_000 }); + await page.setViewportSize({ width: 1280, height: 20_000 }); } export async function expectLoadedTimelineDoesNotScroll(page: Page): Promise { @@ -75,6 +102,33 @@ export async function expectLoadedTimelineDoesNotScroll(page: Page): Promise { + await expect + .poll(() => + page.evaluate((agentId) => { + const raw = localStorage.getItem("@paseo:replica-cache"); + if (!raw) return false; + const cache = JSON.parse(raw) as { + hosts?: Array<{ + timeline?: { + agentId?: string; + items?: unknown[]; + } | null; + }>; + }; + const timeline = cache.hosts?.find((host) => host.timeline?.agentId === agentId)?.timeline; + return timeline?.items?.length === 50; + }, agent.agentId), + ) + .toBe(true); + + await page.reload(); + await expectTimelinePromptVisible(page, agent.newestPrompt); +} + export async function holdNextOlderTimelinePage( page: Page, agent: LongTimelineAgent, @@ -90,6 +144,106 @@ export async function holdNextOlderTimelinePage( }; } +export async function holdOlderHistoryPages( + page: Page, + agent: LongTimelineAgent, +): Promise { + const gate: OlderTimelinePagesGate = await holdAgentOlderTimelinePages(page, agent.agentId); + return { + async expectRequestedPages(count) { + await gate.waitForRequestCount(count); + expect(gate.getRequestCount()).toBe(count); + }, + async expectSettledWithRequestedPages(count) { + await waitForTimelineGeometryToSettle(page); + expect(gate.getRequestCount()).toBe(count); + }, + releasePage(pageNumber) { + gate.releasePage(pageNumber); + }, + }; +} + +export async function rememberTimelineViewport(page: Page): Promise { + return readTimelineViewport(page); +} + +export async function expectTimelineViewportAnchoredAfterPrepend( + page: Page, + before: TimelineViewportSnapshot, +): Promise { + await expect + .poll(async () => (await readTimelineViewport(page)).scrollHeight) + .toBeGreaterThan(before.scrollHeight); + await waitForTimelineGeometryToSettle(page); + const after = await readTimelineViewport(page); + const contentGrowth = after.scrollHeight - before.scrollHeight; + const scrollAdjustment = after.scrollTop - before.scrollTop; + expect(Math.abs(contentGrowth - scrollAdjustment)).toBeLessThanOrEqual(2); +} + +export async function rememberOlderHistoryLoadingOperation( + page: Page, +): Promise { + const slot = page.getByTestId("load-older-history-spinner"); + await expect(slot).toBeVisible(); + const marker = `older-history-loading-${Date.now()}`; + await slot.evaluate((element, operationMarker) => { + const candidates = [element, ...Array.from(element.querySelectorAll("*"))]; + const animated = candidates.find( + (candidate) => getComputedStyle(candidate).animationName !== "none", + ); + if (!(animated instanceof HTMLElement)) { + throw new Error("Expected the older-history loader to contain an animated element"); + } + animated.dataset.olderHistoryLoadingOperation = operationMarker; + }, marker); + const animated = page.locator(`[data-older-history-loading-operation="${marker}"]`); + await expect + .poll(async () => { + const startTime = await animated.evaluate((element) => element.getAnimations()[0]?.startTime); + return typeof startTime === "number" ? startTime : null; + }) + .not.toBeNull(); + const animationStartTime = await animated.evaluate((element) => { + const startTime = element.getAnimations()[0]?.startTime; + return typeof startTime === "number" ? startTime : null; + }); + return { animationStartTime, marker }; +} + +export async function expectSameOlderHistoryLoadingOperation( + page: Page, + operation: OlderHistoryLoadingOperation, +): Promise { + const animated = page.locator(`[data-older-history-loading-operation="${operation.marker}"]`); + await expect(animated).toBeVisible(); + const animationStartTime = await animated.evaluate((element) => { + const startTime = element.getAnimations()[0]?.startTime; + return typeof startTime === "number" ? startTime : null; + }); + expect(animationStartTime).toBe(operation.animationStartTime); +} + +export async function expectTimelineAtHistoryStart(page: Page): Promise { + await expect + .poll(async () => (await readTimelineViewport(page)).scrollTop) + .toBeLessThanOrEqual(HISTORY_START_THRESHOLD_PX); +} + +export async function holdBootstrapTimelinePage( + page: Page, + agent: LongTimelineAgent, +): Promise { + return delayAgentBootstrapTailResponse(page, agent.agentId); +} + +export async function sendLiveTurnBeforeHydration(agent: LongTimelineAgent): Promise { + await agent.client.sendAgentMessage(agent.agentId, LIVE_BEFORE_HYDRATION_PROMPT); + await agent.client.waitForFinish(agent.agentId, 15_000); + return LIVE_BEFORE_HYDRATION_PROMPT; +} + export async function scrollTimelineToOldestLoadedEdge(page: Page): Promise { const scroll = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); await scroll.hover(); @@ -109,31 +263,95 @@ export async function scrollTimelineToOldestLoadedEdge(page: Page): Promise { +export async function userScrollsTimelineToHistoryStart(page: Page): Promise { const scroll = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); - const previousHeight = await scroll.evaluate((element) => { + await scroll.hover(); + for (let step = 0; step < 60; step += 1) { + if ((await readTimelineViewport(page)).scrollTop <= HISTORY_START_THRESHOLD_PX) { + break; + } + await page.mouse.wheel(0, -1_000); + await page.evaluate( + () => + new Promise((resolve) => { + requestAnimationFrame(() => resolve()); + }), + ); + } + await expect + .poll(async () => (await readTimelineViewport(page)).scrollTop) + .toBeLessThanOrEqual(HISTORY_START_THRESHOLD_PX); +} + +export async function scrollTimelineToNewestLoadedEdge(page: Page): Promise { + const scroll = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); + await scroll.evaluate((element) => { if (!(element instanceof HTMLElement)) { throw new Error("Agent chat scroll element is not an HTMLElement"); } - return element.scrollHeight; + element.scrollTop = element.scrollHeight; + element.dispatchEvent(new Event("scroll", { bubbles: true })); }); +} - await page.evaluate( - () => - new Promise((resolve) => { - requestAnimationFrame(() => requestAnimationFrame(() => resolve())); +export async function scrollTimelineUntilOlderHistoryIsReachable( + page: Page, + oldestPrompt: string, +): Promise { + const scroll = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); + const prompt = scroll.getByText(oldestPrompt, { exact: true }); + for (let attempt = 0; attempt < 10; attempt += 1) { + if ((await prompt.count()) > 0) { + await expect(prompt).toBeVisible(); + return; + } + const previousHeight = await readTimelineViewport(page); + await userScrollsTimelineToHistoryStart(page); + await expect + .poll(async () => (await readTimelineViewport(page)).scrollHeight) + .toBeGreaterThan(previousHeight.scrollHeight); + await waitForTimelineGeometryToSettle(page); + } + await expect(prompt).toBeVisible(); +} + +async function readTimelineViewport(page: Page): Promise { + const scroll = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); + return scroll.evaluate((element) => { + if (!(element instanceof HTMLElement)) { + throw new Error("Agent chat scroll element is not an HTMLElement"); + } + return { scrollHeight: element.scrollHeight, scrollTop: element.scrollTop }; + }); +} + +async function waitForTimelineGeometryToSettle(page: Page): Promise { + const scroll = page.locator('[data-testid="agent-chat-scroll"]:visible').first(); + await scroll.evaluate( + (element) => + new Promise((resolve, reject) => { + if (!(element instanceof HTMLElement)) { + reject(new Error("Agent chat scroll element is not an HTMLElement")); + return; + } + const startedAt = performance.now(); + let stableFrames = 0; + let previous = `${element.scrollTop}:${element.scrollHeight}`; + const sample = () => { + const current = `${element.scrollTop}:${element.scrollHeight}`; + stableFrames = current === previous ? stableFrames + 1 : 0; + previous = current; + if (stableFrames >= 4) { + resolve(); + return; + } + if (performance.now() - startedAt > 5_000) { + reject(new Error("Timeline geometry did not settle")); + return; + } + requestAnimationFrame(sample); + }; + requestAnimationFrame(sample); }), ); - await scrollTimelineToOldestLoadedEdge(page); - await expect - .poll(async () => - scroll.evaluate((element) => { - if (!(element instanceof HTMLElement)) { - throw new Error("Agent chat scroll element is not an HTMLElement"); - } - return element.scrollHeight; - }), - ) - .toBeGreaterThan(previousHeight); - await scrollTimelineToOldestLoadedEdge(page); } diff --git a/packages/app/src/agent-stream/history-start-pagination.test.ts b/packages/app/src/agent-stream/history-start-pagination.test.ts index 2801f479f4..a5e25cc99d 100644 --- a/packages/app/src/agent-stream/history-start-pagination.test.ts +++ b/packages/app/src/agent-stream/history-start-pagination.test.ts @@ -2,10 +2,12 @@ import { describe, expect, it } from "vitest"; import { createHistoryStartPaginationState, evaluateHistoryStartPagination, - rearmHistoryStartPagination, + isHistoryStartLoadingOperation, + settleHistoryStartPagination, + type HistoryStartPaginationInput, } from "./history-start-pagination"; -const visibleHistoryStart = { +const visibleHistoryStart: HistoryStartPaginationInput = { distanceFromHistoryStart: 0, hasOlderHistory: true, isLoadingOlderHistory: false, @@ -14,47 +16,110 @@ const visibleHistoryStart = { }; describe("history start pagination", () => { - it("loads once for each authoritative history cursor", () => { - const initial = createHistoryStartPaginationState(); - const first = evaluateHistoryStartPagination(initial, visibleHistoryStart); - const duplicate = evaluateHistoryStartPagination(first.state, visibleHistoryStart); - const nextPage = evaluateHistoryStartPagination(first.state, { + it("waits for anchored page geometry before authorizing another page", () => { + const first = evaluateHistoryStartPagination( + createHistoryStartPaginationState(), + visibleHistoryStart, + ); + const inFlight = evaluateHistoryStartPagination(first.state, { + ...visibleHistoryStart, + isLoadingOlderHistory: true, + }); + const pageApplied = evaluateHistoryStartPagination(inFlight.state, { + ...visibleHistoryStart, + isLoadingOlderHistory: true, + progressKey: "epoch-1:10", + }); + + expect([first.shouldLoad, inFlight.shouldLoad, pageApplied.shouldLoad]).toEqual([ + true, + false, + false, + ]); + expect(pageApplied.state).toEqual({ status: "settling", loadedProgressKey: "epoch-1:10" }); + }); + + it("loads one page each time anchored geometry leaves and returns to history start", () => { + const first = evaluateHistoryStartPagination( + createHistoryStartPaginationState(), + visibleHistoryStart, + ); + const pageApplied = evaluateHistoryStartPagination(first.state, { + ...visibleHistoryStart, + progressKey: "epoch-1:10", + }); + const settledAway = settleHistoryStartPagination(pageApplied.state, { + ...visibleHistoryStart, + distanceFromHistoryStart: 300, + progressKey: "epoch-1:10", + }); + const returned = evaluateHistoryStartPagination(settledAway.state, { ...visibleHistoryStart, progressKey: "epoch-1:10", }); - expect([first.shouldLoad, duplicate.shouldLoad, nextPage.shouldLoad]).toEqual([ - true, - false, - true, - ]); + expect([ + first.shouldLoad, + pageApplied.shouldLoad, + settledAway.shouldLoad, + returned.shouldLoad, + ]).toEqual([true, false, false, true]); }); - it("allows the same revision again after the user leaves the history edge", () => { + it("continues the same loading operation when anchored geometry remains at history start", () => { const first = evaluateHistoryStartPagination( createHistoryStartPaginationState(), visibleHistoryStart, ); - const away = evaluateHistoryStartPagination(first.state, { + const pageApplied = evaluateHistoryStartPagination(first.state, { ...visibleHistoryStart, - distanceFromHistoryStart: 200, + progressKey: "epoch-1:10", + }); + const continued = settleHistoryStartPagination(pageApplied.state, { + ...visibleHistoryStart, + progressKey: "epoch-1:10", }); - const returned = evaluateHistoryStartPagination(away.state, visibleHistoryStart); - expect([first.shouldLoad, away.shouldLoad, returned.shouldLoad]).toEqual([true, false, true]); + expect(continued.shouldLoad).toBe(true); + expect(isHistoryStartLoadingOperation(first.state)).toBe(true); + expect(isHistoryStartLoadingOperation(pageApplied.state)).toBe(true); + expect(isHistoryStartLoadingOperation(continued.state)).toBe(true); }); - it("re-arms the same cursor when the user makes another upward edge gesture", () => { + it("latches a request that finishes without cursor progress", () => { const first = evaluateHistoryStartPagination( createHistoryStartPaginationState(), visibleHistoryStart, ); - const retried = evaluateHistoryStartPagination( - rearmHistoryStartPagination(first.state), + const inFlight = evaluateHistoryStartPagination(first.state, { + ...visibleHistoryStart, + isLoadingOlderHistory: true, + }); + const finished = evaluateHistoryStartPagination(inFlight.state, visibleHistoryStart); + const duplicate = evaluateHistoryStartPagination(finished.state, visibleHistoryStart); + const away = evaluateHistoryStartPagination(duplicate.state, { + ...visibleHistoryStart, + distanceFromHistoryStart: 300, + }); + + expect(finished.state).toEqual({ status: "latched" }); + expect([finished.shouldLoad, duplicate.shouldLoad, away.shouldLoad]).toEqual([ + false, + false, + false, + ]); + expect(away.state).toEqual({ status: "ready" }); + }); + + it("does not mistake repeated edge observations for a finished request", () => { + const first = evaluateHistoryStartPagination( + createHistoryStartPaginationState(), visibleHistoryStart, ); + const repeated = evaluateHistoryStartPagination(first.state, visibleHistoryStart); - expect([first.shouldLoad, retried.shouldLoad]).toEqual([true, true]); + expect(repeated.state).toBe(first.state); + expect(repeated.shouldLoad).toBe(false); }); it("waits while history loading is unavailable or already active", () => { diff --git a/packages/app/src/agent-stream/history-start-pagination.ts b/packages/app/src/agent-stream/history-start-pagination.ts index 180ea5cf0f..ffaee4bb23 100644 --- a/packages/app/src/agent-stream/history-start-pagination.ts +++ b/packages/app/src/agent-stream/history-start-pagination.ts @@ -1,33 +1,71 @@ export const HISTORY_START_THRESHOLD_PX = 96; -export interface HistoryStartPaginationState { - requestedProgressKey: string | null; +export type HistoryStartPaginationState = + | { status: "ready" } + | { status: "loading"; requestedProgressKey: string; requestObserved: boolean } + | { status: "settling"; loadedProgressKey: string } + | { status: "latched" }; + +export interface HistoryStartPaginationInput { + distanceFromHistoryStart: number; + hasOlderHistory: boolean; + isLoadingOlderHistory: boolean; + isReady: boolean; + progressKey: string | null; +} + +export interface HistoryStartPaginationTransition { + state: HistoryStartPaginationState; + shouldLoad: boolean; } export function createHistoryStartPaginationState(): HistoryStartPaginationState { - return { requestedProgressKey: null }; + return { status: "ready" }; } -export function rearmHistoryStartPagination( - _state: HistoryStartPaginationState, -): HistoryStartPaginationState { - return createHistoryStartPaginationState(); +export function isHistoryStartLoadingOperation(state: HistoryStartPaginationState): boolean { + return state.status === "loading" || state.status === "settling"; } export function evaluateHistoryStartPagination( state: HistoryStartPaginationState, - input: { - distanceFromHistoryStart: number; - hasOlderHistory: boolean; - isLoadingOlderHistory: boolean; - isReady: boolean; - progressKey: string | null; - }, -): { state: HistoryStartPaginationState; shouldLoad: boolean } { - if (input.distanceFromHistoryStart > HISTORY_START_THRESHOLD_PX) { - return { state: createHistoryStartPaginationState(), shouldLoad: false }; + input: HistoryStartPaginationInput, +): HistoryStartPaginationTransition { + if (state.status === "loading") { + if (input.progressKey !== null && input.progressKey !== state.requestedProgressKey) { + return { + state: { status: "settling", loadedProgressKey: input.progressKey }, + shouldLoad: false, + }; + } + if (input.isLoadingOlderHistory && !state.requestObserved) { + return { state: { ...state, requestObserved: true }, shouldLoad: false }; + } + if (!input.isLoadingOlderHistory && !input.hasOlderHistory) { + return { state: { status: "latched" }, shouldLoad: false }; + } + if ( + state.requestObserved && + !input.isLoadingOlderHistory && + input.progressKey === state.requestedProgressKey + ) { + return { state: { status: "latched" }, shouldLoad: false }; + } + return { state, shouldLoad: false }; + } + + if (state.status === "settling") { + return { state, shouldLoad: false }; + } + + const isAtHistoryStart = input.distanceFromHistoryStart <= HISTORY_START_THRESHOLD_PX; + if (!isAtHistoryStart) { + return state.status === "ready" + ? { state, shouldLoad: false } + : { state: { status: "ready" }, shouldLoad: false }; } if ( + state.status === "latched" || !input.isReady || !input.hasOlderHistory || input.isLoadingOlderHistory || @@ -35,11 +73,42 @@ export function evaluateHistoryStartPagination( ) { return { state, shouldLoad: false }; } - if (state.requestedProgressKey === input.progressKey) { - return { state, shouldLoad: false }; - } return { - state: { requestedProgressKey: input.progressKey }, + state: { + status: "loading", + requestedProgressKey: input.progressKey, + requestObserved: false, + }, + shouldLoad: true, + }; +} + +export function settleHistoryStartPagination( + state: HistoryStartPaginationState, + input: HistoryStartPaginationInput, +): HistoryStartPaginationTransition { + if (state.status !== "settling") { + return { state, shouldLoad: false }; + } + const isAtHistoryStart = input.distanceFromHistoryStart <= HISTORY_START_THRESHOLD_PX; + if ( + !isAtHistoryStart || + !input.isReady || + !input.hasOlderHistory || + input.isLoadingOlderHistory || + input.progressKey === null + ) { + return { + state: isAtHistoryStart ? { status: "latched" } : { status: "ready" }, + shouldLoad: false, + }; + } + return { + state: { + status: "loading", + requestedProgressKey: input.progressKey, + requestObserved: false, + }, shouldLoad: true, }; } diff --git a/packages/app/src/agent-stream/strategy-native.tsx b/packages/app/src/agent-stream/strategy-native.tsx index 2a38fa26d2..75264ce634 100644 --- a/packages/app/src/agent-stream/strategy-native.tsx +++ b/packages/app/src/agent-stream/strategy-native.tsx @@ -34,7 +34,10 @@ import { import { createHistoryStartPaginationState, evaluateHistoryStartPagination, - rearmHistoryStartPagination, + isHistoryStartLoadingOperation, + settleHistoryStartPagination, + type HistoryStartPaginationInput, + type HistoryStartPaginationTransition, } from "./history-start-pagination"; const DEFAULT_MAINTAIN_VISIBLE_CONTENT_POSITION = Object.freeze({ @@ -53,6 +56,8 @@ const historyStartSlotStyle: ViewStyle = { paddingTop: 4, paddingBottom: 8, }; +const HISTORY_START_SETTLE_FRAMES = 2; + interface HistoryRowDisplayVariants { regular?: StreamItem; compact?: StreamItem; @@ -116,7 +121,11 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat const [isNativeViewportSettling, setIsNativeViewportSettling] = useState(false); const nativeViewportSettlingFrameIdRef = useRef(null); const historyStartReadyRef = useRef(false); - const historyStartPaginationStateRef = useRef(createHistoryStartPaginationState()); + const [historyStartPaginationState, setHistoryStartPaginationState] = useState( + createHistoryStartPaginationState, + ); + const historyStartPaginationStateRef = useRef(historyStartPaginationState); + const historyStartSettleFrameIdRef = useRef(null); const historyItems = useMemo(() => { if (segments.historyVirtualized.length === 0) { @@ -145,22 +154,63 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat ), [displayStateHistoryRows, historyRowRevision?.contentById], ); - const evaluateHistoryStart = useStableEvent(() => { + const getHistoryStartPaginationInput = useStableEvent((): HistoryStartPaginationInput => { const metrics = streamViewportMetricsRef.current; const hasMeasuredViewport = metrics.viewportMeasuredForKey === metrics.containerKey && metrics.contentMeasuredForKey === metrics.containerKey; - const result = evaluateHistoryStartPagination(historyStartPaginationStateRef.current, { + return { distanceFromHistoryStart: metrics.contentHeight - metrics.viewportHeight - metrics.offsetY, hasOlderHistory, isLoadingOlderHistory, isReady: historyStartReadyRef.current && hasMeasuredViewport, progressKey: olderHistoryProgressKey, - }); - historyStartPaginationStateRef.current = result.state; - if (result.shouldLoad) { - onNearHistoryStart(); + }; + }); + const applyHistoryStartPaginationTransition = useStableEvent( + (transition: HistoryStartPaginationTransition) => { + const previousState = historyStartPaginationStateRef.current; + historyStartPaginationStateRef.current = transition.state; + if (transition.state !== previousState) { + setHistoryStartPaginationState(transition.state); + } + if (transition.shouldLoad) { + onNearHistoryStart(); + } + }, + ); + const evaluateHistoryStart = useStableEvent(() => { + const transition = evaluateHistoryStartPagination( + historyStartPaginationStateRef.current, + getHistoryStartPaginationInput(), + ); + applyHistoryStartPaginationTransition(transition); + }); + const scheduleHistoryStartSettle = useStableEvent(() => { + const pendingFrame = historyStartSettleFrameIdRef.current; + if (pendingFrame !== null) { + cancelAnimationFrame(pendingFrame); } + let remainingFrames = HISTORY_START_SETTLE_FRAMES; + const settle = () => { + if (historyStartPaginationStateRef.current.status !== "settling") { + historyStartSettleFrameIdRef.current = null; + return; + } + const input = getHistoryStartPaginationInput(); + if (input.isLoadingOlderHistory || remainingFrames > 0) { + remainingFrames -= input.isLoadingOlderHistory ? 0 : 1; + historyStartSettleFrameIdRef.current = requestAnimationFrame(settle); + return; + } + historyStartSettleFrameIdRef.current = null; + const transition = settleHistoryStartPagination( + historyStartPaginationStateRef.current, + input, + ); + applyHistoryStartPaginationTransition(transition); + }; + historyStartSettleFrameIdRef.current = requestAnimationFrame(settle); }); const clearNativeViewportSettling = useCallback(() => { @@ -261,7 +311,9 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat clearNativeViewportSettling(); setIsNativeViewportSettling(false); historyStartReadyRef.current = false; - historyStartPaginationStateRef.current = createHistoryStartPaginationState(); + const initialHistoryStartState = createHistoryStartPaginationState(); + historyStartPaginationStateRef.current = initialHistoryStartState; + setHistoryStartPaginationState(initialHistoryStartState); const frame = requestAnimationFrame(() => { historyStartReadyRef.current = true; evaluateHistoryStart(); @@ -269,6 +321,11 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat return () => { cancelAnimationFrame(frame); clearPendingUserScrollEnd(); + const settleFrame = historyStartSettleFrameIdRef.current; + if (settleFrame !== null) { + historyStartSettleFrameIdRef.current = null; + cancelAnimationFrame(settleFrame); + } }; }, [agentId, clearNativeViewportSettling, clearPendingUserScrollEnd, evaluateHistoryStart]); @@ -369,16 +426,10 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat }); const handleScrollBeginDrag = useStableEvent((event: NativeSyntheticEvent) => { - if (!isLoadingOlderHistory) { - historyStartPaginationStateRef.current = rearmHistoryStartPagination( - historyStartPaginationStateRef.current, - ); - } clearPendingUserScrollEnd(); isUserScrollActiveRef.current = true; scrollKeyboardDismiss.onScrollBeginDrag(event); bottomAnchorController.beginUserScroll(); - evaluateHistoryStart(); }); // Defer drag end so momentum can take ownership, but capture the terminal @@ -454,11 +505,23 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat contentHeight: nextContentHeight, }); evaluateHistoryStart(); + if (historyStartPaginationStateRef.current.status === "settling") { + scheduleHistoryStartSettle(); + } }); useEffect(() => { evaluateHistoryStart(); - }, [evaluateHistoryStart, hasOlderHistory, isLoadingOlderHistory, olderHistoryProgressKey]); + if (historyStartPaginationStateRef.current.status === "settling") { + scheduleHistoryStartSettle(); + } + }, [ + evaluateHistoryStart, + hasOlderHistory, + isLoadingOlderHistory, + olderHistoryProgressKey, + scheduleHistoryStartSettle, + ]); const renderItem = useStableEvent( ({ item, index }: ListRenderItemInfo): ReactElement | null => { @@ -499,20 +562,21 @@ function NativeStreamViewport(props: StreamRenderInput & { strategy: StreamStrat ]); const historyFooterContent = useMemo(() => { - if (!hasOlderHistory && !isLoadingOlderHistory) { + const isLoadingOperation = isHistoryStartLoadingOperation(historyStartPaginationState); + if (!hasOlderHistory && !isLoadingOperation) { return null; } return ( - {isLoadingOlderHistory ? ( + {isLoadingOperation ? ( ) : null} ); - }, [hasOlderHistory, isLoadingOlderHistory]); + }, [hasOlderHistory, historyStartPaginationState]); // RN's FlatList strictMode keeps its internal renderItem wrapper stable when // data or the live header changes, preserving the row identities above. diff --git a/packages/app/src/agent-stream/strategy-web.test.tsx b/packages/app/src/agent-stream/strategy-web.test.tsx index c125293cf5..43e4a2cc58 100644 --- a/packages/app/src/agent-stream/strategy-web.test.tsx +++ b/packages/app/src/agent-stream/strategy-web.test.tsx @@ -285,76 +285,6 @@ describe("createWebStreamStrategy", () => { expect(scrollTo).not.toHaveBeenCalled(); }); - it("fires near-history-start when the user scrolls near the top", async () => { - const strategy = createWebStreamStrategy({ isMobileBreakpoint: true }); - const viewportRef = React.createRef(); - const onNearHistoryStart = vi.fn(); - container = document.createElement("div"); - document.body.appendChild(container); - root = createRoot(container); - - act(() => { - root?.render( - <> - {strategy.render({ - agentId: "agent", - segments: { - historyVirtualized: [], - historyMounted: [userMessage(1), userMessage(2)], - liveHead: [], - }, - boundary: { - hasVirtualizedHistory: false, - hasMountedHistory: true, - hasLiveHead: false, - }, - renderers: createRenderers(vi.fn()), - listEmptyComponent: null, - viewportRef, - routeBottomAnchorRequest: null, - isAuthoritativeHistoryReady: true, - onNearBottomChange: vi.fn(), - onNearHistoryStart, - isLoadingOlderHistory: false, - hasOlderHistory: true, - olderHistoryProgressKey: "epoch-1:20", - scrollEnabled: true, - listStyle: null, - baseListContentContainerStyle: null, - forwardListContentContainerStyle: null, - })} - , - ); - }); - - const scrollContainer = container.querySelector('[data-testid="agent-chat-scroll"]'); - if (!(scrollContainer instanceof HTMLElement)) { - throw new Error("Expected agent chat scroll container"); - } - Object.defineProperty(scrollContainer, "clientHeight", { configurable: true, value: 400 }); - Object.defineProperty(scrollContainer, "scrollHeight", { configurable: true, value: 1200 }); - Object.defineProperty(scrollContainer, "scrollTop", { configurable: true, value: 64 }); - - await act(async () => { - await new Promise((resolve) => requestAnimationFrame(resolve)); - }); - - expect(onNearHistoryStart).not.toHaveBeenCalled(); - - act(() => { - scrollContainer.dispatchEvent(new WheelEvent("wheel", { deltaY: -1 })); - scrollContainer?.dispatchEvent(new Event("scroll")); - }); - - expect(onNearHistoryStart).toHaveBeenCalledTimes(1); - - act(() => { - scrollContainer.dispatchEvent(new WheelEvent("wheel", { deltaY: -1 })); - }); - - expect(onNearHistoryStart).toHaveBeenCalledTimes(2); - }); - it("waits for bottom anchoring before evaluating a delayed initial tail", async () => { HTMLElement.prototype.scrollTo = vi.fn(function ( this: HTMLElement, diff --git a/packages/app/src/agent-stream/strategy-web.tsx b/packages/app/src/agent-stream/strategy-web.tsx index 7df586cef2..91d277b326 100644 --- a/packages/app/src/agent-stream/strategy-web.tsx +++ b/packages/app/src/agent-stream/strategy-web.tsx @@ -19,13 +19,22 @@ import { createStreamStrategy } from "./strategy"; import { createHistoryStartPaginationState, evaluateHistoryStartPagination, - rearmHistoryStartPagination, + isHistoryStartLoadingOperation, + settleHistoryStartPagination, + type HistoryStartPaginationInput, + type HistoryStartPaginationTransition, } from "./history-start-pagination"; interface CreateWebStreamStrategyInput { isMobileBreakpoint: boolean; } +interface HistoryStartPrependAnchor { + progressKey: string; + scrollHeight: number; + scrollTop: number; +} + type ScrollBehaviorLike = "auto" | "smooth"; const WEB_BOTTOM_SETTLE_TIMEOUT_MS = 200; @@ -33,6 +42,7 @@ const USER_SCROLL_DELTA_EPSILON = 1; const BOTTOM_OVERSCROLL_TOLERANCE_PX = 2; const AUTO_SCROLL_BOTTOM_THRESHOLD_PX = 64; const AUTO_SCROLL_RESUME_THRESHOLD_PX = 1; +const HISTORY_START_SETTLE_FRAMES = 2; const ThemedLoadingSpinner = withUnistyles(LoadingSpinner); const foregroundMutedColorMapping = (theme: Theme) => ({ @@ -146,7 +156,13 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool const pendingAutoScrollTimeoutRef = useRef(null); const pendingVirtualRowMeasureFramesRef = useRef(new Map()); const historyStartReadyRef = useRef(false); - const historyStartPaginationStateRef = useRef(createHistoryStartPaginationState()); + const [historyStartPaginationState, setHistoryStartPaginationState] = useState( + createHistoryStartPaginationState, + ); + const historyStartPaginationStateRef = useRef(historyStartPaginationState); + const historyStartPrependAnchorRef = useRef(null); + const historyStartPrependAnchorActiveRef = useRef(false); + const historyStartSettleFrameRef = useRef(null); const shouldUseVirtualizer = segments.historyVirtualized.length > 0; const { renderHistoryVirtualizedRow, @@ -176,6 +192,9 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool }); useEffect(() => { rowVirtualizer.shouldAdjustScrollPositionOnItemSizeChange = (_item, _delta, instance) => { + if (historyStartPrependAnchorActiveRef.current) { + return false; + } const viewportHeight = instance.scrollRect?.height ?? 0; const scrollOffset = instance.scrollOffset ?? 0; const remainingDistance = instance.getTotalSize() - (scrollOffset + viewportHeight); @@ -187,25 +206,130 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool }, [rowVirtualizer]); const virtualRows = rowVirtualizer.getVirtualItems(); const virtualTotalSize = rowVirtualizer.getTotalSize(); - const evaluateHistoryStart = useStableEvent(() => { + const getHistoryStartPaginationInput = useStableEvent((): HistoryStartPaginationInput | null => { const scrollContainer = scrollContainerRef.current; if (!scrollContainer) { - return; + return null; } const bottomAnchorSettled = !followOutputRef.current || isScrollContainerNearBottom(scrollContainer); - const result = evaluateHistoryStartPagination(historyStartPaginationStateRef.current, { + return { distanceFromHistoryStart: scrollContainer.scrollTop, hasOlderHistory, isLoadingOlderHistory, isReady: historyStartReadyRef.current && bottomAnchorSettled, progressKey: olderHistoryProgressKey, - }); - historyStartPaginationStateRef.current = result.state; - if (result.shouldLoad) { - onNearHistoryStart(); - } + }; }); + const applyHistoryStartPaginationTransition = useStableEvent( + (transition: HistoryStartPaginationTransition) => { + const previousState = historyStartPaginationStateRef.current; + historyStartPaginationStateRef.current = transition.state; + if (transition.state !== previousState) { + setHistoryStartPaginationState(transition.state); + } + if (!isHistoryStartLoadingOperation(transition.state)) { + historyStartPrependAnchorRef.current = null; + historyStartPrependAnchorActiveRef.current = false; + } + if (!transition.shouldLoad || olderHistoryProgressKey === null) { + return; + } + const scrollContainer = scrollContainerRef.current; + if (!scrollContainer) { + return; + } + historyStartPrependAnchorRef.current = { + progressKey: olderHistoryProgressKey, + scrollHeight: scrollContainer.scrollHeight, + scrollTop: scrollContainer.scrollTop, + }; + historyStartPrependAnchorActiveRef.current = false; + onNearHistoryStart(); + }, + ); + const evaluateHistoryStart = useStableEvent(() => { + const input = getHistoryStartPaginationInput(); + if (!input) { + return; + } + const transition = evaluateHistoryStartPagination( + historyStartPaginationStateRef.current, + input, + ); + applyHistoryStartPaginationTransition(transition); + }); + const applyHistoryStartPrependAnchor = useStableEvent(() => { + const scrollContainer = scrollContainerRef.current; + const anchor = historyStartPrependAnchorRef.current; + if (!scrollContainer || !anchor || !historyStartPrependAnchorActiveRef.current) { + return; + } + const contentGrowth = scrollContainer.scrollHeight - anchor.scrollHeight; + scrollContainer.scrollTop = anchor.scrollTop + contentGrowth; + lastKnownScrollTopRef.current = scrollContainer.scrollTop; + }); + const scheduleHistoryStartPrependSettle = useStableEvent(() => { + const pendingFrame = historyStartSettleFrameRef.current; + if (pendingFrame !== null) { + window.cancelAnimationFrame(pendingFrame); + } + let stableFrames = 0; + let previousGeometry = ""; + const sample = () => { + applyHistoryStartPrependAnchor(); + const scrollContainer = scrollContainerRef.current; + const input = getHistoryStartPaginationInput(); + if (!scrollContainer || !input || !historyStartPrependAnchorActiveRef.current) { + historyStartSettleFrameRef.current = null; + return; + } + const geometry = `${scrollContainer.scrollHeight}:${scrollContainer.scrollTop}`; + const hasPendingRowMeasurements = pendingVirtualRowMeasureFramesRef.current.size > 0; + if ( + input.isLoadingOlderHistory || + hasPendingRowMeasurements || + geometry !== previousGeometry + ) { + stableFrames = 0; + } else { + stableFrames += 1; + } + previousGeometry = geometry; + if (stableFrames < HISTORY_START_SETTLE_FRAMES) { + historyStartSettleFrameRef.current = window.requestAnimationFrame(sample); + return; + } + historyStartSettleFrameRef.current = null; + historyStartPrependAnchorActiveRef.current = false; + const transition = settleHistoryStartPagination( + historyStartPaginationStateRef.current, + input, + ); + historyStartPrependAnchorRef.current = null; + applyHistoryStartPaginationTransition(transition); + }; + historyStartSettleFrameRef.current = window.requestAnimationFrame(sample); + }); + + useLayoutEffect(() => { + const anchor = historyStartPrependAnchorRef.current; + if (!anchor || anchor.progressKey === olderHistoryProgressKey) { + return; + } + historyStartPrependAnchorActiveRef.current = true; + evaluateHistoryStart(); + applyHistoryStartPrependAnchor(); + scheduleHistoryStartPrependSettle(); + }, [ + applyHistoryStartPrependAnchor, + evaluateHistoryStart, + olderHistoryProgressKey, + scheduleHistoryStartPrependSettle, + segments.historyMounted, + segments.historyVirtualized, + virtualTotalSize, + ]); const measureVirtualizedRowElement = useCallback( (node: HTMLDivElement | null) => { @@ -335,7 +459,11 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool }, [cancelPendingStickToBottom, evaluateHistoryStart, updateScrollMetrics]); useEffect(() => { - historyStartPaginationStateRef.current = createHistoryStartPaginationState(); + const initialHistoryStartState = createHistoryStartPaginationState(); + historyStartPaginationStateRef.current = initialHistoryStartState; + setHistoryStartPaginationState(initialHistoryStartState); + historyStartPrependAnchorRef.current = null; + historyStartPrependAnchorActiveRef.current = false; const frame = window.requestAnimationFrame(() => { historyStartReadyRef.current = true; evaluateHistoryStart(); @@ -343,6 +471,11 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool return () => { window.cancelAnimationFrame(frame); historyStartReadyRef.current = false; + const settleFrame = historyStartSettleFrameRef.current; + if (settleFrame !== null) { + historyStartSettleFrameRef.current = null; + window.cancelAnimationFrame(settleFrame); + } }; }, [evaluateHistoryStart, props.agentId]); @@ -423,6 +556,10 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool updateScrollMetrics(); evaluateHistoryStart(); const observer = new ResizeObserver(() => { + if (historyStartPrependAnchorActiveRef.current) { + applyHistoryStartPrependAnchor(); + scheduleHistoryStartPrependSettle(); + } updateScrollMetrics(); evaluateHistoryStart(); if (!followOutputRef.current) { @@ -437,7 +574,13 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool return () => { observer.disconnect(); }; - }, [evaluateHistoryStart, scheduleStickToBottom, updateScrollMetrics]); + }, [ + applyHistoryStartPrependAnchor, + evaluateHistoryStart, + scheduleHistoryStartPrependSettle, + scheduleStickToBottom, + updateScrollMetrics, + ]); useEffect(() => { const scrollContainer = scrollContainerRef.current; @@ -447,14 +590,8 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool const handleWheel = (event: WheelEvent) => { if (event.deltaY < 0) { - if (!isLoadingOlderHistory) { - historyStartPaginationStateRef.current = rearmHistoryStartPagination( - historyStartPaginationStateRef.current, - ); - } pendingUserScrollUpIntentRef.current = true; cancelPendingStickToBottom(); - evaluateHistoryStart(); } }; const handlePointerDown = () => { @@ -477,14 +614,8 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool } const previousTouchY = lastTouchClientYRef.current; if (previousTouchY !== null && touch.clientY > previousTouchY + 1) { - if (!isLoadingOlderHistory) { - historyStartPaginationStateRef.current = rearmHistoryStartPagination( - historyStartPaginationStateRef.current, - ); - } pendingUserScrollUpIntentRef.current = true; cancelPendingStickToBottom(); - evaluateHistoryStart(); } lastTouchClientYRef.current = touch.clientY; }; @@ -513,7 +644,7 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool scrollContainer.removeEventListener("touchend", handleTouchEnd); scrollContainer.removeEventListener("touchcancel", handleTouchEnd); }; - }, [cancelPendingStickToBottom, evaluateHistoryStart, handleDomScroll, isLoadingOlderHistory]); + }, [cancelPendingStickToBottom, handleDomScroll]); useEffect(() => { const handle: StreamViewportHandle = { @@ -595,20 +726,21 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool return renderLiveAuxiliary(); }, [renderLiveAuxiliary]); const historyStartSlot = useMemo(() => { - if (!hasOlderHistory && !isLoadingOlderHistory) { + const isLoadingOperation = isHistoryStartLoadingOperation(historyStartPaginationState); + if (!hasOlderHistory && !isLoadingOperation) { return null; } return (
- {isLoadingOlderHistory ? ( + {isLoadingOperation ? ( ) : null}
); - }, [hasOlderHistory, isLoadingOlderHistory]); + }, [hasOlderHistory, historyStartPaginationState]); const shouldRenderEmpty = !boundary.hasMountedHistory && !boundary.hasVirtualizedHistory && diff --git a/packages/app/src/assistant-image/acquisition-cache.test.ts b/packages/app/src/assistant-image/acquisition-cache.test.ts new file mode 100644 index 0000000000..5ad2789054 --- /dev/null +++ b/packages/app/src/assistant-image/acquisition-cache.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import { createAssistantImageAcquisitionCache } from "./acquisition-cache"; + +describe("assistant image acquisition cache", () => { + it("evicts a rejected acquisition so the next request can retry", async () => { + const cache = createAssistantImageAcquisitionCache({ capacity: 2 }); + let attempts = 0; + + await expect( + cache.acquire("image", async () => { + attempts += 1; + throw new Error("first attempt failed"); + }), + ).rejects.toThrow("first attempt failed"); + const recovered = await cache.acquire("image", async () => { + attempts += 1; + return "recovered"; + }); + + expect({ attempts, recovered, size: cache.size() }).toEqual({ + attempts: 2, + recovered: "recovered", + size: 1, + }); + }); + + it("bounds successful acquisitions and evicts the least recently used entry", async () => { + const cache = createAssistantImageAcquisitionCache({ capacity: 2 }); + const located: string[] = []; + const locate = async (key: string) => { + located.push(key); + return key; + }; + + await cache.acquire("a", async () => await locate("a")); + await cache.acquire("b", async () => await locate("b")); + await cache.acquire("a", async () => await locate("a-again")); + await cache.acquire("c", async () => await locate("c")); + await cache.acquire("b", async () => await locate("b-again")); + + expect({ located, size: cache.size() }).toEqual({ + located: ["a", "b", "c", "b-again"], + size: 2, + }); + }); +}); diff --git a/packages/app/src/assistant-image/acquisition-cache.ts b/packages/app/src/assistant-image/acquisition-cache.ts new file mode 100644 index 0000000000..6882c1c578 --- /dev/null +++ b/packages/app/src/assistant-image/acquisition-cache.ts @@ -0,0 +1,45 @@ +export interface AssistantImageAcquisitionCache { + acquire(key: string, locate: () => Promise): Promise; + size(): number; +} + +export function createAssistantImageAcquisitionCache(input: { + capacity: number; +}): AssistantImageAcquisitionCache { + if (!Number.isInteger(input.capacity) || input.capacity < 1) { + throw new Error("Assistant image acquisition cache capacity must be a positive integer."); + } + const entries = new Map>(); + + return { + acquire(key, locate) { + const cached = entries.get(key); + if (cached) { + entries.delete(key); + entries.set(key, cached); + return cached; + } + const pending = locate(); + entries.set(key, pending); + if (entries.size > input.capacity) { + const leastRecentlyUsedKey = entries.keys().next().value; + if (leastRecentlyUsedKey !== undefined) { + entries.delete(leastRecentlyUsedKey); + } + } + void (async () => { + try { + await pending; + } catch { + if (entries.get(key) === pending) { + entries.delete(key); + } + } + })(); + return pending; + }, + size() { + return entries.size; + }, + }; +} diff --git a/packages/app/src/assistant-image/lifecycle.test.ts b/packages/app/src/assistant-image/lifecycle.test.ts new file mode 100644 index 0000000000..c1783f35d7 --- /dev/null +++ b/packages/app/src/assistant-image/lifecycle.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "vitest"; +import { createAssistantImageLifecycle, transitionAssistantImageLifecycle } from "./lifecycle"; + +describe("assistant image lifecycle", () => { + it("keeps preview URL recreation in the public loading state", () => { + const loading = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), { + type: "preview_created", + uri: "blob:first", + aspectRatio: null, + }); + const loaded = transitionAssistantImageLifecycle(loading, { + type: "image_loaded", + uri: "blob:first", + aspectRatio: 1.5, + }); + + const recreating = transitionAssistantImageLifecycle(loaded, { + type: "preview_released", + }); + + expect(recreating).toEqual({ status: "loading", uri: null, aspectRatio: null }); + }); + + it("publishes loaded only after the recreated URL loads", () => { + const recreating = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), { + type: "preview_created", + uri: "blob:recreated", + aspectRatio: 1.5, + }); + + expect(recreating).toEqual({ + status: "loading", + uri: "blob:recreated", + aspectRatio: 1.5, + }); + + const recreated = transitionAssistantImageLifecycle(recreating, { + type: "image_loaded", + uri: "blob:recreated", + aspectRatio: 0.75, + }); + + expect(recreated).toEqual({ + status: "loaded", + uri: "blob:recreated", + aspectRatio: 0.75, + }); + }); + + it("publishes failed for a terminal image failure on the current URI", () => { + const loading = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), { + type: "preview_created", + uri: "blob:current", + aspectRatio: null, + }); + const failed = transitionAssistantImageLifecycle(loading, { + type: "failed", + uri: "blob:current", + message: "Unable to load image preview.", + }); + + expect(failed).toEqual({ + status: "failed", + message: "Unable to load image preview.", + }); + }); + + it("ignores a stale load callback from a replaced URI", () => { + const current = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), { + type: "preview_created", + uri: "blob:current", + aspectRatio: 1.25, + }); + + const afterStaleLoad = transitionAssistantImageLifecycle(current, { + type: "image_loaded", + uri: "blob:released", + aspectRatio: 2, + }); + + expect(afterStaleLoad).toEqual(current); + }); + + it("ignores a stale error callback from a replaced URI", () => { + const current = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), { + type: "preview_created", + uri: "blob:current", + aspectRatio: null, + }); + + const afterStaleError = transitionAssistantImageLifecycle(current, { + type: "failed", + uri: "blob:released", + message: "Image unavailable", + }); + + expect(afterStaleError).toEqual(current); + }); +}); diff --git a/packages/app/src/assistant-image/lifecycle.ts b/packages/app/src/assistant-image/lifecycle.ts new file mode 100644 index 0000000000..5093caf2d5 --- /dev/null +++ b/packages/app/src/assistant-image/lifecycle.ts @@ -0,0 +1,43 @@ +export type AssistantImageLifecycle = + | { status: "loading"; uri: string | null; aspectRatio: number | null } + | { status: "loaded"; uri: string; aspectRatio: number } + | { status: "failed"; message: string }; + +export type AssistantImageLifecycleEvent = + | { type: "preview_created"; uri: string; aspectRatio: number | null } + | { type: "preview_released" } + | { type: "image_loaded"; uri: string; aspectRatio: number } + | { type: "failed"; uri: string; message: string }; + +export function createAssistantImageLifecycle(): AssistantImageLifecycle { + return { status: "loading", uri: null, aspectRatio: null }; +} + +export function transitionAssistantImageLifecycle( + state: AssistantImageLifecycle, + event: AssistantImageLifecycleEvent, +): AssistantImageLifecycle { + if (event.type === "image_loaded") { + if (state.status !== "loading" || state.uri !== event.uri) { + return state; + } + return { + status: "loaded", + uri: event.uri, + aspectRatio: event.aspectRatio, + }; + } + if (event.type === "failed") { + if (state.status === "failed" || state.uri !== event.uri) { + return state; + } + return { status: "failed", message: event.message }; + } + if (event.type === "preview_created") { + return { status: "loading", uri: event.uri, aspectRatio: event.aspectRatio }; + } + if (event.type === "preview_released") { + return { status: "loading", uri: null, aspectRatio: null }; + } + return state; +} diff --git a/packages/app/src/assistant-image/use-assistant-image.ts b/packages/app/src/assistant-image/use-assistant-image.ts new file mode 100644 index 0000000000..c433882292 --- /dev/null +++ b/packages/app/src/assistant-image/use-assistant-image.ts @@ -0,0 +1,407 @@ +import { useCallback, useEffect, useMemo, useReducer, useState } from "react"; +import { Image } from "react-native"; +import { useTranslation } from "react-i18next"; +import type { DaemonClient } from "@getpaseo/client/internal/daemon-client"; +import type { AttachmentMetadata } from "@/attachments/types"; +import { + persistAttachmentFromBytes, + persistAttachmentFromDataUrl, + releaseAttachmentPreviewUrl, + resolveAttachmentPreviewUrl, +} from "@/attachments/service"; +import { + createPreviewAttachmentId, + getFileNameFromPath, + parseImageDataUrl, +} from "@/attachments/utils"; +import { + getAssistantImageMetadata, + setAssistantImageMetadata, +} from "@/utils/assistant-image-metadata"; +import { resolveAssistantImageSource } from "@/utils/assistant-image-source"; +import type { AssistantImageSourceResolution } from "@/utils/assistant-image-source"; +import { createAssistantImageAcquisitionCache } from "./acquisition-cache"; +import { + createAssistantImageLifecycle, + transitionAssistantImageLifecycle, + type AssistantImageLifecycle, + type AssistantImageLifecycleEvent, +} from "./lifecycle"; + +interface AssistantImageRenderBinding { + uri: string; + onLoad: () => void; + onError: () => void; +} + +export type AssistantImageResult = + | { + status: "loading"; + binding: AssistantImageRenderBinding | null; + aspectRatio: number | null; + } + | { + status: "loaded"; + binding: AssistantImageRenderBinding; + aspectRatio: number; + } + | { status: "failed"; message: string }; + +interface UseAssistantImageInput { + source: string; + client?: DaemonClient | null; + workspaceRoot?: string; + serverId?: string; +} + +type PreviewUrlState = + | { status: "waiting" } + | { status: "loading" } + | { status: "loaded"; uri: string } + | { status: "failed"; error: unknown }; + +type AttachmentAcquisitionState = + | { status: "waiting" } + | { status: "loading" } + | { status: "loaded"; attachment: AttachmentMetadata } + | { status: "failed"; error: unknown }; + +interface AttachmentAcquisition { + key: string; + locate: () => Promise; +} + +interface DataImage { + mimeType: string; + base64: string; + cacheKey: string; +} + +const attachmentAcquisitionCache = createAssistantImageAcquisitionCache({ + capacity: 500, +}); + +function acquireAttachment(acquisition: AttachmentAcquisition): Promise { + return attachmentAcquisitionCache.acquire(acquisition.key, acquisition.locate); +} + +function useAttachmentAcquisition( + acquisition: AttachmentAcquisition | null, +): AttachmentAcquisitionState { + const [state, setState] = useState({ status: "waiting" }); + + useEffect(() => { + let disposed = false; + if (!acquisition) { + setState({ status: "waiting" }); + return; + } + + setState({ status: "loading" }); + void (async () => { + try { + const attachment = await acquireAttachment(acquisition); + if (!disposed) { + setState({ status: "loaded", attachment }); + } + } catch (error) { + if (!disposed) { + setState({ status: "failed", error }); + } + } + })(); + return () => { + disposed = true; + }; + }, [acquisition]); + + return state; +} + +function createFileAcquisition(input: { + client?: DaemonClient | null; + resolution: AssistantImageSourceResolution | null; + serverId?: string; + unavailableMessage: string; +}): AttachmentAcquisition | null { + if (!input.client || input.resolution?.kind !== "file_rpc") { + return null; + } + const { client, resolution } = input; + return { + key: `file:${input.serverId ?? "unknown-server"}:${resolution.cwd}:${resolution.path}`, + locate: async () => { + const file = await client.readFile(resolution.cwd, resolution.path); + if (file.kind !== "image") { + throw new Error(input.unavailableMessage); + } + return await persistAttachmentFromBytes({ + id: createPreviewAttachmentId({ + mimeType: file.mime, + path: file.path || resolution.path, + size: file.size, + modifiedAt: file.modifiedAt, + contentLength: file.bytes.byteLength, + }), + bytes: file.bytes, + mimeType: file.mime, + fileName: getFileNameFromPath(file.path || resolution.path), + }); + }, + }; +} + +function createDataImageAcquisition(input: { + source: string; + dataImage: DataImage | null; +}): AttachmentAcquisition | null { + if (!input.dataImage) { + return null; + } + const { dataImage, source } = input; + return { + key: dataImage.cacheKey, + locate: async () => + await persistAttachmentFromDataUrl({ + id: createPreviewAttachmentId({ + mimeType: dataImage.mimeType, + contentLength: dataImage.base64.length, + }), + dataUrl: source, + mimeType: dataImage.mimeType, + }), + }; +} + +function usePreviewUrl(attachment: AttachmentMetadata | null | undefined): PreviewUrlState { + const [state, setState] = useState({ status: "waiting" }); + const id = attachment?.id; + const storageType = attachment?.storageType; + const storageKey = attachment?.storageKey; + const mimeType = attachment?.mimeType; + + useEffect(() => { + let disposed = false; + let previewUrl: string | null = null; + const current = attachment; + + if (!current) { + setState({ status: "waiting" }); + return; + } + + setState({ status: "loading" }); + void (async () => { + try { + const uri = await resolveAttachmentPreviewUrl(current); + if (disposed) { + await releaseAttachmentPreviewUrl({ attachment: current, url: uri }); + return; + } + previewUrl = uri; + setState({ status: "loaded", uri }); + } catch (error) { + if (!disposed) { + setState({ status: "failed", error }); + } + } + })(); + + return () => { + disposed = true; + if (previewUrl) { + void releaseAttachmentPreviewUrl({ attachment: current, url: previewUrl }); + } + }; + }, [attachment, id, mimeType, storageKey, storageType]); + + return state; +} + +function lifecycleReducer( + state: AssistantImageLifecycle, + event: AssistantImageLifecycleEvent, +): AssistantImageLifecycle { + return transitionAssistantImageLifecycle(state, event); +} + +function errorMessage(error: unknown, fallback: string): string { + return error instanceof Error ? error.message : fallback; +} + +function getAcquisitionFailure(input: { + hasResolution: boolean; + isFileSource: boolean; + isDataImage: boolean; + fileAttachment: AttachmentAcquisitionState; + dataImageAttachment: AttachmentAcquisitionState; + preview: PreviewUrlState; + hasDirectUri: boolean; + fallbackMessage: string; +}): AssistantImageResult | null { + if (!input.hasResolution) { + return { status: "failed", message: input.fallbackMessage }; + } + if (input.isFileSource && input.fileAttachment.status === "failed") { + return { + status: "failed", + message: errorMessage(input.fileAttachment.error, input.fallbackMessage), + }; + } + if (input.isDataImage && input.dataImageAttachment.status === "failed") { + return { + status: "failed", + message: errorMessage(input.dataImageAttachment.error, input.fallbackMessage), + }; + } + if (!input.hasDirectUri && input.preview.status === "failed") { + return { + status: "failed", + message: errorMessage(input.preview.error, input.fallbackMessage), + }; + } + return null; +} + +export function useAssistantImage({ + source, + client, + workspaceRoot, + serverId, +}: UseAssistantImageInput): AssistantImageResult { + const { t } = useTranslation(); + const resolution = useMemo( + () => resolveAssistantImageSource({ source, workspaceRoot }), + [source, workspaceRoot], + ); + const dataImage = useMemo(() => parseImageDataUrl(source), [source]); + const fileAcquisition = useMemo( + () => + createFileAcquisition({ + client, + resolution, + serverId, + unavailableMessage: t("message.attachments.imagePreviewUnavailable"), + }), + [client, resolution, serverId, t], + ); + const dataImageAcquisition = useMemo( + () => createDataImageAcquisition({ source, dataImage }), + [dataImage, source], + ); + const fileAttachment = useAttachmentAcquisition(fileAcquisition); + const dataImageAttachment = useAttachmentAcquisition(dataImageAcquisition); + const filePreview = usePreviewUrl( + fileAttachment.status === "loaded" ? fileAttachment.attachment : null, + ); + const dataImagePreview = usePreviewUrl( + dataImageAttachment.status === "loaded" ? dataImageAttachment.attachment : null, + ); + const [lifecycle, dispatch] = useReducer(lifecycleReducer, undefined, () => + createAssistantImageLifecycle(), + ); + + const directUri = resolution?.kind === "direct" && !dataImage ? resolution.uri : null; + const preview = dataImage ? dataImagePreview : filePreview; + const previewUri = preview.status === "loaded" ? preview.uri : null; + const uri = directUri ?? previewUri; + const cachedMetadata = useMemo( + () => getAssistantImageMetadata({ source, workspaceRoot, serverId }), + [serverId, source, workspaceRoot], + ); + + useEffect(() => { + if (!uri) { + dispatch({ type: "preview_released" }); + return; + } + + dispatch({ + type: "preview_created", + uri, + aspectRatio: cachedMetadata?.aspectRatio ?? null, + }); + }, [cachedMetadata, uri]); + + const handleImageError = useCallback(() => { + if (uri) { + dispatch({ + type: "failed", + uri, + message: t("message.attachments.imageUnavailable"), + }); + } + }, [t, uri]); + const handleImageLoad = useCallback(() => { + if (!uri) { + return; + } + Image.getSize( + uri, + (width, height) => { + const metadata = setAssistantImageMetadata( + { source, workspaceRoot, serverId }, + { width, height }, + ); + if (!metadata) { + dispatch({ + type: "failed", + uri, + message: t("message.attachments.imageUnavailable"), + }); + return; + } + dispatch({ type: "image_loaded", uri, aspectRatio: metadata.aspectRatio }); + }, + () => { + dispatch({ + type: "failed", + uri, + message: t("message.attachments.imageUnavailable"), + }); + }, + ); + }, [serverId, source, t, uri, workspaceRoot]); + + const acquisitionFailure = getAcquisitionFailure({ + hasResolution: resolution !== null, + isFileSource: resolution?.kind === "file_rpc", + isDataImage: dataImage !== null, + fileAttachment, + dataImageAttachment, + preview, + hasDirectUri: directUri !== null, + fallbackMessage: t("message.attachments.imagePreviewLoadFailed"), + }); + if (acquisitionFailure) { + return acquisitionFailure; + } + const hasCurrentLifecycleUri = lifecycle.status !== "failed" && lifecycle.uri === uri; + let binding: AssistantImageRenderBinding | null = null; + if (hasCurrentLifecycleUri && lifecycle.uri) { + binding = { + uri: lifecycle.uri, + onLoad: handleImageLoad, + onError: handleImageError, + }; + } + if (lifecycle.status === "loaded" && lifecycle.uri === uri) { + return { + status: "loaded", + binding: { + uri: lifecycle.uri, + onLoad: handleImageLoad, + onError: handleImageError, + }, + aspectRatio: lifecycle.aspectRatio, + }; + } + if (lifecycle.status === "failed") { + return lifecycle; + } + return { + status: "loading", + binding, + aspectRatio: hasCurrentLifecycleUri ? lifecycle.aspectRatio : null, + }; +} diff --git a/packages/app/src/components/message.tsx b/packages/app/src/components/message.tsx index e1870bd729..dfb0cfb715 100644 --- a/packages/app/src/components/message.tsx +++ b/packages/app/src/components/message.tsx @@ -29,7 +29,6 @@ import { } from "react"; import type { ComponentType, ReactNode } from "react"; import { MarkdownIt, type ASTNode, type RenderRules } from "react-native-markdown-display"; -import { useQuery } from "@tanstack/react-query"; import MaskedView from "@react-native-masked-view/masked-view"; import { Circle, @@ -75,19 +74,7 @@ import { splitMarkdownBlocks } from "@/utils/split-markdown-blocks"; import { formatDuration, formatMessageTimestamp } from "@/utils/time"; import { writeMarkdownToRichClipboard } from "@/utils/rich-clipboard"; import { getDefaultMarkdownClipboardEnvironment } from "@/utils/rich-clipboard-default-environment"; -import { - getAssistantImageLoadStateFromMetadata, - getAssistantImageMetadata, - setAssistantImageMetadata, - type AssistantImageLoadState, -} from "@/utils/assistant-image-metadata"; import { setAssistantMarkdownBlockHeight } from "@/utils/assistant-message-height-estimate"; -import { resolveAssistantImageSource } from "@/utils/assistant-image-source"; -import { - createPreviewAttachmentId, - getFileNameFromPath, - parseImageDataUrl, -} from "@/attachments/utils"; import { getAgentAttachmentPillContent } from "@/attachments/attachment-pill-content"; import { PlanCard } from "./plan-card"; import { useToolCallSheet } from "./tool-call-sheet"; @@ -102,8 +89,7 @@ import { useAssistantLinkPress, } from "@/assistant-file-links"; import { getCompactionMarkerLabel } from "./message-compaction-label"; -import { useAttachmentPreviewUrl } from "@/attachments/use-attachment-preview-url"; -import { persistAttachmentFromBytes, persistAttachmentFromDataUrl } from "@/attachments/service"; +import { useAssistantImage } from "@/assistant-image/use-assistant-image"; import { AttachmentFrame, AttachmentLabel, @@ -764,11 +750,21 @@ export const assistantMessageStylesheet = StyleSheet.create((theme) => ({ imageSurface: { width: "100%", overflow: "hidden", + position: "relative", }, image: { width: "100%", height: "100%", }, + imageLoadingOverlay: { + position: "absolute", + top: 0, + right: 0, + bottom: 0, + left: 0, + alignItems: "center", + justifyContent: "center", + }, imageState: { alignItems: "center", justifyContent: "center", @@ -785,123 +781,6 @@ export const assistantMessageStylesheet = StyleSheet.create((theme) => ({ const ASSISTANT_IMAGE_MIN_HEIGHT = 160; -const AssistantMarkdownResolvedImage = memo(function AssistantMarkdownResolvedImage({ - uri, - alt, - containerStyle, - source, - workspaceRoot, - serverId, -}: { - uri: string; - alt?: string; - containerStyle?: StyleProp; - source: string; - workspaceRoot?: string; - serverId?: string; -}) { - const cachedMetadata = useMemo( - () => getAssistantImageMetadata({ source, workspaceRoot, serverId }), - [serverId, source, workspaceRoot], - ); - const [loadState, setLoadState] = useState(() => - getAssistantImageLoadStateFromMetadata(cachedMetadata), - ); - - useEffect(() => { - if (cachedMetadata) { - setLoadState(getAssistantImageLoadStateFromMetadata(cachedMetadata)); - return () => {}; - } - - setLoadState({ status: "loading" }); - let cancelled = false; - - Image.getSize( - uri, - (width, height) => { - if (cancelled) { - return; - } - if (width > 0 && height > 0) { - const metadata = setAssistantImageMetadata( - { source, workspaceRoot, serverId }, - { width, height }, - ); - setLoadState({ - status: "ready", - aspectRatio: metadata?.aspectRatio ?? width / height, - }); - } - }, - () => { - if (cancelled) { - return; - } - setLoadState({ status: "error" }); - }, - ); - - return () => { - cancelled = true; - }; - }, [cachedMetadata, serverId, source, uri, workspaceRoot]); - - const handleImageError = useCallback(() => { - setLoadState({ status: "error" }); - }, []); - const { t } = useTranslation(); - const surfaceStyle = useMemo>( - () => [ - assistantMessageStylesheet.imageSurface, - loadState.status === "ready" - ? { aspectRatio: loadState.aspectRatio } - : { height: ASSISTANT_IMAGE_MIN_HEIGHT }, - ], - [loadState], - ); - const frameStyle = useMemo>( - () => [assistantMessageStylesheet.imageFrame, containerStyle], - [containerStyle], - ); - const stateSurfaceStyle = useMemo>( - () => [surfaceStyle, assistantMessageStylesheet.imageState], - [surfaceStyle], - ); - const imageSource = useMemo(() => ({ uri }), [uri]); - - if (loadState.status !== "ready") { - return ( - - - {loadState.status === "loading" ? ( - - ) : null} - {loadState.status === "error" ? ( - - {t("message.attachments.imageUnavailable")} - - ) : null} - - - ); - } - - return ( - - - - - - ); -}); - function AssistantMarkdownImage({ source, alt, @@ -917,12 +796,6 @@ function AssistantMarkdownImage({ workspaceRoot?: string; serverId?: string; }) { - const { t } = useTranslation(); - const resolution = useMemo( - () => resolveAssistantImageSource({ source, workspaceRoot }), - [source, workspaceRoot], - ); - const dataImage = useMemo(() => parseImageDataUrl(source), [source]); const containerStyle = useMemo>( () => ({ marginTop: hasLeadingContent ? 16 : 0, @@ -930,64 +803,25 @@ function AssistantMarkdownImage({ }), [hasLeadingContent], ); - - const query = useQuery({ - queryKey: [ - "assistantMarkdownImage", - serverId ?? "unknown-server", - resolution?.kind === "file_rpc" ? resolution.cwd : null, - resolution?.kind === "file_rpc" ? resolution.path : null, - ], - enabled: Boolean(client && resolution?.kind === "file_rpc"), - staleTime: 30_000, - queryFn: async () => { - if (!client || !resolution || resolution.kind !== "file_rpc") { - return null; - } - - const file = await client.readFile(resolution.cwd, resolution.path); - if (file.kind !== "image") { - throw new Error(t("message.attachments.imagePreviewUnavailable")); - } - - return await persistAttachmentFromBytes({ - id: createPreviewAttachmentId({ - mimeType: file.mime, - path: file.path || resolution.path, - size: file.size, - modifiedAt: file.modifiedAt, - contentLength: file.bytes.byteLength, - }), - bytes: file.bytes, - mimeType: file.mime, - fileName: getFileNameFromPath(file.path || resolution.path), - }); - }, - }); - const dataImageQuery = useQuery({ - queryKey: ["assistantMarkdownDataImage", dataImage?.cacheKey ?? null], - enabled: dataImage !== null, - staleTime: 30_000, - queryFn: async () => { - if (!dataImage) { - return null; - } - - return await persistAttachmentFromDataUrl({ - id: createPreviewAttachmentId({ - mimeType: dataImage.mimeType, - contentLength: dataImage.base64.length, - }), - dataUrl: source, - mimeType: dataImage.mimeType, - }); - }, - }); - - const fileAssetUri = useAttachmentPreviewUrl(query.data); - const dataImageAssetUri = useAttachmentPreviewUrl(dataImageQuery.data); - const directUri = resolution?.kind === "direct" && !dataImage ? resolution.uri : null; - const resolvedUri = directUri ?? dataImageAssetUri ?? fileAssetUri ?? null; + const image = useAssistantImage({ source, client, workspaceRoot, serverId }); + const binding = image.status === "failed" ? null : image.binding; + const aspectRatio = image.status === "failed" ? null : image.aspectRatio; + const imageUri = binding?.uri ?? ""; + const imageSource = useMemo(() => ({ uri: imageUri }), [imageUri]); + const frameStyle = useMemo>( + () => [assistantMessageStylesheet.imageFrame, containerStyle], + [containerStyle], + ); + const imageSizeStyle = useMemo(() => { + if (aspectRatio) { + return { aspectRatio }; + } + return { height: ASSISTANT_IMAGE_MIN_HEIGHT }; + }, [aspectRatio]); + const surfaceStyle = useMemo>( + () => [assistantMessageStylesheet.imageSurface, imageSizeStyle], + [imageSizeStyle], + ); const stateFrameStyle = useMemo>( () => [ @@ -999,20 +833,15 @@ function AssistantMarkdownImage({ [containerStyle], ); - if (resolvedUri) { + if (image.status === "failed") { return ( - + + {image.message} + ); } - if (query.isLoading || dataImageQuery.isLoading) { + if (!binding) { return ( @@ -1020,29 +849,27 @@ function AssistantMarkdownImage({ ); } - const errorText = resolveAssistantImageErrorText( - query.error, - dataImageQuery.error, - t("message.attachments.imagePreviewLoadFailed"), - ); - return ( - - {errorText} + + + + {image.status === "loading" ? ( + + + + ) : null} + ); } -function resolveAssistantImageErrorText( - fileError: unknown, - dataError: unknown, - fallbackText: string, -): string { - if (fileError instanceof Error) return fileError.message; - if (dataError instanceof Error) return dataError.message; - return fallbackText; -} - function getInlineCodeAutoLinkUrl( markdownParser: ReturnType, content: string, diff --git a/packages/app/src/contexts/session-context.tsx b/packages/app/src/contexts/session-context.tsx index 8a627d6148..f92322c17c 100644 --- a/packages/app/src/contexts/session-context.tsx +++ b/packages/app/src/contexts/session-context.tsx @@ -40,7 +40,12 @@ import type { AgentPermissionResponse } from "@getpaseo/protocol/agent-types"; import { getHostRuntimeStore, useHostRuntimeIsConnected } from "@/runtime/host-runtime"; import { useVoiceAudioEngineOptional, useVoiceRuntimeOptional } from "@/contexts/voice-context"; import type { AudioPlaybackSource } from "@/voice/audio-engine-types"; -import { useSessionStore, type MessageEntry, type SessionState } from "@/stores/session-store"; +import { + selectAgentTimelineState, + useSessionStore, + type MessageEntry, + type SessionState, +} from "@/stores/session-store"; import { useWorkspaceSetupStore } from "@/stores/workspace-setup-store"; import { sendOsNotification } from "@/utils/os-notifications"; import { getIsAppActivelyVisible, getIsAppVisible } from "@/utils/app-visibility"; @@ -192,11 +197,6 @@ type WorkspaceSetupProgressPayload = Extract< type SessionStoreActions = ReturnType; type SetInitializingAgents = SessionStoreActions["setInitializingAgents"]; -type SetAgentStreamState = SessionStoreActions["setAgentStreamState"]; -type SetAgentTimelineCursor = SessionStoreActions["setAgentTimelineCursor"]; -type MarkAgentHistorySynchronized = SessionStoreActions["markAgentHistorySynchronized"]; -type SetAgentAuthoritativeHistoryApplied = - SessionStoreActions["setAgentAuthoritativeHistoryApplied"]; function clearAgentInitializingFlag( setInitializingAgents: SetInitializingAgents, @@ -229,65 +229,6 @@ function handleTimelineError(input: { } } -function applyTimelineStreamPatches(input: { - result: ProcessTimelineResponseOutput; - agentId: string; - serverId: string; - currentTail: StreamItem[]; - currentHead: StreamItem[]; - setAgentStreamState: SetAgentStreamState; - setAgentTimelineCursor: SetAgentTimelineCursor; -}): void { - const { - result, - agentId, - serverId, - currentTail, - currentHead, - setAgentStreamState, - setAgentTimelineCursor, - } = input; - - if ( - result.tail !== currentTail || - result.head !== currentHead || - result.acknowledgedClientMessageIds.length > 0 - ) { - setAgentStreamState(serverId, agentId, { - ...(result.tail !== currentTail ? { tail: result.tail } : {}), - ...(result.head !== currentHead ? { head: result.head } : {}), - ...(result.acknowledgedClientMessageIds.length > 0 - ? { acknowledgedClientMessageIds: result.acknowledgedClientMessageIds } - : {}), - }); - } - - if (result.cursorChanged) { - setAgentTimelineCursor(serverId, (prev) => { - const current = prev.get(agentId); - if (!result.cursor) { - if (!current) { - return prev; - } - const next = new Map(prev); - next.delete(agentId); - return next; - } - if ( - current && - current.epoch === result.cursor.epoch && - current.startSeq === result.cursor.startSeq && - current.endSeq === result.cursor.endSeq - ) { - return prev; - } - const next = new Map(prev); - next.set(agentId, result.cursor); - return next; - }); - } -} - function executeTimelineSideEffects(input: { sideEffects: TimelineReducerSideEffect[]; agentId: string; @@ -308,8 +249,6 @@ function finalizeTimelineApplication(input: { serverId: string; shouldMarkAuthoritativeHistoryApplied: boolean; setInitializingAgents: SetInitializingAgents; - setAgentAuthoritativeHistoryApplied: SetAgentAuthoritativeHistoryApplied; - markAgentHistorySynchronized: MarkAgentHistorySynchronized; }): void { const { result, @@ -318,17 +257,13 @@ function finalizeTimelineApplication(input: { serverId, shouldMarkAuthoritativeHistoryApplied, setInitializingAgents, - setAgentAuthoritativeHistoryApplied, - markAgentHistorySynchronized, } = input; if (result.clearInitializing) { clearAgentInitializingFlag(setInitializingAgents, serverId, agentId); } if (shouldMarkAuthoritativeHistoryApplied) { - setAgentAuthoritativeHistoryApplied(serverId, agentId, true); useCreateFlowStore.getState().clearByAgent({ serverId, agentId }); - markAgentHistorySynchronized(serverId, agentId); const session = useSessionStore.getState().sessions[serverId]; const agent = session?.agents.get(agentId) ?? session?.agentDetails.get(agentId); if (agent && agent.status !== "running") { @@ -414,14 +349,10 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider const setAgentStreamState = useSessionStore((state) => state.setAgentStreamState); const clearAgentStreamHead = useSessionStore((state) => state.clearAgentStreamHead); const setAgentTimelineCursor = useSessionStore((state) => state.setAgentTimelineCursor); - const setAgentTimelineHasOlder = useSessionStore((state) => state.setAgentTimelineHasOlder); const setInitializingAgents = useSessionStore((state) => state.setInitializingAgents); const bumpHistorySyncGeneration = useSessionStore((state) => state.bumpHistorySyncGeneration); - const markAgentHistorySynchronized = useSessionStore( - (state) => state.markAgentHistorySynchronized, - ); - const setAgentAuthoritativeHistoryApplied = useSessionStore( - (state) => state.setAgentAuthoritativeHistoryApplied, + const applyAgentTimelineResponseState = useSessionStore( + (state) => state.applyAgentTimelineResponseState, ); const setAgents = useSessionStore((state) => state.setAgents); const setWorkspaces = useSessionStore((state) => state.setWorkspaces); @@ -651,15 +582,6 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider session?.messageSubmissions.get(agentId), ); - setAgentTimelineHasOlder(serverId, (prev) => { - if (prev.get(agentId) === payload.hasOlder) { - return prev; - } - const next = new Map(prev); - next.set(agentId, payload.hasOlder); - return next; - }); - // Call pure reducer const result = processTimelineResponse({ payload, @@ -683,14 +605,13 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider return; } - applyTimelineStreamPatches({ - result, - agentId, - serverId, - currentTail, - currentHead, - setAgentStreamState, - setAgentTimelineCursor, + applyAgentTimelineResponseState(serverId, agentId, { + items: result.tail, + head: result.head, + range: result.cursorChanged ? (result.cursor ?? null) : (currentCursor ?? null), + older: payload.hasOlder ? "available" : "none", + synchronized: shouldMarkAuthoritativeHistoryApplied, + acknowledgedClientMessageIds: result.acknowledgedClientMessageIds, }); executeTimelineSideEffects({ @@ -706,20 +627,9 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider serverId, shouldMarkAuthoritativeHistoryApplied, setInitializingAgents, - setAgentAuthoritativeHistoryApplied, - markAgentHistorySynchronized, }); }, - [ - markAgentHistorySynchronized, - recoverTimelineGap, - serverId, - setAgentAuthoritativeHistoryApplied, - setAgentStreamState, - setAgentTimelineCursor, - setAgentTimelineHasOlder, - setInitializingAgents, - ], + [applyAgentTimelineResponseState, recoverTimelineGap, serverId, setInitializingAgents], ); useEffect(() => { @@ -730,16 +640,20 @@ function SessionProviderInternal({ children, serverId, client }: SessionProvider const sync = createViewedTimelineSync({ initialDeliveryMode, setSubscription: (agentIds) => client.setAgentTimelineSubscription(agentIds), - readCursor: (agentId) => - useSessionStore.getState().sessions[serverId]?.agentTimelineCursor.get(agentId), + readCursor: (agentId) => { + const timeline = selectAgentTimelineState( + useSessionStore.getState().sessions[serverId], + agentId, + ); + return timeline.status === "synced" ? (timeline.range ?? undefined) : undefined; + }, hasAuthoritativeHistory: (agentId) => - useSessionStore - .getState() - .sessions[serverId]?.agentAuthoritativeHistoryApplied.get(agentId) === true, + selectAgentTimelineState(useSessionStore.getState().sessions[serverId], agentId).status === + "synced", fetchPage: async (agentId, request) => { const session = useSessionStore.getState().sessions[serverId]; const initKey = getInitKey(serverId, agentId); - const shouldInitialize = session?.agentAuthoritativeHistoryApplied.get(agentId) !== true; + const shouldInitialize = selectAgentTimelineState(session, agentId).status !== "synced"; if (shouldInitialize) { if (!getInitDeferred(initKey)) { const deferred = createInitDeferred(initKey, request.direction ?? "tail"); diff --git a/packages/app/src/hooks/use-agent-initialization.test.ts b/packages/app/src/hooks/use-agent-initialization.test.ts index 52fb58e1ea..e78388a908 100644 --- a/packages/app/src/hooks/use-agent-initialization.test.ts +++ b/packages/app/src/hooks/use-agent-initialization.test.ts @@ -111,6 +111,47 @@ describe("ensureAgentIsInitialized", () => { expect(getInitDeferred(getInitKey(serverId, agentId))?.requestDirection).toBe("tail"); }); + it("requests a bounded projected tail after restoring painted replica items", () => { + const client = new FakeDaemonClient(); + const runtime = new FakeTimelineRuntime(); + useSessionStore.getState().restoreSessionReplica(serverId, { + agents: new Map(), + workspaces: new Map(), + emptyProjects: new Map(), + timeline: { + agentId, + items: [ + { + kind: "assistant_message", + id: "painted-item", + text: "Painted before hydration", + timestamp: new Date("2026-07-27T10:00:00.000Z"), + }, + ], + }, + }); + + void ensureAgentIsInitialized({ + serverId, + agentId, + client: client as never, + runtime, + setAgentInitializing: bindSetAgentInitializing(), + }); + + expect(runtime.requests).toEqual([ + { + serverId, + agentId, + request: { + direction: "tail", + limit: TIMELINE_FETCH_PAGE_SIZE, + projection: "projected", + }, + }, + ]); + }); + it("times out initialization after 65 seconds", async () => { vi.useFakeTimers(); const client = new FakeDaemonClient(); diff --git a/packages/app/src/hooks/use-agent-initialization.ts b/packages/app/src/hooks/use-agent-initialization.ts index ae80d352ea..ade27abacd 100644 --- a/packages/app/src/hooks/use-agent-initialization.ts +++ b/packages/app/src/hooks/use-agent-initialization.ts @@ -1,7 +1,7 @@ import { useCallback, useMemo } from "react"; import { useTranslation } from "react-i18next"; import type { DaemonClient } from "@getpaseo/client/internal/daemon-client"; -import { useSessionStore } from "@/stores/session-store"; +import { selectAgentTimelineState, useSessionStore } from "@/stores/session-store"; import { createInitDeferred, getInitDeferred, @@ -52,8 +52,9 @@ export function ensureAgentIsInitialized(input: EnsureAgentIsInitializedInput): } const session = useSessionStore.getState().sessions[serverId]; - const cursor = session?.agentTimelineCursor.get(agentId); - const hasAuthoritativeHistory = session?.agentAuthoritativeHistoryApplied.get(agentId) === true; + const timeline = selectAgentTimelineState(session, agentId); + const cursor = timeline.status === "synced" ? (timeline.range ?? undefined) : undefined; + const hasAuthoritativeHistory = timeline.status === "synced"; const timelineRequest = planInitialAgentTimelineSync({ cursor, hasAuthoritativeHistory }); const deferred = createInitDeferred(key, timelineRequest.direction); diff --git a/packages/app/src/hooks/use-load-older-agent-history.ts b/packages/app/src/hooks/use-load-older-agent-history.ts index 1bf8b9557a..6de39066d1 100644 --- a/packages/app/src/hooks/use-load-older-agent-history.ts +++ b/packages/app/src/hooks/use-load-older-agent-history.ts @@ -2,7 +2,11 @@ import { useCallback } from "react"; import { useTranslation } from "react-i18next"; import type { ToastApi } from "@/components/toast-host"; import { i18n } from "@/i18n/i18next"; -import { useSessionStore, type AgentTimelineCursorState } from "@/stores/session-store"; +import { + selectAgentTimelineState, + useSessionStore, + type AgentTimelineCursorState, +} from "@/stores/session-store"; import { planTimelineOlderFetch } from "@/timeline/timeline-sync-plan"; import { getHostRuntimeStore } from "@/runtime/host-runtime"; @@ -70,15 +74,17 @@ export function useLoadOlderAgentHistory({ toast?: ToastApi | null; }) { const { t } = useTranslation(); - const hasOlder = - useSessionStore((state) => state.sessions[serverId]?.agentTimelineHasOlder.get(agentId)) === - true; + const hasOlder = useSessionStore((state) => { + const timeline = selectAgentTimelineState(state.sessions[serverId], agentId); + return timeline.status === "synced" && timeline.older === "available"; + }); const isLoadingOlder = useSessionStore((state) => state.sessions[serverId]?.agentTimelineOlderFetchInFlight.get(agentId), ) === true; const progressKey = useSessionStore((state) => { - const cursor = state.sessions[serverId]?.agentTimelineCursor.get(agentId); + const timeline = selectAgentTimelineState(state.sessions[serverId], agentId); + const cursor = timeline.status === "synced" ? timeline.range : null; return cursor ? `${cursor.epoch}:${cursor.startSeq}` : null; }); const setOlderFetchInFlight = useSessionStore( @@ -101,6 +107,7 @@ export function useLoadOlderAgentHistory({ const loadOlder = useCallback(() => { const session = useSessionStore.getState().sessions[serverId]; + const timeline = selectAgentTimelineState(session, agentId); void loadOlderAgentHistory(agentId, { client: session?.client ? { @@ -108,8 +115,8 @@ export function useLoadOlderAgentHistory({ getHostRuntimeStore().fetchAgentTimeline(serverId, timelineAgentId, request), } : null, - cursor: session?.agentTimelineCursor.get(agentId), - hasOlder: session?.agentTimelineHasOlder.get(agentId) === true, + cursor: timeline.status === "synced" ? (timeline.range ?? undefined) : undefined, + hasOlder: timeline.status === "synced" && timeline.older === "available", isLoadingOlder: session?.agentTimelineOlderFetchInFlight.get(agentId) === true, setInFlight, toast, diff --git a/packages/app/src/panels/agent-panel.tsx b/packages/app/src/panels/agent-panel.tsx index 7057169f11..dbc2ab46fb 100644 --- a/packages/app/src/panels/agent-panel.tsx +++ b/packages/app/src/panels/agent-panel.tsx @@ -72,7 +72,7 @@ import { WorkspaceDraftAgentTab } from "@/composer/draft/workspace-tab"; import { useCreateFlowStore } from "@/stores/create-flow-store"; import { buildDraftStoreKey, generateDraftId } from "@/stores/draft-keys"; import { usePanelStore } from "@/stores/panel-store"; -import { type Agent, useSessionStore } from "@/stores/session-store"; +import { selectAgentTimelineState, type Agent, useSessionStore } from "@/stores/session-store"; import { useWorkspaceLayoutStore } from "@/stores/workspace-layout-store"; import { buildWorkspaceTabPersistenceKey } from "@/workspace-tabs/model"; import type { Theme } from "@/styles/theme"; @@ -792,11 +792,12 @@ function ChatAgentContent({ const historySyncGeneration = useSessionStore( (state) => state.sessions[serverId]?.historySyncGeneration ?? 0, ); - const hasAppliedAuthoritativeHistory = useSessionStore((state) => + const replicaTimelineStatus = useSessionStore((state) => agentId - ? state.sessions[serverId]?.agentAuthoritativeHistoryApplied?.get(agentId) === true - : false, + ? selectAgentTimelineState(state.sessions[serverId], agentId).status + : ("cold" as const), ); + const hasAppliedAuthoritativeHistory = replicaTimelineStatus === "synced"; const agentHistorySyncGeneration = useSessionStore((state) => agentId ? (state.sessions[serverId]?.agentHistorySyncGeneration?.get(agentId) ?? -1) : -1, ); @@ -832,7 +833,8 @@ function ChatAgentContent({ kind: "idle", }); - const hasHydratedHistoryBefore = hasAppliedAuthoritativeHistory; + const hasHydratedHistoryBefore = + hasAppliedAuthoritativeHistory || replicaTimelineStatus === "painted"; const attentionController = useAgentAttentionClear({ agentId, diff --git a/packages/app/src/runtime/replica-cache/index.test.ts b/packages/app/src/runtime/replica-cache/index.test.ts index 69800f03ac..3ae4114d2a 100644 --- a/packages/app/src/runtime/replica-cache/index.test.ts +++ b/packages/app/src/runtime/replica-cache/index.test.ts @@ -4,6 +4,7 @@ import { normalizeAgentSnapshot } from "@/utils/agent-snapshots"; import { normalizeEmptyProjectDescriptor, normalizeWorkspaceDescriptor, + selectAgentTimelineState, useSessionStore, } from "@/stores/session-store"; import type { StreamItem } from "@/types/stream"; @@ -166,13 +167,14 @@ describe("ReplicaCache", () => { expect(session?.agents.get("agent-1")?.updatedAt).toBeInstanceOf(Date); expect(session?.workspaces.get("workspace-1")?.statusEnteredAt).toBeInstanceOf(Date); expect(session?.agentStreamTail.get("agent-1")).toEqual([message("message-1", "Cached")]); - expect(session?.agentAuthoritativeHistoryApplied.get("agent-1")).toBe(true); - expect(session?.agentTimelineCursor.get("agent-1")).toEqual({ - epoch: "epoch-1", - startSeq: 1, - endSeq: 12, + expect(session?.agentAuthoritativeHistoryApplied).toEqual(new Map()); + expect(session?.agentTimelineCursor).toEqual(new Map()); + expect(session?.agentTimelineHasOlder).toEqual(new Map()); + expect(session?.agentHistorySyncGeneration).toEqual(new Map()); + expect(selectAgentTimelineState(session, "agent-1")).toEqual({ + status: "painted", + items: [message("message-1", "Cached")], }); - expect(session?.agentTimelineHasOlder.get("agent-1")).toBe(true); }); it("persists only the focused agent view with a short timeline tail", async () => { @@ -216,6 +218,13 @@ describe("ReplicaCache", () => { expect(Array.from(session?.emptyProjects.keys() ?? [])).toEqual([]); expect(Array.from(timelines?.keys() ?? [])).toEqual(["agent-2"]); expect(timelines?.get("agent-2")).toEqual(secondTimeline.slice(-50)); + + const persisted = JSON.parse(storage.values.get("@paseo:replica-cache") ?? "null") as { + version: number; + hosts: Array<{ timeline: Record | null }>; + }; + expect(persisted.version).toBe(2); + expect(Object.keys(persisted.hosts[0]?.timeline ?? {}).sort()).toEqual(["agentId", "items"]); }); it("evicts the least recently written host when the cache exceeds its byte budget", async () => { @@ -243,14 +252,38 @@ describe("ReplicaCache", () => { expect(Object.keys(useSessionStore.getState().sessions).sort()).toEqual(["host-a", "host-c"]); }); - it("drops malformed or unknown cache versions", async () => { + it("rejects version 1 cache data and overwrites it on flush", async () => { const storage = new MemoryStorage(); - storage.values.set("@paseo:replica-cache", JSON.stringify({ version: 999, hosts: [] })); + storage.values.set( + "@paseo:replica-cache", + JSON.stringify({ + version: 1, + hosts: [ + { + serverId: SERVER_ID, + agents: [], + workspaces: [], + emptyProjects: [], + timeline: { + agentId: "agent-1", + items: [], + cursor: { epoch: "poisoned", startSeq: 1, endSeq: 100 }, + hasOlder: false, + }, + }, + ], + }), + ); const cache = new ReplicaCache(storage); cache.setHosts([SERVER_ID]); await cache.restore(); + await cache.flush(); expect(useSessionStore.getState().sessions[SERVER_ID]).toBeUndefined(); + expect(JSON.parse(storage.values.get("@paseo:replica-cache") ?? "null")).toEqual({ + version: 2, + hosts: [], + }); }); }); diff --git a/packages/app/src/runtime/replica-cache/index.ts b/packages/app/src/runtime/replica-cache/index.ts index b9bfab5c0c..2cad88aecc 100644 --- a/packages/app/src/runtime/replica-cache/index.ts +++ b/packages/app/src/runtime/replica-cache/index.ts @@ -11,6 +11,7 @@ import { import { normalizeEmptyProjectDescriptor, normalizeWorkspaceDescriptor, + selectAgentTimelineState, useSessionStore, type Agent, type SessionReplica, @@ -37,14 +38,6 @@ const StoredAgentSchema = z.object({ const StoredTimelineSchema = z.object({ agentId: z.string(), items: z.unknown(), - cursor: z - .object({ - epoch: z.string(), - startSeq: z.number().int().nonnegative(), - endSeq: z.number().int().nonnegative(), - }) - .nullable(), - hasOlder: z.boolean(), }); const StoredHostSchema = z.object({ @@ -141,8 +134,6 @@ function deserializeTimeline(stored: StoredHost["timeline"]): SessionReplica["ti return { agentId: stored.agentId, items: decoded, - cursor: stored.cursor, - hasOlder: stored.hasOlder, }; } @@ -376,24 +367,24 @@ export class ReplicaCache { focusedAgentId ? session.messageSubmissions.get(focusedAgentId) : undefined, ), ); - const items = focusedAgentId - ? session.agentStreamTail - .get(focusedAgentId) - ?.filter( + const timelineState = focusedAgentId + ? selectAgentTimelineState(session, focusedAgentId) + : { status: "cold" as const }; + const items = + timelineState.status === "cold" + ? undefined + : timelineState.items.filter( (item) => item.kind !== "user_message" || item.messageId !== undefined || !item.clientMessageId || !localSubmissionIds.has(item.clientMessageId), - ) - : undefined; + ); const timeline = focusedAgent && items ? { agentId: focusedAgent.id, items: encodeDates(items.slice(-MAX_TIMELINE_ITEMS)), - cursor: session.agentTimelineCursor.get(focusedAgent.id) ?? null, - hasOlder: session.agentTimelineHasOlder.get(focusedAgent.id) ?? false, } : null; const stored: StoredHost = { diff --git a/packages/app/src/stores/session-store.test.ts b/packages/app/src/stores/session-store.test.ts index 1588fd114a..cd386660fe 100644 --- a/packages/app/src/stores/session-store.test.ts +++ b/packages/app/src/stores/session-store.test.ts @@ -5,9 +5,11 @@ import type { WorkspaceDescriptorPayload } from "@getpaseo/protocol/messages"; import { normalizeWorkspaceDescriptor, + selectAgentTimelineState, useSessionStore, type WorkspaceDescriptor, } from "./session-store"; +import type { StreamItem } from "../types/stream"; import { patchWorkspaceScripts } from "../contexts/session-workspace-scripts"; function createWorkspace( @@ -53,6 +55,52 @@ function getTestSessionReferences() { }; } +describe("agent timeline state", () => { + it("commits canonical items, range, and older availability as one synced state", () => { + initializeTestSession(); + const items: StreamItem[] = [ + { + kind: "assistant_message", + id: "canonical-row", + text: "canonical", + timestamp: new Date("2026-07-27T10:00:00.000Z"), + }, + ]; + + useSessionStore.getState().applyAgentTimelineResponseState("test-server", "agent-1", { + items, + head: [], + range: { epoch: "epoch-1", startSeq: 51, endSeq: 100 }, + older: "available", + synchronized: true, + }); + + expect( + selectAgentTimelineState(useSessionStore.getState().sessions["test-server"], "agent-1"), + ).toEqual({ + status: "synced", + items, + range: { epoch: "epoch-1", startSeq: 51, endSeq: 100 }, + older: "available", + }); + }); + + it("represents an empty authoritative timeline without inventing a range", () => { + initializeTestSession(); + useSessionStore.getState().applyAgentTimelineResponseState("test-server", "agent-1", { + items: [], + head: [], + range: null, + older: "none", + synchronized: true, + }); + + expect( + selectAgentTimelineState(useSessionStore.getState().sessions["test-server"], "agent-1"), + ).toEqual({ status: "synced", items: [], range: null, older: "none" }); + }); +}); + describe("normalizeWorkspaceDescriptor", () => { it("normalizes workspace scripts and invalid activity timestamps", () => { const scripts = [ diff --git a/packages/app/src/stores/session-store.ts b/packages/app/src/stores/session-store.ts index 3a5504310e..1d59abdfeb 100644 --- a/packages/app/src/stores/session-store.ts +++ b/packages/app/src/stores/session-store.ts @@ -336,8 +336,6 @@ export interface AgentTimelineCursorState { export interface SessionReplicaTimeline { agentId: string; items: StreamItem[]; - cursor: AgentTimelineCursorState | null; - hasOlder: boolean; } export interface SessionReplica { @@ -347,6 +345,33 @@ export interface SessionReplica { timeline: SessionReplicaTimeline | null; } +export type AgentTimelineState = + | { status: "cold" } + | { status: "painted"; items: StreamItem[] } + | { + status: "synced"; + items: StreamItem[]; + range: AgentTimelineCursorState | null; + older: "available" | "none"; + }; + +export function selectAgentTimelineState( + session: SessionState | undefined, + agentId: string, +): AgentTimelineState { + if (!session) return { status: "cold" }; + const items = session.agentStreamTail.get(agentId) ?? []; + if (session.agentAuthoritativeHistoryApplied.get(agentId) === true) { + return { + status: "synced", + items, + range: session.agentTimelineCursor.get(agentId) ?? null, + older: session.agentTimelineHasOlder.get(agentId) === true ? "available" : "none", + }; + } + return items.length > 0 ? { status: "painted", items } : { status: "cold" }; +} + export type WorkspaceRestoreStatus = "restoring" | "failed" | "needs-host-upgrade"; // Per-session state @@ -520,6 +545,18 @@ interface SessionStoreActions { agentId: string, applied: boolean, ) => void; + applyAgentTimelineResponseState: ( + serverId: string, + agentId: string, + state: { + items: StreamItem[]; + head: StreamItem[]; + range: AgentTimelineCursorState | null; + older: "available" | "none"; + synchronized: boolean; + acknowledgedClientMessageIds: string[]; + }, + ) => void; // Initializing agents setInitializingAgents: ( @@ -758,16 +795,8 @@ export const useSessionStore = create()( const session = createInitialSessionState(serverId, null); const timeline = replica.timeline; const agentStreamTail = new Map(); - const agentTimelineCursor = new Map(); - const agentTimelineHasOlder = new Map(); - const agentAuthoritativeHistoryApplied = new Map(); - const agentHistorySyncGeneration = new Map(); if (timeline) { agentStreamTail.set(timeline.agentId, timeline.items); - agentTimelineHasOlder.set(timeline.agentId, timeline.hasOlder); - agentAuthoritativeHistoryApplied.set(timeline.agentId, true); - agentHistorySyncGeneration.set(timeline.agentId, session.historySyncGeneration); - if (timeline.cursor) agentTimelineCursor.set(timeline.agentId, timeline.cursor); } const agentLastActivity = new Map(prev.agentLastActivity); for (const agent of replica.agents.values()) { @@ -784,10 +813,6 @@ export const useSessionStore = create()( workspaces: replica.workspaces, emptyProjects: replica.emptyProjects, agentStreamTail, - agentTimelineCursor, - agentTimelineHasOlder, - agentAuthoritativeHistoryApplied, - agentHistorySyncGeneration, }, }, agentLastActivity, @@ -1457,6 +1482,61 @@ export const useSessionStore = create()( }); }, + applyAgentTimelineResponseState: (serverId, agentId, state) => { + set((prev) => { + const session = prev.sessions[serverId]; + if (!session) return prev; + + const nextTail = new Map(session.agentStreamTail); + nextTail.set(agentId, state.items); + const nextHead = new Map(session.agentStreamHead); + if (state.head.length > 0) nextHead.set(agentId, state.head); + else nextHead.delete(agentId); + const nextCursor = new Map(session.agentTimelineCursor); + if (state.range) nextCursor.set(agentId, state.range); + else nextCursor.delete(agentId); + const nextHasOlder = new Map(session.agentTimelineHasOlder); + nextHasOlder.set(agentId, state.older === "available"); + const nextAuthoritative = new Map(session.agentAuthoritativeHistoryApplied); + const nextSyncGeneration = new Map(session.agentHistorySyncGeneration); + const currentSubmissions = session.messageSubmissions.get(agentId) ?? []; + const observedSubmissions = observeMessageSubmissionCanonical( + currentSubmissions, + state.acknowledgedClientMessageIds, + ); + let messageSubmissions = session.messageSubmissions; + if (observedSubmissions !== currentSubmissions) { + messageSubmissions = new Map(session.messageSubmissions); + if (observedSubmissions.length > 0) { + messageSubmissions.set(agentId, observedSubmissions); + } else { + messageSubmissions.delete(agentId); + } + } + if (state.synchronized) { + nextAuthoritative.set(agentId, true); + nextSyncGeneration.set(agentId, session.historySyncGeneration); + } + + return { + ...prev, + sessions: { + ...prev.sessions, + [serverId]: { + ...session, + agentStreamTail: nextTail, + agentStreamHead: nextHead, + agentTimelineCursor: nextCursor, + agentTimelineHasOlder: nextHasOlder, + agentAuthoritativeHistoryApplied: nextAuthoritative, + agentHistorySyncGeneration: nextSyncGeneration, + messageSubmissions, + }, + }, + }; + }); + }, + // Initializing agents setInitializingAgents: (serverId, state) => { set((prev) => { diff --git a/packages/app/src/timeline/session-stream-reducers.ts b/packages/app/src/timeline/session-stream-reducers.ts index 2690bfb500..57b75107ca 100644 --- a/packages/app/src/timeline/session-stream-reducers.ts +++ b/packages/app/src/timeline/session-stream-reducers.ts @@ -1,5 +1,5 @@ import type { AgentStreamEventPayload } from "@getpaseo/protocol/messages"; -import { useSessionStore } from "@/stores/session-store"; +import { selectAgentTimelineState, useSessionStore } from "@/stores/session-store"; import type { AssistantMessageItem, StreamItem } from "@/types/stream"; import { applyStreamEvent, @@ -255,7 +255,6 @@ function applyTimelineReplacePath(args: { acknowledgedClientMessageIds, }; } - interface IncrementalAcceptResult { acceptedUnits: TimelineUnit[]; cursor: TimelineCursor | undefined; @@ -575,6 +574,43 @@ function reconcileOverlappingProjectedReasoning(params: { : { tail: params.tail, head: params.head, reconciled: false }; } +function reconcileProjectedUserOverlay(params: { + tail: StreamItem[]; + head: StreamItem[]; + unit: TimelineUnit; +}): { tail: StreamItem[]; head: StreamItem[]; reconciled: boolean } { + const { event } = params.unit; + if (event.type !== "timeline" || event.item.type !== "user_message") { + return { tail: params.tail, head: params.head, reconciled: false }; + } + const userItem = event.item; + const overlayIndex = params.head.findIndex( + (item) => + item.kind === "user_message" && + matchesLocalUserMessageIdentity( + { + messageId: userItem.messageId, + clientMessageId: userItem.clientMessageId, + text: userItem.text, + }, + item, + ), + ); + const overlay = params.head[overlayIndex]; + if (overlayIndex < 0 || !overlay || overlay.kind !== "user_message") { + return { tail: params.tail, head: params.head, reconciled: false }; + } + const canonical = reduceStreamUpdate([], event, params.unit.timestamp, { + source: "canonical", + })[0]; + if (!canonical || canonical.kind !== "user_message") { + return { tail: params.tail, head: params.head, reconciled: false }; + } + const head = [...params.head]; + head[overlayIndex] = mergeCanonicalUserWithLocalPresentation(canonical, overlay); + return { tail: params.tail, head, reconciled: true }; +} + function reconcileOverlappingProjectedStreamItems(params: { tail: StreamItem[]; head: StreamItem[]; @@ -588,13 +624,16 @@ function reconcileOverlappingProjectedStreamItems(params: { if (params.currentEndSeq === undefined) return { tail, head, reconciledUnits }; for (const unit of params.units) { - let reconciled = reconcileOverlappingProjectedAssistant({ - tail, - head, - unit, - epoch: params.epoch, - currentEndSeq: params.currentEndSeq, - }); + let reconciled = reconcileProjectedUserOverlay({ tail, head, unit }); + if (!reconciled.reconciled) { + reconciled = reconcileOverlappingProjectedAssistant({ + tail, + head, + unit, + epoch: params.epoch, + currentEndSeq: params.currentEndSeq, + }); + } if (!reconciled.reconciled) { reconciled = reconcileOverlappingProjectedReasoning({ tail, @@ -955,6 +994,7 @@ export interface ProcessAgentStreamEventInput { currentTail: StreamItem[]; currentHead: StreamItem[]; currentCursor: TimelineCursor | undefined; + hasAuthoritativeBaseline?: boolean; timestamp: Date; } @@ -989,6 +1029,7 @@ export interface ProcessAgentStreamEventsInput { currentTail: StreamItem[]; currentHead: StreamItem[]; currentCursor: TimelineCursor | undefined; + hasAuthoritativeBaseline?: boolean; } export type AgentStreamReducerSnapshot = Omit; @@ -1017,8 +1058,9 @@ function processTimelineSequencingGate(input: { seq: number | undefined; epoch: string | undefined; currentCursor: TimelineCursor | undefined; + hasAuthoritativeBaseline: boolean; }): TimelineSequencingGateResult { - const { event, seq, epoch, currentCursor } = input; + const { event, seq, epoch, currentCursor, hasAuthoritativeBaseline } = input; const base: TimelineSequencingGateResult = { shouldApplyStreamEvent: true, nextTimelineCursor: null, @@ -1029,6 +1071,9 @@ function processTimelineSequencingGate(input: { if (event.type !== "timeline" || typeof seq !== "number" || typeof epoch !== "string") { return base; } + if (!hasAuthoritativeBaseline) { + return base; + } const decision = classifySessionTimelineSeq({ cursor: currentCursor ? { epoch: currentCursor.epoch, endSeq: currentCursor.endSeq } : null, @@ -1085,9 +1130,24 @@ function processTimelineSequencingGate(input: { export function processAgentStreamEvent( input: ProcessAgentStreamEventInput, ): ProcessAgentStreamEventOutput { - const { event, seq, epoch, currentTail, currentHead, currentCursor, timestamp } = input; + const { + event, + seq, + epoch, + currentTail, + currentHead, + currentCursor, + timestamp, + hasAuthoritativeBaseline = true, + } = input; - const sequencing = processTimelineSequencingGate({ event, seq, epoch, currentCursor }); + const sequencing = processTimelineSequencingGate({ + event, + seq, + epoch, + currentCursor, + hasAuthoritativeBaseline, + }); const timelineCursor = event.type === "timeline" && seq !== undefined && epoch !== undefined ? { epoch, seq } @@ -1096,30 +1156,49 @@ export function processAgentStreamEvent( // ------------------------------------------------------------------ // Apply stream event to tail/head // ------------------------------------------------------------------ - const applied = sequencing.shouldApplyStreamEvent - ? applyStreamEvent({ - tail: sequencing.resetLiveTimeline ? [] : currentTail, - head: sequencing.resetLiveTimeline ? [] : currentHead, - event, - timestamp, - source: "live", - timelineCursor, - }) - : { - tail: currentTail, - head: currentHead, - changedTail: false, - changedHead: false, - }; + let streamResult: ReturnType; + if (!sequencing.shouldApplyStreamEvent) { + streamResult = { + tail: currentTail, + head: currentHead, + changedTail: false, + changedHead: false, + }; + } else if (!hasAuthoritativeBaseline) { + const overlay = applyStreamEvent({ + tail: currentHead, + head: [], + event, + timestamp, + source: "live", + timelineCursor, + }); + streamResult = { + tail: currentTail, + head: [...overlay.tail, ...overlay.head], + changedTail: false, + changedHead: overlay.changedTail || overlay.changedHead, + }; + } else { + streamResult = applyStreamEvent({ + tail: sequencing.resetLiveTimeline ? [] : currentTail, + head: sequencing.resetLiveTimeline ? [] : currentHead, + event, + timestamp, + source: "live", + timelineCursor, + }); + } + const { tail, head, changedTail, changedHead } = streamResult; return { - tail: applied.tail, - head: applied.head, - changedTail: applied.changedTail, - changedHead: applied.changedHead, + tail, + head, + changedTail, + changedHead, cursor: sequencing.nextTimelineCursor, cursorChanged: sequencing.cursorChanged, - acknowledgedClientMessageIds: applied.acknowledgedClientMessageIds ?? [], + acknowledgedClientMessageIds: streamResult.acknowledgedClientMessageIds ?? [], sideEffects: sequencing.sideEffects, }; } @@ -1144,6 +1223,7 @@ export function processAgentStreamEvents( currentTail: tail, currentHead: head, currentCursor: cursor, + hasAuthoritativeBaseline: input.hasAuthoritativeBaseline, timestamp: reducerEvent.timestamp, }); @@ -1281,10 +1361,12 @@ export function createSessionAgentStreamReducerQueue( return createAgentStreamReducerQueue({ getSnapshot: (agentId) => { const session = useSessionStore.getState().sessions[serverId]; + const timeline = selectAgentTimelineState(session, agentId); return { - currentTail: session?.agentStreamTail.get(agentId) ?? [], + currentTail: timeline.status === "cold" ? [] : timeline.items, currentHead: session?.agentStreamHead.get(agentId) ?? [], - currentCursor: session?.agentTimelineCursor.get(agentId), + currentCursor: timeline.status === "synced" ? (timeline.range ?? undefined) : undefined, + hasAuthoritativeBaseline: timeline.status === "synced", }; }, commit: (agentId, result, events) => { diff --git a/packages/app/src/utils/assistant-image-metadata.test.ts b/packages/app/src/utils/assistant-image-metadata.test.ts index e6138dc715..d4b99655f2 100644 --- a/packages/app/src/utils/assistant-image-metadata.test.ts +++ b/packages/app/src/utils/assistant-image-metadata.test.ts @@ -3,7 +3,6 @@ import { clearAssistantImageMetadataCache, estimateAssistantMessageHeightFromCache, extractAssistantImageSources, - getAssistantImageLoadStateFromMetadata, getAssistantImageMetadata, setAssistantImageMetadata, } from "./assistant-image-metadata"; @@ -42,23 +41,6 @@ describe("assistant image metadata", () => { }); }); - it("maps missing metadata to the image loading state", () => { - expect(getAssistantImageLoadStateFromMetadata(null)).toEqual({ status: "loading" }); - }); - - it("maps cached metadata to the image ready state", () => { - expect( - getAssistantImageLoadStateFromMetadata({ - width: 900, - height: 1600, - aspectRatio: 9 / 16, - }), - ).toEqual({ - status: "ready", - aspectRatio: 9 / 16, - }); - }); - it("estimates assistant message height from cached image metadata", () => { setAssistantImageMetadata( { diff --git a/packages/app/src/utils/assistant-image-metadata.ts b/packages/app/src/utils/assistant-image-metadata.ts index b11d26baf4..9f6e159b0d 100644 --- a/packages/app/src/utils/assistant-image-metadata.ts +++ b/packages/app/src/utils/assistant-image-metadata.ts @@ -8,11 +8,6 @@ export interface AssistantImageMetadata { aspectRatio: number; } -export type AssistantImageLoadState = - | { status: "loading" } - | { status: "ready"; aspectRatio: number } - | { status: "error" }; - const assistantImageMetadataCache = new Map(); const assistantImageParseCache = new Map(); const ASSISTANT_IMAGE_METADATA_CACHE_LIMIT = 500; @@ -131,15 +126,6 @@ export function getAssistantImageMetadata(input: { return null; } -export function getAssistantImageLoadStateFromMetadata( - metadata: AssistantImageMetadata | null, -): AssistantImageLoadState { - if (!metadata) { - return { status: "loading" }; - } - return { status: "ready", aspectRatio: metadata.aspectRatio }; -} - export function setAssistantImageMetadata( input: { source: string; diff --git a/packages/server/src/server/agent/providers/mock-load-test-agent.ts b/packages/server/src/server/agent/providers/mock-load-test-agent.ts index ffab16c79f..986c2160db 100644 --- a/packages/server/src/server/agent/providers/mock-load-test-agent.ts +++ b/packages/server/src/server/agent/providers/mock-load-test-agent.ts @@ -36,6 +36,8 @@ export const MOCK_LOAD_TEST_DEFAULT_MODEL_ID = "five-minute-stream"; const MOCK_LOAD_TEST_MODE_ID = "load-test"; const MOCK_LOAD_TEST_DURATION_MS = 5 * 60 * 1000; const MOCK_LOAD_TEST_INTERVAL_MS = 40; +const ONE_PIXEL_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Wl4Kj8AAAAASUVORK5CYII="; const CAPABILITIES: AgentCapabilityFlags = { supportsStreaming: true, @@ -1117,9 +1119,14 @@ export class MockLoadTestAgentSession implements AgentSession { }), ); } else { + const imageBytes = Buffer.from(ONE_PIXEL_PNG_BASE64, "base64"); + const imagePayload = Buffer.concat([ + imageBytes, + Buffer.alloc(Math.max(0, largePayload.bytes - imageBytes.length)), + ]).toString("base64"); this.emitTimeline(turn.turnId, { type: "assistant_message", - text: `data:image/png;base64,${payload}`, + text: `![Synthetic image](data:image/png;base64,${imagePayload})`, messageId: turn.assistantMessageId, }); }