diff --git a/packages/app/e2e/agent-tab-image-stability.spec.ts b/packages/app/e2e/agent-tab-image-stability.spec.ts index 70d5227025..81e1ac9c2e 100644 --- a/packages/app/e2e/agent-tab-image-stability.spec.ts +++ b/packages/app/e2e/agent-tab-image-stability.spec.ts @@ -36,31 +36,70 @@ async function createMockAgent( return { id: agent.id, ...input }; } -async function observeImagePreviewErrors(page: Page): Promise { - await page.evaluate((errorText) => { - const root = document.documentElement; - root.dataset.agentTabImageErrorFlash = "false"; - const inspect = () => { - const errorVisible = Array.from(document.querySelectorAll("*")).some((element) => { - if (element.children.length > 0 || element.textContent?.trim() !== errorText) { - return false; - } +async function observeImagePreviewInstability(page: Page, imageAgentId: string): Promise { + await page.evaluate( + ({ alt, errorText, tabTestId }) => { + const root = document.documentElement; + root.dataset.agentTabImageErrorFlash = "false"; + root.dataset.agentTabImageMissingFlash = "false"; + const isVisible = (element: Element) => { const rect = element.getBoundingClientRect(); return rect.width > 0 && rect.height > 0; + }; + const inspect = () => { + const errorVisible = Array.from(document.querySelectorAll("*")).some((element) => { + if (element.children.length > 0 || element.textContent?.trim() !== errorText) { + return false; + } + return isVisible(element); + }); + if (errorVisible) { + root.dataset.agentTabImageErrorFlash = "true"; + } + const imageTab = document.querySelector(`[data-testid="${tabTestId}"]`); + if (imageTab?.getAttribute("aria-selected") !== "true") { + return; + } + const imageVisible = Array.from(document.querySelectorAll('[role="img"]')).some( + (element) => element.getAttribute("aria-label") === alt && isVisible(element), + ); + if (!imageVisible) { + root.dataset.agentTabImageMissingFlash = "true"; + } + }; + const observer = new MutationObserver(inspect); + observer.observe(document.body, { + attributes: true, + childList: true, + subtree: true, + characterData: true, }); - if (errorVisible) { - root.dataset.agentTabImageErrorFlash = "true"; - } - }; - const observer = new MutationObserver(inspect); - observer.observe(document.body, { childList: true, subtree: true, characterData: true }); - window.setTimeout(() => observer.disconnect(), 2_000); - }, IMAGE_PREVIEW_ERROR); + window.setTimeout(() => observer.disconnect(), 5_000); + }, + { + alt: IMAGE_ALT, + errorText: IMAGE_PREVIEW_ERROR, + tabTestId: `workspace-tab-agent_${imageAgentId}`, + }, + ); } -async function expectNoObservedImagePreviewError(page: Page): Promise { +async function expectNoObservedImagePreviewInstability(page: Page): Promise { await page.waitForTimeout(250); expect(await page.locator("html").getAttribute("data-agent-tab-image-error-flash")).toBe("false"); + expect(await page.locator("html").getAttribute("data-agent-tab-image-missing-flash")).toBe( + "false", + ); +} + +async function switchToSettledAgentTab(page: Page, agent: ArchiveTabAgent): Promise { + const tab = page.getByRole("button", { name: agent.title, exact: true }); + await tab.click(); + await expect(page).toHaveTitle(agent.title); + await expect(page.locator('[data-testid="agent-chat-scroll"]:visible').first()).toBeVisible({ + timeout: 30_000, + }); + await expect(tab).toHaveAttribute("aria-selected", "true"); } test("switching between settled agent tabs keeps the image timeline valid", async ({ @@ -83,12 +122,12 @@ test("switching between settled agent tabs keeps the image timeline valid", asyn await openWorkspaceWithAgents(page, [otherAgent, imageAgent]); await expect(page.getByRole("img", { name: IMAGE_ALT })).toBeVisible({ timeout: 30_000 }); - await page.getByRole("button", { name: otherAgent.title, exact: true }).click(); - await observeImagePreviewErrors(page); - await page.getByRole("button", { name: imageAgent.title, exact: true }).click(); + await switchToSettledAgentTab(page, otherAgent); + await observeImagePreviewInstability(page, imageAgent.id); + await switchToSettledAgentTab(page, imageAgent); - await expect(page.getByRole("img", { name: IMAGE_ALT })).toHaveCount(1, { timeout: 30_000 }); - await expectNoObservedImagePreviewError(page); + await expect(page.getByRole("img", { name: IMAGE_ALT })).toBeVisible({ timeout: 30_000 }); + await expectNoObservedImagePreviewInstability(page); }); test("reopening an image timeline after editing another draft keeps its image available", async ({ @@ -116,9 +155,9 @@ test("reopening an image timeline after editing another draft keeps its image av .getByRole("textbox", { name: "Message agent..." }) .fill("Keep this draft while I check history"); await openSessions(page); - await observeImagePreviewErrors(page); + await observeImagePreviewInstability(page, imageAgent.id); await clickSessionRow(page, imageAgent.title); await expect(page.getByRole("img", { name: IMAGE_ALT })).toBeVisible({ timeout: 30_000 }); - await expectNoObservedImagePreviewError(page); + await expectNoObservedImagePreviewInstability(page); }); diff --git a/packages/app/e2e/agent-timeline-pagination.spec.ts b/packages/app/e2e/agent-timeline-pagination.spec.ts index 64a83ba434..0f562987f1 100644 --- a/packages/app/e2e/agent-timeline-pagination.spec.ts +++ b/packages/app/e2e/agent-timeline-pagination.spec.ts @@ -1,4 +1,4 @@ -import { test } from "./fixtures"; +import { expect, test } from "./fixtures"; import { expectSameOlderHistoryLoadingOperation, expectTimelineAtHistoryStart, @@ -86,6 +86,51 @@ test.describe("Agent timeline pagination", () => { } }); + test("finishes loading an older page while live output continues", async ({ page }) => { + test.setTimeout(120_000); + const agent = await seedLongMockAgentTimeline({ turns: 40 }); + try { + const history = await holdOlderHistoryPages(page, agent); + await openAgentTimeline(page, agent); + await userScrollsTimelineToHistoryStart(page); + await history.expectRequestedPages(1); + + await agent.client.sendAgentMessage(agent.agentId, "keep streaming while history settles"); + await agent.client.waitForAgentUpsert( + agent.agentId, + (snapshot) => snapshot.status === "running", + ); + history.releasePage(1); + + await expect(page.getByTestId("load-older-history-spinner")).toBeHidden({ timeout: 5_000 }); + const running = await agent.client.fetchAgents({ scope: "active" }); + expect(running.entries.find((entry) => entry.agent.id === agent.agentId)?.agent.status).toBe( + "running", + ); + } finally { + await agent.cleanup(); + } + }); + + test("keeps the visible timeline anchored when the final page finishes", async ({ page }) => { + test.setTimeout(120_000); + const agent = await seedLongMockAgentTimeline({ turns: 40 }); + try { + const history = await holdOlderHistoryPages(page, agent); + await openAgentTimeline(page, agent); + await userScrollsTimelineToHistoryStart(page); + await history.expectRequestedPages(1); + const position = await rememberTimelinePromptPosition(page, agent.initialTailOldestPrompt); + + history.releasePage(1); + + await expectTimelinePromptPositionPreserved(page, position); + await history.expectSettledWithRequestedPages(1); + } finally { + await agent.cleanup(); + } + }); + test("continues one loading operation while older pages still leave history start exposed", async ({ page, }) => { diff --git a/packages/app/src/agent-stream/history-start-settle-scheduler.test.ts b/packages/app/src/agent-stream/history-start-settle-scheduler.test.ts index d47650b057..2a8894299e 100644 --- a/packages/app/src/agent-stream/history-start-settle-scheduler.test.ts +++ b/packages/app/src/agent-stream/history-start-settle-scheduler.test.ts @@ -5,6 +5,7 @@ describe("history start settle scheduler", () => { it("does not restart its countdown when geometry keeps changing", () => { const frames = new Map void>(); let nextFrameId = 1; + const onFrame = vi.fn(); const onSettle = vi.fn(); const scheduler = createHistoryStartSettleScheduler({ settleFrames: 2, @@ -17,6 +18,7 @@ describe("history start settle scheduler", () => { cancelFrame: (id) => frames.delete(id), isSettling: () => true, isLoading: () => false, + onFrame, onSettle, }); const runNextFrame = () => { @@ -37,6 +39,7 @@ describe("history start settle scheduler", () => { runNextFrame(); expect(onSettle).toHaveBeenCalledTimes(1); + expect(onFrame).toHaveBeenCalledTimes(3); expect(frames.size).toBe(0); }); }); diff --git a/packages/app/src/agent-stream/history-start-settle-scheduler.ts b/packages/app/src/agent-stream/history-start-settle-scheduler.ts index 6c93472fca..3049ab8ed4 100644 --- a/packages/app/src/agent-stream/history-start-settle-scheduler.ts +++ b/packages/app/src/agent-stream/history-start-settle-scheduler.ts @@ -9,12 +9,14 @@ export function createHistoryStartSettleScheduler(input: { cancelFrame: (id: number) => void; isSettling: () => boolean; isLoading: () => boolean; + onFrame?: () => void; onSettle: () => void; }): HistoryStartSettleScheduler { let frameId: number | null = null; let remainingFrames = 0; const tick = () => { + input.onFrame?.(); if (!input.isSettling()) { frameId = null; remainingFrames = 0; diff --git a/packages/app/src/agent-stream/strategy-web.tsx b/packages/app/src/agent-stream/strategy-web.tsx index 1001754d2d..dd341543bd 100644 --- a/packages/app/src/agent-stream/strategy-web.tsx +++ b/packages/app/src/agent-stream/strategy-web.tsx @@ -26,6 +26,10 @@ import { type HistoryStartPaginationInput, type HistoryStartPaginationTransition, } from "./history-start-pagination"; +import { + createHistoryStartSettleScheduler, + type HistoryStartSettleScheduler, +} from "./history-start-settle-scheduler"; interface CreateWebStreamStrategyInput { isMobileBreakpoint: boolean; @@ -170,10 +174,11 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool const [historyStartPaginationState, setHistoryStartPaginationState] = useState( createHistoryStartPaginationState, ); + const [isHistoryStartSlotReserved, setIsHistoryStartSlotReserved] = useState(hasOlderHistory); const historyStartPaginationStateRef = useRef(historyStartPaginationState); const historyStartPrependAnchorRef = useRef(null); const historyStartPrependAnchorActiveRef = useRef(false); - const historyStartSettleFrameRef = useRef(null); + const historyStartSettleSchedulerRef = useRef(null); const shouldUseVirtualizer = segments.historyVirtualized.length > 0; const { renderHistoryVirtualizedRow, @@ -319,46 +324,41 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool lastKnownScrollTopRef.current = scrollContainer.scrollTop; }); const scheduleHistoryStartPrependSettle = useStableEvent(() => { - const pendingFrame = historyStartSettleFrameRef.current; - if (pendingFrame !== null) { - window.cancelAnimationFrame(pendingFrame); + let scheduler = historyStartSettleSchedulerRef.current; + if (!scheduler) { + scheduler = createHistoryStartSettleScheduler({ + settleFrames: HISTORY_START_SETTLE_FRAMES, + requestFrame: (callback) => window.requestAnimationFrame(callback), + cancelFrame: (frame) => window.cancelAnimationFrame(frame), + isSettling: () => + historyStartPaginationStateRef.current.status === "settling" && + historyStartPrependAnchorActiveRef.current, + isLoading: () => { + const input = getHistoryStartPaginationInput(); + return ( + !input || + input.isLoadingOlderHistory || + pendingVirtualRowMeasureFramesRef.current.size > 0 + ); + }, + onFrame: applyHistoryStartPrependAnchor, + onSettle: () => { + const input = getHistoryStartPaginationInput(); + if (!input) { + return; + } + historyStartPrependAnchorActiveRef.current = false; + const transition = settleHistoryStartPagination( + historyStartPaginationStateRef.current, + input, + ); + historyStartPrependAnchorRef.current = null; + applyHistoryStartPaginationTransition(transition); + }, + }); + historyStartSettleSchedulerRef.current = scheduler; } - let stableFrames = 0; - let previousGeometry = ""; - const sample = () => { - applyHistoryStartPrependAnchor(); - const scrollContainer = scrollContainerRef.current; - const input = getHistoryStartPaginationInput(); - if (!scrollContainer || !input || !historyStartPrependAnchorActiveRef.current) { - historyStartSettleFrameRef.current = null; - return; - } - const geometry = `${scrollContainer.scrollHeight}:${scrollContainer.scrollTop}`; - const hasPendingRowMeasurements = pendingVirtualRowMeasureFramesRef.current.size > 0; - if ( - input.isLoadingOlderHistory || - hasPendingRowMeasurements || - geometry !== previousGeometry - ) { - stableFrames = 0; - } else { - stableFrames += 1; - } - previousGeometry = geometry; - if (stableFrames < HISTORY_START_SETTLE_FRAMES) { - historyStartSettleFrameRef.current = window.requestAnimationFrame(sample); - return; - } - historyStartSettleFrameRef.current = null; - historyStartPrependAnchorActiveRef.current = false; - const transition = settleHistoryStartPagination( - historyStartPaginationStateRef.current, - input, - ); - historyStartPrependAnchorRef.current = null; - applyHistoryStartPaginationTransition(transition); - }; - historyStartSettleFrameRef.current = window.requestAnimationFrame(sample); + scheduler.schedule(); }); useLayoutEffect(() => { @@ -520,11 +520,8 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool return () => { window.cancelAnimationFrame(frame); historyStartReadyRef.current = false; - const settleFrame = historyStartSettleFrameRef.current; - if (settleFrame !== null) { - historyStartSettleFrameRef.current = null; - window.cancelAnimationFrame(settleFrame); - } + historyStartSettleSchedulerRef.current?.cancel(); + historyStartSettleSchedulerRef.current = null; }; }, [evaluateHistoryStart, props.agentId]); @@ -776,9 +773,14 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool const liveAuxiliary = useMemo(() => { return renderLiveAuxiliary(); }, [renderLiveAuxiliary]); + useEffect(() => { + if (hasOlderHistory || isHistoryStartLoadingOperation(historyStartPaginationState)) { + setIsHistoryStartSlotReserved(true); + } + }, [hasOlderHistory, historyStartPaginationState]); const historyStartSlot = useMemo(() => { const isLoadingOperation = isHistoryStartLoadingOperation(historyStartPaginationState); - if (!hasOlderHistory && !isLoadingOperation) { + if (!isHistoryStartSlotReserved && !hasOlderHistory && !isLoadingOperation) { return null; } return ( @@ -791,7 +793,7 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool ) : null} ); - }, [hasOlderHistory, historyStartPaginationState]); + }, [hasOlderHistory, historyStartPaginationState, isHistoryStartSlotReserved]); const shouldRenderEmpty = !boundary.hasMountedHistory && !boundary.hasVirtualizedHistory && diff --git a/packages/app/src/assistant-image/acquisition-cache.test.ts b/packages/app/src/assistant-image/acquisition-cache.test.ts index 099f88caee..6de05bd1ef 100644 --- a/packages/app/src/assistant-image/acquisition-cache.test.ts +++ b/packages/app/src/assistant-image/acquisition-cache.test.ts @@ -63,6 +63,7 @@ describe("assistant image acquisition cache", () => { cached: "persisted attachment", unavailableCalls: 0, }); + expect(cache.peek("message:image")).toBe("persisted attachment"); }); it("scopes file acquisitions to the rendered message occurrence", () => { diff --git a/packages/app/src/assistant-image/acquisition-cache.ts b/packages/app/src/assistant-image/acquisition-cache.ts index 6d21379bff..cbe3ade702 100644 --- a/packages/app/src/assistant-image/acquisition-cache.ts +++ b/packages/app/src/assistant-image/acquisition-cache.ts @@ -2,6 +2,7 @@ import { createPreviewAttachmentId } from "@/attachments/utils"; export interface AssistantImageAcquisitionCache { acquire(key: string, locate: () => Promise): Promise; + peek(key: string): T | undefined; size(): number; } @@ -49,6 +50,8 @@ export function createAssistantImageAcquisitionCache(input: { } interface CacheEntry { pending: Promise; + resolved: boolean; + value?: T; release: (() => void) | null; } const entries = new Map(); @@ -70,7 +73,7 @@ export function createAssistantImageAcquisitionCache(input: { return cached.pending; } const pending = locate(); - const entry: CacheEntry = { pending, release: null }; + const entry: CacheEntry = { pending, resolved: false, release: null }; entries.set(key, entry); if (entries.size > input.capacity) { const leastRecentlyUsedKey = entries.keys().next().value; @@ -84,8 +87,13 @@ export function createAssistantImageAcquisitionCache(input: { void (async () => { try { const value = await pending; + const release = input.onRetain?.(value) ?? null; if (entries.get(key) === entry) { - entry.release = input.onRetain?.(value) ?? null; + entry.value = value; + entry.resolved = true; + entry.release = release; + } else { + release?.(); } } catch { evict(key, entry); @@ -93,6 +101,15 @@ export function createAssistantImageAcquisitionCache(input: { })(); return pending; }, + peek(key) { + const entry = entries.get(key); + if (!entry?.resolved) { + return undefined; + } + entries.delete(key); + entries.set(key, entry); + return entry.value; + }, size() { return entries.size; }, diff --git a/packages/app/src/assistant-image/lifecycle.test.ts b/packages/app/src/assistant-image/lifecycle.test.ts index c1783f35d7..ed7729dd38 100644 --- a/packages/app/src/assistant-image/lifecycle.test.ts +++ b/packages/app/src/assistant-image/lifecycle.test.ts @@ -47,6 +47,27 @@ describe("assistant image lifecycle", () => { }); }); + it("does not reset a loaded image when the same preview is reported again", () => { + const loading = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), { + type: "preview_created", + uri: "blob:current", + aspectRatio: null, + }); + const loaded = transitionAssistantImageLifecycle(loading, { + type: "image_loaded", + uri: "blob:current", + aspectRatio: 1.5, + }); + + const repeated = transitionAssistantImageLifecycle(loaded, { + type: "preview_created", + uri: "blob:current", + aspectRatio: 1.5, + }); + + expect(repeated).toBe(loaded); + }); + it("publishes failed for a terminal image failure on the current URI", () => { const loading = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), { type: "preview_created", diff --git a/packages/app/src/assistant-image/lifecycle.ts b/packages/app/src/assistant-image/lifecycle.ts index 5093caf2d5..bf0db18c97 100644 --- a/packages/app/src/assistant-image/lifecycle.ts +++ b/packages/app/src/assistant-image/lifecycle.ts @@ -34,6 +34,9 @@ export function transitionAssistantImageLifecycle( return { status: "failed", message: event.message }; } if (event.type === "preview_created") { + if (state.status === "loaded" && state.uri === event.uri) { + return state; + } return { status: "loading", uri: event.uri, aspectRatio: event.aspectRatio }; } if (event.type === "preview_released") { diff --git a/packages/app/src/assistant-image/use-assistant-image.test.tsx b/packages/app/src/assistant-image/use-assistant-image.test.tsx new file mode 100644 index 0000000000..0ea29f9070 --- /dev/null +++ b/packages/app/src/assistant-image/use-assistant-image.test.tsx @@ -0,0 +1,104 @@ +/** + * @vitest-environment jsdom + */ +import { act, cleanup, renderHook, waitFor } from "@testing-library/react"; +import { afterEach, describe, expect, it } from "vitest"; +import type { ImageLoadEvent } from "react-native"; +import { setAssistantImageMetadata } from "@/utils/assistant-image-metadata"; +import { useAssistantImage } from "./use-assistant-image"; + +describe("useAssistantImage", () => { + afterEach(cleanup); + + it("keeps a successfully loaded image when cached dimensions are available", async () => { + const source = "https://example.com/expiring-image.png"; + setAssistantImageMetadata({ source }, { width: 640, height: 320 }); + const { result } = renderHook(() => + useAssistantImage({ source, occurrenceKey: "agent:message:image" }), + ); + + await waitFor(() => expect(result.current.status).toBe("loading")); + const binding = result.current.status === "failed" ? null : result.current.binding; + if (!binding) { + throw new Error("Expected a direct image render binding"); + } + act(() => { + binding.onLoad({ + nativeEvent: { source: { uri: source, width: 0, height: 0 } }, + } as ImageLoadEvent); + }); + + expect(result.current).toMatchObject({ status: "loaded", aspectRatio: 2 }); + }); + + it("uses browser image dimensions from the successful load event", async () => { + const source = "https://example.com/browser-image.png"; + const { result } = renderHook(() => + useAssistantImage({ source, occurrenceKey: "agent:message:browser-image" }), + ); + + await waitFor(() => expect(result.current.status).toBe("loading")); + const binding = result.current.status === "failed" ? null : result.current.binding; + if (!binding) { + throw new Error("Expected a direct image render binding"); + } + act(() => { + binding.onLoad({ + nativeEvent: { target: { naturalWidth: 800, naturalHeight: 400 } }, + } as unknown as ImageLoadEvent); + }); + + expect(result.current).toMatchObject({ status: "loaded", aspectRatio: 2 }); + }); + + it("uses the rendered browser image when React Native Web clears the load target", async () => { + const source = "https://example.com/react-native-web-image.png"; + const { result } = renderHook(() => + useAssistantImage({ source, occurrenceKey: "agent:message:react-native-web-image" }), + ); + + await waitFor(() => expect(result.current.status).toBe("loading")); + const binding = result.current.status === "failed" ? null : result.current.binding; + if (!binding) { + throw new Error("Expected a direct image render binding"); + } + const root = document.createElement("div"); + const image = document.createElement("img"); + Object.defineProperties(image, { + naturalWidth: { value: 900 }, + naturalHeight: { value: 600 }, + }); + root.append(image); + binding.onRef(root); + act(() => { + binding.onLoad({ nativeEvent: { target: null } } as unknown as ImageLoadEvent); + }); + + expect(result.current).toMatchObject({ status: "loaded", aspectRatio: 1.5 }); + }); + + it("restores a previously loaded URI without returning to loading", async () => { + const source = "https://example.com/remounted-image.png"; + const first = renderHook(() => + useAssistantImage({ source, occurrenceKey: "agent:message:remounted-image" }), + ); + await waitFor(() => expect(first.result.current.status).toBe("loading")); + const binding = first.result.current.status === "failed" ? null : first.result.current.binding; + if (!binding) { + throw new Error("Expected a direct image render binding"); + } + act(() => { + binding.onLoad({ + nativeEvent: { target: { naturalWidth: 800, naturalHeight: 400 } }, + } as unknown as ImageLoadEvent); + }); + expect(first.result.current.status).toBe("loaded"); + first.unmount(); + + const remounted = renderHook(() => + useAssistantImage({ source, occurrenceKey: "agent:message:remounted-image" }), + ); + + expect(remounted.result.current).toMatchObject({ status: "loaded", aspectRatio: 2 }); + }); +}); diff --git a/packages/app/src/assistant-image/use-assistant-image.ts b/packages/app/src/assistant-image/use-assistant-image.ts index e2955669d0..10e2432ea6 100644 --- a/packages/app/src/assistant-image/use-assistant-image.ts +++ b/packages/app/src/assistant-image/use-assistant-image.ts @@ -1,8 +1,10 @@ -import { useCallback, useEffect, useMemo, useReducer, useState } from "react"; -import { Image } from "react-native"; +import { useCallback, useEffect, useMemo, useReducer, useRef, useState } from "react"; +import type { ImageLoadEvent } from "react-native"; import { useTranslation } from "react-i18next"; import type { DaemonClient } from "@getpaseo/client/internal/daemon-client"; import type { AttachmentMetadata } from "@/attachments/types"; +import { isWeb } from "@/constants/platform"; +import { useStableEvent } from "@/hooks/use-stable-event"; import { retainAttachmentForGarbageCollection } from "@/attachments/gc-retention"; import { persistAttachmentFromBytes, @@ -36,10 +38,41 @@ import { runAssistantImageOperationWithRetry } from "./retry"; interface AssistantImageRenderBinding { uri: string; - onLoad: () => void; + onRef: (instance: unknown) => void; + onLoad: (event: ImageLoadEvent) => void; onError: () => void; } +function getImageLoadDimensions( + event: ImageLoadEvent, + renderedImage: unknown, +): { width: number; height: number } | null { + const nativeEvent = event.nativeEvent as ImageLoadEvent["nativeEvent"] & { + target?: { naturalWidth?: unknown; naturalHeight?: unknown }; + }; + const source = nativeEvent.source; + if (source && source.width > 0 && source.height > 0) { + return source; + } + const { naturalWidth, naturalHeight } = nativeEvent.target ?? {}; + if ( + typeof naturalWidth === "number" && + typeof naturalHeight === "number" && + naturalWidth > 0 && + naturalHeight > 0 + ) { + return { width: naturalWidth, height: naturalHeight }; + } + + if (isWeb && renderedImage instanceof HTMLElement) { + const image = renderedImage.querySelector("img"); + if (image && image.naturalWidth > 0 && image.naturalHeight > 0) { + return { width: image.naturalWidth, height: image.naturalHeight }; + } + } + return null; +} + export type AssistantImageResult = | { status: "loading"; @@ -89,6 +122,45 @@ const attachmentAcquisitionCache = createAssistantImageAcquisitionCache retainAttachmentForGarbageCollection(attachment.id), }); +interface CachedPreviewUrl { + attachment: AttachmentMetadata; + uri: string; +} + +const previewUrlCache = createAssistantImageAcquisitionCache({ + capacity: 500, + onRetain: + ({ attachment, uri }) => + () => { + void releaseAttachmentPreviewUrl({ attachment, url: uri }); + }, +}); + +const LOADED_IMAGE_CACHE_CAPACITY = 500; +const loadedImageCache = new Map(); + +function getLoadedImageAspectRatio(uri: string): number | null { + const aspectRatio = loadedImageCache.get(uri); + if (aspectRatio === undefined) { + return null; + } + loadedImageCache.delete(uri); + loadedImageCache.set(uri, aspectRatio); + return aspectRatio; +} + +function rememberLoadedImage(uri: string, aspectRatio: number): void { + loadedImageCache.delete(uri); + loadedImageCache.set(uri, aspectRatio); + if (loadedImageCache.size <= LOADED_IMAGE_CACHE_CAPACITY) { + return; + } + const leastRecentlyUsedUri = loadedImageCache.keys().next().value; + if (leastRecentlyUsedUri !== undefined) { + loadedImageCache.delete(leastRecentlyUsedUri); + } +} + function acquireAttachment(acquisition: AttachmentAcquisition): Promise { return attachmentAcquisitionCache.acquire(acquisition.key, acquisition.locate); } @@ -96,37 +168,72 @@ function acquireAttachment(acquisition: AttachmentAcquisition): Promise({ status: "waiting" }); + const acquisitionKey = acquisition?.key ?? null; + const [entry, setEntry] = useState<{ + key: string | null; + state: AttachmentAcquisitionState; + }>(() => { + const cached = acquisitionKey ? attachmentAcquisitionCache.peek(acquisitionKey) : undefined; + return { + key: acquisitionKey, + state: cached ? { status: "loaded", attachment: cached } : { status: "waiting" }, + }; + }); + const acquireCurrent = useStableEvent(async () => { + if (!acquisition) { + return null; + } + return await acquireAttachment(acquisition); + }); useEffect(() => { let disposed = false; - if (!acquisition) { - setState({ status: "waiting" }); + if (!acquisitionKey) { + setEntry({ key: null, state: { status: "waiting" } }); return; } - setState({ status: "loading" }); + const cached = attachmentAcquisitionCache.peek(acquisitionKey); + if (cached) { + setEntry({ key: acquisitionKey, state: { status: "loaded", attachment: cached } }); + return; + } + + setEntry({ key: acquisitionKey, state: { status: "loading" } }); void (async () => { try { const attachment = await runAssistantImageOperationWithRetry({ - operation: async () => await acquireAttachment(acquisition), + operation: async () => { + const current = await acquireCurrent(); + if (!current) { + throw new Error("Assistant image acquisition is unavailable."); + } + return current; + }, shouldStop: () => disposed, }); if (!disposed) { - setState({ status: "loaded", attachment }); + setEntry({ key: acquisitionKey, state: { status: "loaded", attachment } }); } } catch (error) { if (!disposed) { - setState({ status: "failed", error }); + setEntry({ key: acquisitionKey, state: { status: "failed", error } }); } } })(); return () => { disposed = true; }; - }, [acquisition]); + }, [acquireCurrent, acquisitionKey]); - return state; + if (!acquisitionKey) { + return { status: "waiting" }; + } + const cached = attachmentAcquisitionCache.peek(acquisitionKey); + if (cached) { + return { status: "loaded", attachment: cached }; + } + return entry.key === acquisitionKey ? entry.state : { status: "waiting" }; } function createFileAcquisition(input: { @@ -197,51 +304,72 @@ function createDataImageAcquisition(input: { } function usePreviewUrl(attachment: AttachmentMetadata | null | undefined): PreviewUrlState { - const [state, setState] = useState({ status: "waiting" }); const id = attachment?.id; const storageType = attachment?.storageType; const storageKey = attachment?.storageKey; const mimeType = attachment?.mimeType; + const previewKey = + id && storageType && storageKey && mimeType + ? `${id}:${storageType}:${storageKey}:${mimeType}` + : null; + const [entry, setEntry] = useState<{ key: string | null; state: PreviewUrlState }>(() => { + const cached = previewKey ? previewUrlCache.peek(previewKey) : undefined; + return { + key: previewKey, + state: cached ? { status: "loaded", uri: cached.uri } : { status: "waiting" }, + }; + }); + const getCurrentAttachment = useStableEvent(() => attachment ?? null); useEffect(() => { let disposed = false; - let previewUrl: string | null = null; - const current = attachment; + const current = getCurrentAttachment(); - if (!current) { - setState({ status: "waiting" }); + if (!current || !previewKey) { + setEntry({ key: null, state: { status: "waiting" } }); return; } - setState({ status: "loading" }); + const cached = previewUrlCache.peek(previewKey); + if (cached) { + setEntry({ key: previewKey, state: { status: "loaded", uri: cached.uri } }); + return; + } + + setEntry({ key: previewKey, state: { status: "loading" } }); void (async () => { try { - const uri = await runAssistantImageOperationWithRetry({ - operation: async () => await resolveAttachmentPreviewUrl(current), + const preview = await runAssistantImageOperationWithRetry({ + operation: async () => + await previewUrlCache.acquire(previewKey, async () => ({ + attachment: current, + uri: await resolveAttachmentPreviewUrl(current), + })), shouldStop: () => disposed, }); - if (disposed) { - await releaseAttachmentPreviewUrl({ attachment: current, url: uri }); - return; + if (!disposed) { + setEntry({ key: previewKey, state: { status: "loaded", uri: preview.uri } }); } - previewUrl = uri; - setState({ status: "loaded", uri }); } catch (error) { if (!disposed) { - setState({ status: "failed", error }); + setEntry({ key: previewKey, state: { status: "failed", error } }); } } })(); return () => { disposed = true; - if (previewUrl) { - void releaseAttachmentPreviewUrl({ attachment: current, url: previewUrl }); - } }; - }, [attachment, id, mimeType, storageKey, storageType]); + }, [getCurrentAttachment, previewKey]); - return state; + if (!previewKey) { + return { status: "waiting" }; + } + const cached = previewUrlCache.peek(previewKey); + if (cached) { + return { status: "loaded", uri: cached.uri }; + } + return entry.key === previewKey ? entry.state : { status: "waiting" }; } function lifecycleReducer( @@ -325,10 +453,6 @@ export function useAssistantImage({ const dataImagePreview = usePreviewUrl( dataImageAttachment.status === "loaded" ? dataImageAttachment.attachment : null, ); - const [lifecycle, dispatch] = useReducer(lifecycleReducer, undefined, () => - createAssistantImageLifecycle(), - ); - const directUri = resolution?.kind === "direct" && !dataImage ? resolution.uri : null; const preview = dataImage ? dataImagePreview : filePreview; const previewUri = preview.status === "loaded" ? preview.uri : null; @@ -337,6 +461,31 @@ export function useAssistantImage({ () => getAssistantImageMetadata({ source, workspaceRoot, serverId }), [serverId, source, workspaceRoot], ); + const [lifecycle, dispatchLifecycle] = useReducer( + lifecycleReducer, + uri, + (initialUri): AssistantImageLifecycle => { + if (initialUri) { + const aspectRatio = getLoadedImageAspectRatio(initialUri); + if (aspectRatio !== null) { + return { status: "loaded", uri: initialUri, aspectRatio }; + } + } + return createAssistantImageLifecycle(); + }, + ); + const dispatch = useCallback((event: AssistantImageLifecycleEvent) => { + if (event.type === "image_loaded") { + rememberLoadedImage(event.uri, event.aspectRatio); + } else if (event.type === "failed" && event.uri) { + loadedImageCache.delete(event.uri); + } + dispatchLifecycle(event); + }, []); + const renderedImageRef = useRef(null); + const handleImageRef = useCallback((instance: unknown) => { + renderedImageRef.current = instance; + }, []); useEffect(() => { if (!uri) { @@ -349,7 +498,7 @@ export function useAssistantImage({ uri, aspectRatio: cachedMetadata?.aspectRatio ?? null, }); - }, [cachedMetadata, uri]); + }, [cachedMetadata, dispatch, uri]); const handleImageError = useCallback(() => { if (uri) { @@ -359,37 +508,29 @@ export function useAssistantImage({ message: t("message.attachments.imageUnavailable"), }); } - }, [t, uri]); - const handleImageLoad = useCallback(() => { - if (!uri) { - return; - } - Image.getSize( - uri, - (width, height) => { - const metadata = setAssistantImageMetadata( - { source, workspaceRoot, serverId }, - { width, height }, - ); - if (!metadata) { - dispatch({ - type: "failed", - uri, - message: t("message.attachments.imageUnavailable"), - }); - return; - } - dispatch({ type: "image_loaded", uri, aspectRatio: metadata.aspectRatio }); - }, - () => { + }, [dispatch, t, uri]); + const handleImageLoad = useCallback( + (event: ImageLoadEvent) => { + if (!uri) { + return; + } + const dimensions = getImageLoadDimensions(event, renderedImageRef.current); + const metadata = dimensions + ? setAssistantImageMetadata({ source, workspaceRoot, serverId }, dimensions) + : null; + const aspectRatio = metadata?.aspectRatio ?? cachedMetadata?.aspectRatio ?? null; + if (!aspectRatio) { dispatch({ type: "failed", uri, message: t("message.attachments.imageUnavailable"), }); - }, - ); - }, [serverId, source, t, uri, workspaceRoot]); + return; + } + dispatch({ type: "image_loaded", uri, aspectRatio }); + }, + [cachedMetadata, dispatch, serverId, source, t, uri, workspaceRoot], + ); const acquisitionFailure = getAcquisitionFailure({ hasResolution: resolution !== null, @@ -409,6 +550,7 @@ export function useAssistantImage({ if (hasCurrentLifecycleUri && lifecycle.uri) { binding = { uri: lifecycle.uri, + onRef: handleImageRef, onLoad: handleImageLoad, onError: handleImageError, }; @@ -418,6 +560,7 @@ export function useAssistantImage({ status: "loaded", binding: { uri: lifecycle.uri, + onRef: handleImageRef, onLoad: handleImageLoad, onError: handleImageError, }, diff --git a/packages/app/src/components/message.tsx b/packages/app/src/components/message.tsx index 3c1cf35d07..8a85a86459 100644 --- a/packages/app/src/components/message.tsx +++ b/packages/app/src/components/message.tsx @@ -860,12 +860,12 @@ function AssistantMarkdownImage({ return ( - +