puter/paseo
1
0
Fork 0
mirror of https://github.com/getpaseo/paseo.git synced 2026-11-06 08:24:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Stabilize history settlement and image previews

Browse Source
This commit is contained in:
Mohamed Boudra
2026-07-28 16:45:39 +02:00
parent 0c26cc4d0e
commit d10786e40e
12 changed files with 519 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/e2e/agent-tab-image-stability.spec.ts
+64 -25
View File
@@ -36,31 +36,70 @@ async function createMockAgent(
return { id: agent.id, ...input };
}
async function observeImagePreviewErrors(page: Page): Promise<void> {
await page.evaluate((errorText) => {
const root = document.documentElement;
root.dataset.agentTabImageErrorFlash = "false";
const inspect = () => {
const errorVisible = Array.from(document.querySelectorAll("*")).some((element) => {
if (element.children.length > 0 || element.textContent?.trim() !== errorText) {
return false;
}
async function observeImagePreviewInstability(page: Page, imageAgentId: string): Promise<void> {
await page.evaluate(
({ alt, errorText, tabTestId }) => {
const root = document.documentElement;
root.dataset.agentTabImageErrorFlash = "false";
root.dataset.agentTabImageMissingFlash = "false";
const isVisible = (element: Element) => {
const rect = element.getBoundingClientRect();
return rect.width > 0 && rect.height > 0;
};
const inspect = () => {
const errorVisible = Array.from(document.querySelectorAll("*")).some((element) => {
if (element.children.length > 0 || element.textContent?.trim() !== errorText) {
return false;
}
return isVisible(element);
});
if (errorVisible) {
root.dataset.agentTabImageErrorFlash = "true";
}
const imageTab = document.querySelector(`[data-testid="${tabTestId}"]`);
if (imageTab?.getAttribute("aria-selected") !== "true") {
return;
}
const imageVisible = Array.from(document.querySelectorAll('[role="img"]')).some(
(element) => element.getAttribute("aria-label") === alt && isVisible(element),
);
if (!imageVisible) {
root.dataset.agentTabImageMissingFlash = "true";
}
};
const observer = new MutationObserver(inspect);
observer.observe(document.body, {
attributes: true,
childList: true,
subtree: true,
characterData: true,
});
if (errorVisible) {
root.dataset.agentTabImageErrorFlash = "true";
}
};
const observer = new MutationObserver(inspect);
observer.observe(document.body, { childList: true, subtree: true, characterData: true });
window.setTimeout(() => observer.disconnect(), 2_000);
}, IMAGE_PREVIEW_ERROR);
window.setTimeout(() => observer.disconnect(), 5_000);
},
{
alt: IMAGE_ALT,
errorText: IMAGE_PREVIEW_ERROR,
tabTestId: `workspace-tab-agent_${imageAgentId}`,
},
);
}
async function expectNoObservedImagePreviewError(page: Page): Promise<void> {
async function expectNoObservedImagePreviewInstability(page: Page): Promise<void> {
await page.waitForTimeout(250);
expect(await page.locator("html").getAttribute("data-agent-tab-image-error-flash")).toBe("false");
expect(await page.locator("html").getAttribute("data-agent-tab-image-missing-flash")).toBe(
"false",
);
}
async function switchToSettledAgentTab(page: Page, agent: ArchiveTabAgent): Promise<void> {
const tab = page.getByRole("button", { name: agent.title, exact: true });
await tab.click();
await expect(page).toHaveTitle(agent.title);
await expect(page.locator('[data-testid="agent-chat-scroll"]:visible').first()).toBeVisible({
timeout: 30_000,
});
await expect(tab).toHaveAttribute("aria-selected", "true");
}
test("switching between settled agent tabs keeps the image timeline valid", async ({
@@ -83,12 +122,12 @@ test("switching between settled agent tabs keeps the image timeline valid", asyn
await openWorkspaceWithAgents(page, [otherAgent, imageAgent]);
await expect(page.getByRole("img", { name: IMAGE_ALT })).toBeVisible({ timeout: 30_000 });
await page.getByRole("button", { name: otherAgent.title, exact: true }).click();
await observeImagePreviewErrors(page);
await page.getByRole("button", { name: imageAgent.title, exact: true }).click();
await switchToSettledAgentTab(page, otherAgent);
await observeImagePreviewInstability(page, imageAgent.id);
await switchToSettledAgentTab(page, imageAgent);
await expect(page.getByRole("img", { name: IMAGE_ALT })).toHaveCount(1, { timeout: 30_000 });
await expectNoObservedImagePreviewError(page);
await expect(page.getByRole("img", { name: IMAGE_ALT })).toBeVisible({ timeout: 30_000 });
await expectNoObservedImagePreviewInstability(page);
});
test("reopening an image timeline after editing another draft keeps its image available", async ({
@@ -116,9 +155,9 @@ test("reopening an image timeline after editing another draft keeps its image av
.getByRole("textbox", { name: "Message agent..." })
.fill("Keep this draft while I check history");
await openSessions(page);
await observeImagePreviewErrors(page);
await observeImagePreviewInstability(page, imageAgent.id);
await clickSessionRow(page, imageAgent.title);
await expect(page.getByRole("img", { name: IMAGE_ALT })).toBeVisible({ timeout: 30_000 });
await expectNoObservedImagePreviewError(page);
await expectNoObservedImagePreviewInstability(page);
});
packages/app/e2e/agent-timeline-pagination.spec.ts
+46 -1
View File
@@ -1,4 +1,4 @@
import { test } from "./fixtures";
import { expect, test } from "./fixtures";
import {
expectSameOlderHistoryLoadingOperation,
expectTimelineAtHistoryStart,
@@ -86,6 +86,51 @@ test.describe("Agent timeline pagination", () => {
}
});
test("finishes loading an older page while live output continues", async ({ page }) => {
test.setTimeout(120_000);
const agent = await seedLongMockAgentTimeline({ turns: 40 });
try {
const history = await holdOlderHistoryPages(page, agent);
await openAgentTimeline(page, agent);
await userScrollsTimelineToHistoryStart(page);
await history.expectRequestedPages(1);
await agent.client.sendAgentMessage(agent.agentId, "keep streaming while history settles");
await agent.client.waitForAgentUpsert(
agent.agentId,
(snapshot) => snapshot.status === "running",
);
history.releasePage(1);
await expect(page.getByTestId("load-older-history-spinner")).toBeHidden({ timeout: 5_000 });
const running = await agent.client.fetchAgents({ scope: "active" });
expect(running.entries.find((entry) => entry.agent.id === agent.agentId)?.agent.status).toBe(
"running",
);
} finally {
await agent.cleanup();
}
});
test("keeps the visible timeline anchored when the final page finishes", async ({ page }) => {
test.setTimeout(120_000);
const agent = await seedLongMockAgentTimeline({ turns: 40 });
try {
const history = await holdOlderHistoryPages(page, agent);
await openAgentTimeline(page, agent);
await userScrollsTimelineToHistoryStart(page);
await history.expectRequestedPages(1);
const position = await rememberTimelinePromptPosition(page, agent.initialTailOldestPrompt);
history.releasePage(1);
await expectTimelinePromptPositionPreserved(page, position);
await history.expectSettledWithRequestedPages(1);
} finally {
await agent.cleanup();
}
});
test("continues one loading operation while older pages still leave history start exposed", async ({
page,
}) => {
packages/app/src/agent-stream/history-start-settle-scheduler.test.ts
+3
View File
@@ -5,6 +5,7 @@ describe("history start settle scheduler", () => {
it("does not restart its countdown when geometry keeps changing", () => {
const frames = new Map<number, () => void>();
let nextFrameId = 1;
const onFrame = vi.fn();
const onSettle = vi.fn();
const scheduler = createHistoryStartSettleScheduler({
settleFrames: 2,
@@ -17,6 +18,7 @@ describe("history start settle scheduler", () => {
cancelFrame: (id) => frames.delete(id),
isSettling: () => true,
isLoading: () => false,
onFrame,
onSettle,
});
const runNextFrame = () => {
@@ -37,6 +39,7 @@ describe("history start settle scheduler", () => {
runNextFrame();
expect(onSettle).toHaveBeenCalledTimes(1);
expect(onFrame).toHaveBeenCalledTimes(3);
expect(frames.size).toBe(0);
});
});
packages/app/src/agent-stream/history-start-settle-scheduler.ts
+2
View File
@@ -9,12 +9,14 @@ export function createHistoryStartSettleScheduler(input: {
cancelFrame: (id: number) => void;
isSettling: () => boolean;
isLoading: () => boolean;
onFrame?: () => void;
onSettle: () => void;
}): HistoryStartSettleScheduler {
let frameId: number | null = null;
let remainingFrames = 0;
const tick = () => {
input.onFrame?.();
if (!input.isSettling()) {
frameId = null;
remainingFrames = 0;
packages/app/src/agent-stream/strategy-web.tsx
+49 -47
View File
@@ -26,6 +26,10 @@ import {
type HistoryStartPaginationInput,
type HistoryStartPaginationTransition,
} from "./history-start-pagination";
import {
createHistoryStartSettleScheduler,
type HistoryStartSettleScheduler,
} from "./history-start-settle-scheduler";
interface CreateWebStreamStrategyInput {
isMobileBreakpoint: boolean;
@@ -170,10 +174,11 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool
const [historyStartPaginationState, setHistoryStartPaginationState] = useState(
createHistoryStartPaginationState,
);
const [isHistoryStartSlotReserved, setIsHistoryStartSlotReserved] = useState(hasOlderHistory);
const historyStartPaginationStateRef = useRef(historyStartPaginationState);
const historyStartPrependAnchorRef = useRef<HistoryStartPrependAnchor | null>(null);
const historyStartPrependAnchorActiveRef = useRef(false);
const historyStartSettleFrameRef = useRef<number | null>(null);
const historyStartSettleSchedulerRef = useRef<HistoryStartSettleScheduler | null>(null);
const shouldUseVirtualizer = segments.historyVirtualized.length > 0;
const {
renderHistoryVirtualizedRow,
@@ -319,46 +324,41 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool
lastKnownScrollTopRef.current = scrollContainer.scrollTop;
});
const scheduleHistoryStartPrependSettle = useStableEvent(() => {
const pendingFrame = historyStartSettleFrameRef.current;
if (pendingFrame !== null) {
window.cancelAnimationFrame(pendingFrame);
let scheduler = historyStartSettleSchedulerRef.current;
if (!scheduler) {
scheduler = createHistoryStartSettleScheduler({
settleFrames: HISTORY_START_SETTLE_FRAMES,
requestFrame: (callback) => window.requestAnimationFrame(callback),
cancelFrame: (frame) => window.cancelAnimationFrame(frame),
isSettling: () =>
historyStartPaginationStateRef.current.status === "settling" &&
historyStartPrependAnchorActiveRef.current,
isLoading: () => {
const input = getHistoryStartPaginationInput();
return (
!input ||
input.isLoadingOlderHistory ||
pendingVirtualRowMeasureFramesRef.current.size > 0
);
},
onFrame: applyHistoryStartPrependAnchor,
onSettle: () => {
const input = getHistoryStartPaginationInput();
if (!input) {
return;
}
historyStartPrependAnchorActiveRef.current = false;
const transition = settleHistoryStartPagination(
historyStartPaginationStateRef.current,
input,
);
historyStartPrependAnchorRef.current = null;
applyHistoryStartPaginationTransition(transition);
},
});
historyStartSettleSchedulerRef.current = scheduler;
}
let stableFrames = 0;
let previousGeometry = "";
const sample = () => {
applyHistoryStartPrependAnchor();
const scrollContainer = scrollContainerRef.current;
const input = getHistoryStartPaginationInput();
if (!scrollContainer || !input || !historyStartPrependAnchorActiveRef.current) {
historyStartSettleFrameRef.current = null;
return;
}
const geometry = `${scrollContainer.scrollHeight}:${scrollContainer.scrollTop}`;
const hasPendingRowMeasurements = pendingVirtualRowMeasureFramesRef.current.size > 0;
if (
input.isLoadingOlderHistory ||
hasPendingRowMeasurements ||
geometry !== previousGeometry
) {
stableFrames = 0;
} else {
stableFrames += 1;
}
previousGeometry = geometry;
if (stableFrames < HISTORY_START_SETTLE_FRAMES) {
historyStartSettleFrameRef.current = window.requestAnimationFrame(sample);
return;
}
historyStartSettleFrameRef.current = null;
historyStartPrependAnchorActiveRef.current = false;
const transition = settleHistoryStartPagination(
historyStartPaginationStateRef.current,
input,
);
historyStartPrependAnchorRef.current = null;
applyHistoryStartPaginationTransition(transition);
};
historyStartSettleFrameRef.current = window.requestAnimationFrame(sample);
scheduler.schedule();
});
useLayoutEffect(() => {
@@ -520,11 +520,8 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool
return () => {
window.cancelAnimationFrame(frame);
historyStartReadyRef.current = false;
const settleFrame = historyStartSettleFrameRef.current;
if (settleFrame !== null) {
historyStartSettleFrameRef.current = null;
window.cancelAnimationFrame(settleFrame);
}
historyStartSettleSchedulerRef.current?.cancel();
historyStartSettleSchedulerRef.current = null;
};
}, [evaluateHistoryStart, props.agentId]);
@@ -776,9 +773,14 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool
const liveAuxiliary = useMemo(() => {
return renderLiveAuxiliary();
}, [renderLiveAuxiliary]);
useEffect(() => {
if (hasOlderHistory || isHistoryStartLoadingOperation(historyStartPaginationState)) {
setIsHistoryStartSlotReserved(true);
}
}, [hasOlderHistory, historyStartPaginationState]);
const historyStartSlot = useMemo(() => {
const isLoadingOperation = isHistoryStartLoadingOperation(historyStartPaginationState);
if (!hasOlderHistory && !isLoadingOperation) {
if (!isHistoryStartSlotReserved && !hasOlderHistory && !isLoadingOperation) {
return null;
}
return (
@@ -791,7 +793,7 @@ function WebStreamViewport(props: StreamRenderInput & { isMobileBreakpoint: bool
) : null}
</div>
);
}, [hasOlderHistory, historyStartPaginationState]);
}, [hasOlderHistory, historyStartPaginationState, isHistoryStartSlotReserved]);
const shouldRenderEmpty =
!boundary.hasMountedHistory &&
!boundary.hasVirtualizedHistory &&
packages/app/src/assistant-image/acquisition-cache.test.ts
+1
View File
@@ -63,6 +63,7 @@ describe("assistant image acquisition cache", () => {
cached: "persisted attachment",
unavailableCalls: 0,
});
expect(cache.peek("message:image")).toBe("persisted attachment");
});
it("scopes file acquisitions to the rendered message occurrence", () => {
packages/app/src/assistant-image/acquisition-cache.ts
+19 -2
View File
@@ -2,6 +2,7 @@ import { createPreviewAttachmentId } from "@/attachments/utils";
export interface AssistantImageAcquisitionCache<T> {
acquire(key: string, locate: () => Promise<T>): Promise<T>;
peek(key: string): T | undefined;
size(): number;
}
@@ -49,6 +50,8 @@ export function createAssistantImageAcquisitionCache<T>(input: {
}
interface CacheEntry {
pending: Promise<T>;
resolved: boolean;
value?: T;
release: (() => void) | null;
}
const entries = new Map<string, CacheEntry>();
@@ -70,7 +73,7 @@ export function createAssistantImageAcquisitionCache<T>(input: {
return cached.pending;
}
const pending = locate();
const entry: CacheEntry = { pending, release: null };
const entry: CacheEntry = { pending, resolved: false, release: null };
entries.set(key, entry);
if (entries.size > input.capacity) {
const leastRecentlyUsedKey = entries.keys().next().value;
@@ -84,8 +87,13 @@ export function createAssistantImageAcquisitionCache<T>(input: {
void (async () => {
try {
const value = await pending;
const release = input.onRetain?.(value) ?? null;
if (entries.get(key) === entry) {
entry.release = input.onRetain?.(value) ?? null;
entry.value = value;
entry.resolved = true;
entry.release = release;
} else {
release?.();
}
} catch {
evict(key, entry);
@@ -93,6 +101,15 @@ export function createAssistantImageAcquisitionCache<T>(input: {
})();
return pending;
},
peek(key) {
const entry = entries.get(key);
if (!entry?.resolved) {
return undefined;
}
entries.delete(key);
entries.set(key, entry);
return entry.value;
},
size() {
return entries.size;
},
packages/app/src/assistant-image/lifecycle.test.ts
+21
View File
@@ -47,6 +47,27 @@ describe("assistant image lifecycle", () => {
});
});
it("does not reset a loaded image when the same preview is reported again", () => {
const loading = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), {
type: "preview_created",
uri: "blob:current",
aspectRatio: null,
});
const loaded = transitionAssistantImageLifecycle(loading, {
type: "image_loaded",
uri: "blob:current",
aspectRatio: 1.5,
});
const repeated = transitionAssistantImageLifecycle(loaded, {
type: "preview_created",
uri: "blob:current",
aspectRatio: 1.5,
});
expect(repeated).toBe(loaded);
});
it("publishes failed for a terminal image failure on the current URI", () => {
const loading = transitionAssistantImageLifecycle(createAssistantImageLifecycle(), {
type: "preview_created",
packages/app/src/assistant-image/lifecycle.ts
+3
View File
@@ -34,6 +34,9 @@ export function transitionAssistantImageLifecycle(
return { status: "failed", message: event.message };
}
if (event.type === "preview_created") {
if (state.status === "loaded" && state.uri === event.uri) {
return state;
}
return { status: "loading", uri: event.uri, aspectRatio: event.aspectRatio };
}
if (event.type === "preview_released") {
packages/app/src/assistant-image/use-assistant-image.test.tsx
+104
View File
@@ -0,0 +1,104 @@
/**
* @vitest-environment jsdom
*/
import { act, cleanup, renderHook, waitFor } from "@testing-library/react";
import { afterEach, describe, expect, it } from "vitest";
import type { ImageLoadEvent } from "react-native";
import { setAssistantImageMetadata } from "@/utils/assistant-image-metadata";
import { useAssistantImage } from "./use-assistant-image";
describe("useAssistantImage", () => {
afterEach(cleanup);
it("keeps a successfully loaded image when cached dimensions are available", async () => {
const source = "https://example.com/expiring-image.png";
setAssistantImageMetadata({ source }, { width: 640, height: 320 });
const { result } = renderHook(() =>
useAssistantImage({ source, occurrenceKey: "agent:message:image" }),
);
await waitFor(() => expect(result.current.status).toBe("loading"));
const binding = result.current.status === "failed" ? null : result.current.binding;
if (!binding) {
throw new Error("Expected a direct image render binding");
}
act(() => {
binding.onLoad({
nativeEvent: { source: { uri: source, width: 0, height: 0 } },
} as ImageLoadEvent);
});
expect(result.current).toMatchObject({ status: "loaded", aspectRatio: 2 });
});
it("uses browser image dimensions from the successful load event", async () => {
const source = "https://example.com/browser-image.png";
const { result } = renderHook(() =>
useAssistantImage({ source, occurrenceKey: "agent:message:browser-image" }),
);
await waitFor(() => expect(result.current.status).toBe("loading"));
const binding = result.current.status === "failed" ? null : result.current.binding;
if (!binding) {
throw new Error("Expected a direct image render binding");
}
act(() => {
binding.onLoad({
nativeEvent: { target: { naturalWidth: 800, naturalHeight: 400 } },
} as unknown as ImageLoadEvent);
});
expect(result.current).toMatchObject({ status: "loaded", aspectRatio: 2 });
});
it("uses the rendered browser image when React Native Web clears the load target", async () => {
const source = "https://example.com/react-native-web-image.png";
const { result } = renderHook(() =>
useAssistantImage({ source, occurrenceKey: "agent:message:react-native-web-image" }),
);
await waitFor(() => expect(result.current.status).toBe("loading"));
const binding = result.current.status === "failed" ? null : result.current.binding;
if (!binding) {
throw new Error("Expected a direct image render binding");
}
const root = document.createElement("div");
const image = document.createElement("img");
Object.defineProperties(image, {
naturalWidth: { value: 900 },
naturalHeight: { value: 600 },
});
root.append(image);
binding.onRef(root);
act(() => {
binding.onLoad({ nativeEvent: { target: null } } as unknown as ImageLoadEvent);
});
expect(result.current).toMatchObject({ status: "loaded", aspectRatio: 1.5 });
});
it("restores a previously loaded URI without returning to loading", async () => {
const source = "https://example.com/remounted-image.png";
const first = renderHook(() =>
useAssistantImage({ source, occurrenceKey: "agent:message:remounted-image" }),
);
await waitFor(() => expect(first.result.current.status).toBe("loading"));
const binding = first.result.current.status === "failed" ? null : first.result.current.binding;
if (!binding) {
throw new Error("Expected a direct image render binding");
}
act(() => {
binding.onLoad({
nativeEvent: { target: { naturalWidth: 800, naturalHeight: 400 } },
} as unknown as ImageLoadEvent);
});
expect(first.result.current.status).toBe("loaded");
first.unmount();
const remounted = renderHook(() =>
useAssistantImage({ source, occurrenceKey: "agent:message:remounted-image" }),
);
expect(remounted.result.current).toMatchObject({ status: "loaded", aspectRatio: 2 });
});
});
packages/app/src/assistant-image/use-assistant-image.ts
+205 -62
View File
@@ -1,8 +1,10 @@
import { useCallback, useEffect, useMemo, useReducer, useState } from "react";
import { Image } from "react-native";
import { useCallback, useEffect, useMemo, useReducer, useRef, useState } from "react";
import type { ImageLoadEvent } from "react-native";
import { useTranslation } from "react-i18next";
import type { DaemonClient } from "@getpaseo/client/internal/daemon-client";
import type { AttachmentMetadata } from "@/attachments/types";
import { isWeb } from "@/constants/platform";
import { useStableEvent } from "@/hooks/use-stable-event";
import { retainAttachmentForGarbageCollection } from "@/attachments/gc-retention";
import {
persistAttachmentFromBytes,
@@ -36,10 +38,41 @@ import { runAssistantImageOperationWithRetry } from "./retry";
interface AssistantImageRenderBinding {
uri: string;
onLoad: () => void;
onRef: (instance: unknown) => void;
onLoad: (event: ImageLoadEvent) => void;
onError: () => void;
}
function getImageLoadDimensions(
event: ImageLoadEvent,
renderedImage: unknown,
): { width: number; height: number } | null {
const nativeEvent = event.nativeEvent as ImageLoadEvent["nativeEvent"] & {
target?: { naturalWidth?: unknown; naturalHeight?: unknown };
};
const source = nativeEvent.source;
if (source && source.width > 0 && source.height > 0) {
return source;
}
const { naturalWidth, naturalHeight } = nativeEvent.target ?? {};
if (
typeof naturalWidth === "number" &&
typeof naturalHeight === "number" &&
naturalWidth > 0 &&
naturalHeight > 0
) {
return { width: naturalWidth, height: naturalHeight };
}
if (isWeb && renderedImage instanceof HTMLElement) {
const image = renderedImage.querySelector("img");
if (image && image.naturalWidth > 0 && image.naturalHeight > 0) {
return { width: image.naturalWidth, height: image.naturalHeight };
}
}
return null;
}
export type AssistantImageResult =
| {
status: "loading";
@@ -89,6 +122,45 @@ const attachmentAcquisitionCache = createAssistantImageAcquisitionCache<Attachme
onRetain: (attachment) => retainAttachmentForGarbageCollection(attachment.id),
});
interface CachedPreviewUrl {
attachment: AttachmentMetadata;
uri: string;
}
const previewUrlCache = createAssistantImageAcquisitionCache<CachedPreviewUrl>({
capacity: 500,
onRetain:
({ attachment, uri }) =>
() => {
void releaseAttachmentPreviewUrl({ attachment, url: uri });
},
});
const LOADED_IMAGE_CACHE_CAPACITY = 500;
const loadedImageCache = new Map<string, number>();
function getLoadedImageAspectRatio(uri: string): number | null {
const aspectRatio = loadedImageCache.get(uri);
if (aspectRatio === undefined) {
return null;
}
loadedImageCache.delete(uri);
loadedImageCache.set(uri, aspectRatio);
return aspectRatio;
}
function rememberLoadedImage(uri: string, aspectRatio: number): void {
loadedImageCache.delete(uri);
loadedImageCache.set(uri, aspectRatio);
if (loadedImageCache.size <= LOADED_IMAGE_CACHE_CAPACITY) {
return;
}
const leastRecentlyUsedUri = loadedImageCache.keys().next().value;
if (leastRecentlyUsedUri !== undefined) {
loadedImageCache.delete(leastRecentlyUsedUri);
}
}
function acquireAttachment(acquisition: AttachmentAcquisition): Promise<AttachmentMetadata> {
return attachmentAcquisitionCache.acquire(acquisition.key, acquisition.locate);
}
@@ -96,37 +168,72 @@ function acquireAttachment(acquisition: AttachmentAcquisition): Promise<Attachme
function useAttachmentAcquisition(
acquisition: AttachmentAcquisition | null,
): AttachmentAcquisitionState {
const [state, setState] = useState<AttachmentAcquisitionState>({ status: "waiting" });
const acquisitionKey = acquisition?.key ?? null;
const [entry, setEntry] = useState<{
key: string | null;
state: AttachmentAcquisitionState;
}>(() => {
const cached = acquisitionKey ? attachmentAcquisitionCache.peek(acquisitionKey) : undefined;
return {
key: acquisitionKey,
state: cached ? { status: "loaded", attachment: cached } : { status: "waiting" },
};
});
const acquireCurrent = useStableEvent(async () => {
if (!acquisition) {
return null;
}
return await acquireAttachment(acquisition);
});
useEffect(() => {
let disposed = false;
if (!acquisition) {
setState({ status: "waiting" });
if (!acquisitionKey) {
setEntry({ key: null, state: { status: "waiting" } });
return;
}
setState({ status: "loading" });
const cached = attachmentAcquisitionCache.peek(acquisitionKey);
if (cached) {
setEntry({ key: acquisitionKey, state: { status: "loaded", attachment: cached } });
return;
}
setEntry({ key: acquisitionKey, state: { status: "loading" } });
void (async () => {
try {
const attachment = await runAssistantImageOperationWithRetry({
operation: async () => await acquireAttachment(acquisition),
operation: async () => {
const current = await acquireCurrent();
if (!current) {
throw new Error("Assistant image acquisition is unavailable.");
}
return current;
},
shouldStop: () => disposed,
});
if (!disposed) {
setState({ status: "loaded", attachment });
setEntry({ key: acquisitionKey, state: { status: "loaded", attachment } });
}
} catch (error) {
if (!disposed) {
setState({ status: "failed", error });
setEntry({ key: acquisitionKey, state: { status: "failed", error } });
}
}
})();
return () => {
disposed = true;
};
}, [acquisition]);
}, [acquireCurrent, acquisitionKey]);
return state;
if (!acquisitionKey) {
return { status: "waiting" };
}
const cached = attachmentAcquisitionCache.peek(acquisitionKey);
if (cached) {
return { status: "loaded", attachment: cached };
}
return entry.key === acquisitionKey ? entry.state : { status: "waiting" };
}
function createFileAcquisition(input: {
@@ -197,51 +304,72 @@ function createDataImageAcquisition(input: {
}
function usePreviewUrl(attachment: AttachmentMetadata | null | undefined): PreviewUrlState {
const [state, setState] = useState<PreviewUrlState>({ status: "waiting" });
const id = attachment?.id;
const storageType = attachment?.storageType;
const storageKey = attachment?.storageKey;
const mimeType = attachment?.mimeType;
const previewKey =
id && storageType && storageKey && mimeType
? `${id}:${storageType}:${storageKey}:${mimeType}`
: null;
const [entry, setEntry] = useState<{ key: string | null; state: PreviewUrlState }>(() => {
const cached = previewKey ? previewUrlCache.peek(previewKey) : undefined;
return {
key: previewKey,
state: cached ? { status: "loaded", uri: cached.uri } : { status: "waiting" },
};
});
const getCurrentAttachment = useStableEvent(() => attachment ?? null);
useEffect(() => {
let disposed = false;
let previewUrl: string | null = null;
const current = attachment;
const current = getCurrentAttachment();
if (!current) {
setState({ status: "waiting" });
if (!current || !previewKey) {
setEntry({ key: null, state: { status: "waiting" } });
return;
}
setState({ status: "loading" });
const cached = previewUrlCache.peek(previewKey);
if (cached) {
setEntry({ key: previewKey, state: { status: "loaded", uri: cached.uri } });
return;
}
setEntry({ key: previewKey, state: { status: "loading" } });
void (async () => {
try {
const uri = await runAssistantImageOperationWithRetry({
operation: async () => await resolveAttachmentPreviewUrl(current),
const preview = await runAssistantImageOperationWithRetry({
operation: async () =>
await previewUrlCache.acquire(previewKey, async () => ({
attachment: current,
uri: await resolveAttachmentPreviewUrl(current),
})),
shouldStop: () => disposed,
});
if (disposed) {
await releaseAttachmentPreviewUrl({ attachment: current, url: uri });
return;
if (!disposed) {
setEntry({ key: previewKey, state: { status: "loaded", uri: preview.uri } });
}
previewUrl = uri;
setState({ status: "loaded", uri });
} catch (error) {
if (!disposed) {
setState({ status: "failed", error });
setEntry({ key: previewKey, state: { status: "failed", error } });
}
}
})();
return () => {
disposed = true;
if (previewUrl) {
void releaseAttachmentPreviewUrl({ attachment: current, url: previewUrl });
}
};
}, [attachment, id, mimeType, storageKey, storageType]);
}, [getCurrentAttachment, previewKey]);
return state;
if (!previewKey) {
return { status: "waiting" };
}
const cached = previewUrlCache.peek(previewKey);
if (cached) {
return { status: "loaded", uri: cached.uri };
}
return entry.key === previewKey ? entry.state : { status: "waiting" };
}
function lifecycleReducer(
@@ -325,10 +453,6 @@ export function useAssistantImage({
const dataImagePreview = usePreviewUrl(
dataImageAttachment.status === "loaded" ? dataImageAttachment.attachment : null,
);
const [lifecycle, dispatch] = useReducer(lifecycleReducer, undefined, () =>
createAssistantImageLifecycle(),
);
const directUri = resolution?.kind === "direct" && !dataImage ? resolution.uri : null;
const preview = dataImage ? dataImagePreview : filePreview;
const previewUri = preview.status === "loaded" ? preview.uri : null;
@@ -337,6 +461,31 @@ export function useAssistantImage({
() => getAssistantImageMetadata({ source, workspaceRoot, serverId }),
[serverId, source, workspaceRoot],
);
const [lifecycle, dispatchLifecycle] = useReducer(
lifecycleReducer,
uri,
(initialUri): AssistantImageLifecycle => {
if (initialUri) {
const aspectRatio = getLoadedImageAspectRatio(initialUri);
if (aspectRatio !== null) {
return { status: "loaded", uri: initialUri, aspectRatio };
}
}
return createAssistantImageLifecycle();
},
);
const dispatch = useCallback((event: AssistantImageLifecycleEvent) => {
if (event.type === "image_loaded") {
rememberLoadedImage(event.uri, event.aspectRatio);
} else if (event.type === "failed" && event.uri) {
loadedImageCache.delete(event.uri);
}
dispatchLifecycle(event);
}, []);
const renderedImageRef = useRef<unknown>(null);
const handleImageRef = useCallback((instance: unknown) => {
renderedImageRef.current = instance;
}, []);
useEffect(() => {
if (!uri) {
@@ -349,7 +498,7 @@ export function useAssistantImage({
uri,
aspectRatio: cachedMetadata?.aspectRatio ?? null,
});
}, [cachedMetadata, uri]);
}, [cachedMetadata, dispatch, uri]);
const handleImageError = useCallback(() => {
if (uri) {
@@ -359,37 +508,29 @@ export function useAssistantImage({
message: t("message.attachments.imageUnavailable"),
});
}
}, [t, uri]);
const handleImageLoad = useCallback(() => {
if (!uri) {
return;
}
Image.getSize(
uri,
(width, height) => {
const metadata = setAssistantImageMetadata(
{ source, workspaceRoot, serverId },
{ width, height },
);
if (!metadata) {
dispatch({
type: "failed",
uri,
message: t("message.attachments.imageUnavailable"),
});
return;
}
dispatch({ type: "image_loaded", uri, aspectRatio: metadata.aspectRatio });
},
() => {
}, [dispatch, t, uri]);
const handleImageLoad = useCallback(
(event: ImageLoadEvent) => {
if (!uri) {
return;
}
const dimensions = getImageLoadDimensions(event, renderedImageRef.current);
const metadata = dimensions
? setAssistantImageMetadata({ source, workspaceRoot, serverId }, dimensions)
: null;
const aspectRatio = metadata?.aspectRatio ?? cachedMetadata?.aspectRatio ?? null;
if (!aspectRatio) {
dispatch({
type: "failed",
uri,
message: t("message.attachments.imageUnavailable"),
});
},
);
}, [serverId, source, t, uri, workspaceRoot]);
return;
}
dispatch({ type: "image_loaded", uri, aspectRatio });
},
[cachedMetadata, dispatch, serverId, source, t, uri, workspaceRoot],
);
const acquisitionFailure = getAcquisitionFailure({
hasResolution: resolution !== null,
@@ -409,6 +550,7 @@ export function useAssistantImage({
if (hasCurrentLifecycleUri && lifecycle.uri) {
binding = {
uri: lifecycle.uri,
onRef: handleImageRef,
onLoad: handleImageLoad,
onError: handleImageError,
};
@@ -418,6 +560,7 @@ export function useAssistantImage({
status: "loaded",
binding: {
uri: lifecycle.uri,
onRef: handleImageRef,
onLoad: handleImageLoad,
onError: handleImageError,
},
packages/app/src/components/message.tsx
+2 -2
View File
@@ -860,12 +860,12 @@ function AssistantMarkdownImage({
return (
<View style={frameStyle}>
<View style={surfaceStyle}>
<View style={surfaceStyle} accessibilityRole="image" accessibilityLabel={alt}>
<Image
ref={binding.onRef}
source={imageSource}
style={assistantMessageStylesheet.image}
resizeMode="contain"
accessibilityLabel={alt}
onLoad={binding.onLoad}
onError={binding.onError}
/>