diff --git a/packages/app/src/assistant-image/file-acquisition.test.ts b/packages/app/src/assistant-image/file-acquisition.test.ts new file mode 100644 index 0000000000..552609e75f --- /dev/null +++ b/packages/app/src/assistant-image/file-acquisition.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import type { AttachmentMetadata } from "@/attachments/types"; +import { + createAssistantImageFileAcquisition, + type AssistantImageFileAcquisitionPort, +} from "./file-acquisition"; + +class MemoryFileAcquisitionPort implements AssistantImageFileAcquisitionPort { + readonly reads: Array<{ cwd: string; path: string }> = []; + + async readFile(cwd: string, path: string) { + this.reads.push({ cwd, path }); + return { + kind: "image" as const, + path, + mime: "image/png", + size: 4, + modifiedAt: "1", + bytes: new Uint8Array([1, 2, 3, 4]), + }; + } + + async persist(input: { id: string; mimeType: string; fileName: string | null }) { + return { + id: input.id, + mimeType: input.mimeType, + storageType: "web-indexeddb" as const, + storageKey: input.id, + fileName: input.fileName, + byteSize: 4, + createdAt: 1, + } satisfies AttachmentMetadata; + } +} + +describe("assistant image file acquisition", () => { + it("recreates the same acquisition with a live port after reconnect", async () => { + const common = { + resolution: { kind: "file_rpc" as const, cwd: "/workspace", path: "reconnect.png" }, + serverId: "server", + occurrenceKey: "agent:message:reconnect-image", + unavailableMessage: "Image unavailable", + }; + const disconnected = createAssistantImageFileAcquisition({ ...common, port: null }); + const connectedPort = new MemoryFileAcquisitionPort(); + const connected = createAssistantImageFileAcquisition({ ...common, port: connectedPort }); + + expect(disconnected?.key).toBe(connected?.key); + await expect(disconnected?.locate()).rejects.toThrow("Image unavailable"); + await expect(connected?.locate()).resolves.toMatchObject({ mimeType: "image/png" }); + expect(connectedPort.reads).toEqual([{ cwd: "/workspace", path: "reconnect.png" }]); + }); +}); diff --git a/packages/app/src/assistant-image/file-acquisition.ts b/packages/app/src/assistant-image/file-acquisition.ts new file mode 100644 index 0000000000..f7a80e8536 --- /dev/null +++ b/packages/app/src/assistant-image/file-acquisition.ts @@ -0,0 +1,67 @@ +import type { FileReadResult } from "@getpaseo/client/internal/daemon-client"; +import type { AttachmentMetadata } from "@/attachments/types"; +import { getFileNameFromPath } from "@/attachments/utils"; +import type { AssistantImageSourceResolution } from "@/utils/assistant-image-source"; +import { + createAssistantImageFileAcquisitionKey, + createAssistantImageFilePreviewAttachmentId, +} from "./acquisition-cache"; + +export interface AssistantImageFileAcquisitionPort { + readFile(cwd: string, path: string): Promise; + persist(input: { + id: string; + bytes: Uint8Array; + mimeType: string; + fileName: string | null; + }): Promise; +} + +export interface AssistantImageAcquisition { + key: string; + locate: () => Promise; +} + +export function createAssistantImageFileAcquisition(input: { + port: AssistantImageFileAcquisitionPort | null; + resolution: AssistantImageSourceResolution | null; + serverId?: string; + occurrenceKey: string; + unavailableMessage: string; +}): AssistantImageAcquisition | null { + if (input.resolution?.kind !== "file_rpc") { + return null; + } + const { port, resolution } = input; + return { + key: createAssistantImageFileAcquisitionKey({ + serverId: input.serverId, + occurrenceKey: input.occurrenceKey, + cwd: resolution.cwd, + path: resolution.path, + }), + locate: async () => { + if (!port) { + throw new Error(input.unavailableMessage); + } + const file = await port.readFile(resolution.cwd, resolution.path); + if (file.kind !== "image") { + throw new Error(input.unavailableMessage); + } + return await port.persist({ + id: createAssistantImageFilePreviewAttachmentId({ + serverId: input.serverId, + occurrenceKey: input.occurrenceKey, + mimeType: file.mime, + path: file.path || resolution.path, + size: file.size, + modifiedAt: file.modifiedAt, + contentLength: file.bytes.byteLength, + }), + bytes: file.bytes, + mimeType: file.mime, + fileName: getFileNameFromPath(file.path || resolution.path), + }); + }, + }; +} diff --git a/packages/app/src/assistant-image/load-dimensions.test.ts b/packages/app/src/assistant-image/load-dimensions.test.ts new file mode 100644 index 0000000000..a60f9b1b6d --- /dev/null +++ b/packages/app/src/assistant-image/load-dimensions.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { + type AssistantImageRenderedDimensionsReader, + resolveAssistantImageLoadDimensions, +} from "./load-dimensions"; + +class MemoryRenderedDimensionsReader implements AssistantImageRenderedDimensionsReader { + constructor(private readonly dimensions: { width: number; height: number } | null) {} + + read(): { width: number; height: number } | null { + return this.dimensions; + } +} + +describe("assistant image load dimensions", () => { + it("prefers native source dimensions", () => { + const dimensions = resolveAssistantImageLoadDimensions({ + source: { width: 640, height: 320 }, + target: { naturalWidth: 800, naturalHeight: 400 }, + renderedImage: null, + renderedDimensions: new MemoryRenderedDimensionsReader({ width: 900, height: 600 }), + }); + + expect(dimensions).toEqual({ width: 640, height: 320 }); + }); + + it("uses browser event dimensions when native dimensions are absent", () => { + const dimensions = resolveAssistantImageLoadDimensions({ + source: { width: 0, height: 0 }, + target: { naturalWidth: 800, naturalHeight: 400 }, + renderedImage: null, + renderedDimensions: new MemoryRenderedDimensionsReader({ width: 900, height: 600 }), + }); + + expect(dimensions).toEqual({ width: 800, height: 400 }); + }); + + it("falls back to the rendered image adapter", () => { + const renderedImage = { id: "rendered-image" }; + const dimensions = resolveAssistantImageLoadDimensions({ + target: null, + renderedImage, + renderedDimensions: new MemoryRenderedDimensionsReader({ width: 900, height: 600 }), + }); + + expect(dimensions).toEqual({ width: 900, height: 600 }); + }); +}); diff --git a/packages/app/src/assistant-image/load-dimensions.ts b/packages/app/src/assistant-image/load-dimensions.ts new file mode 100644 index 0000000000..069814ced8 --- /dev/null +++ b/packages/app/src/assistant-image/load-dimensions.ts @@ -0,0 +1,45 @@ +export interface AssistantImageDimensions { + width: number; + height: number; +} + +interface AssistantImageDimensionCandidate { + width?: unknown; + height?: unknown; +} + +export interface AssistantImageRenderedDimensionsReader { + read(renderedImage: unknown): AssistantImageDimensions | null; +} + +function readPositiveDimensions( + candidate: AssistantImageDimensionCandidate | null | undefined, +): AssistantImageDimensions | null { + if ( + typeof candidate?.width !== "number" || + typeof candidate.height !== "number" || + candidate.width <= 0 || + candidate.height <= 0 + ) { + return null; + } + return { width: candidate.width, height: candidate.height }; +} + +export function resolveAssistantImageLoadDimensions(input: { + source?: AssistantImageDimensionCandidate | null; + target?: { naturalWidth?: unknown; naturalHeight?: unknown } | null; + renderedImage: unknown; + renderedDimensions: AssistantImageRenderedDimensionsReader; +}): AssistantImageDimensions | null { + const source = readPositiveDimensions(input.source); + if (source) { + return source; + } + + const target = readPositiveDimensions({ + width: input.target?.naturalWidth, + height: input.target?.naturalHeight, + }); + return target ?? input.renderedDimensions.read(input.renderedImage); +} diff --git a/packages/app/src/assistant-image/use-assistant-image.test.tsx b/packages/app/src/assistant-image/use-assistant-image.test.tsx deleted file mode 100644 index 03a6759a68..0000000000 --- a/packages/app/src/assistant-image/use-assistant-image.test.tsx +++ /dev/null @@ -1,173 +0,0 @@ -/** - * @vitest-environment jsdom - */ -import { act, cleanup, renderHook, waitFor } from "@testing-library/react"; -import { afterEach, describe, expect, it } from "vitest"; -import type { ImageLoadEvent } from "react-native"; -import type { DaemonClient } from "@getpaseo/client/internal/daemon-client"; -import type { AttachmentStore } from "@/attachments/types"; -import { __setAttachmentStoreForTests } from "@/attachments/store"; -import { setAssistantImageMetadata } from "@/utils/assistant-image-metadata"; -import { useAssistantImage } from "./use-assistant-image"; - -describe("useAssistantImage", () => { - afterEach(() => { - cleanup(); - __setAttachmentStoreForTests(null); - }); - - it("keeps a successfully loaded image when cached dimensions are available", async () => { - const source = "https://example.com/expiring-image.png"; - setAssistantImageMetadata({ source }, { width: 640, height: 320 }); - const { result } = renderHook(() => - useAssistantImage({ source, occurrenceKey: "agent:message:image" }), - ); - - await waitFor(() => expect(result.current.status).toBe("loading")); - const binding = result.current.status === "failed" ? null : result.current.binding; - if (!binding) { - throw new Error("Expected a direct image render binding"); - } - act(() => { - binding.onLoad({ - nativeEvent: { source: { uri: source, width: 0, height: 0 } }, - } as ImageLoadEvent); - }); - - expect(result.current).toMatchObject({ status: "loaded", aspectRatio: 2 }); - }); - - it("uses browser image dimensions from the successful load event", async () => { - const source = "https://example.com/browser-image.png"; - const { result } = renderHook(() => - useAssistantImage({ source, occurrenceKey: "agent:message:browser-image" }), - ); - - await waitFor(() => expect(result.current.status).toBe("loading")); - const binding = result.current.status === "failed" ? null : result.current.binding; - if (!binding) { - throw new Error("Expected a direct image render binding"); - } - act(() => { - binding.onLoad({ - nativeEvent: { target: { naturalWidth: 800, naturalHeight: 400 } }, - } as unknown as ImageLoadEvent); - }); - - expect(result.current).toMatchObject({ status: "loaded", aspectRatio: 2 }); - }); - - it("uses the rendered browser image when React Native Web clears the load target", async () => { - const source = "https://example.com/react-native-web-image.png"; - const { result } = renderHook(() => - useAssistantImage({ source, occurrenceKey: "agent:message:react-native-web-image" }), - ); - - await waitFor(() => expect(result.current.status).toBe("loading")); - const binding = result.current.status === "failed" ? null : result.current.binding; - if (!binding) { - throw new Error("Expected a direct image render binding"); - } - const root = document.createElement("div"); - const image = document.createElement("img"); - Object.defineProperties(image, { - naturalWidth: { value: 900 }, - naturalHeight: { value: 600 }, - }); - root.append(image); - binding.onRef(root); - act(() => { - binding.onLoad({ nativeEvent: { target: null } } as unknown as ImageLoadEvent); - }); - - expect(result.current).toMatchObject({ status: "loaded", aspectRatio: 1.5 }); - }); - - it("restores a previously loaded URI without returning to loading", async () => { - const source = "https://example.com/remounted-image.png"; - const first = renderHook(() => - useAssistantImage({ source, occurrenceKey: "agent:message:remounted-image" }), - ); - await waitFor(() => expect(first.result.current.status).toBe("loading")); - const binding = first.result.current.status === "failed" ? null : first.result.current.binding; - if (!binding) { - throw new Error("Expected a direct image render binding"); - } - act(() => { - binding.onLoad({ - nativeEvent: { target: { naturalWidth: 800, naturalHeight: 400 } }, - } as unknown as ImageLoadEvent); - }); - expect(first.result.current.status).toBe("loaded"); - first.unmount(); - - const remounted = renderHook(() => - useAssistantImage({ source, occurrenceKey: "agent:message:remounted-image" }), - ); - - expect(remounted.result.current).toMatchObject({ status: "loaded", aspectRatio: 2 }); - }); - - it("restarts a failed file acquisition when the daemon client reconnects", async () => { - const attachmentStore: AttachmentStore = { - storageType: "web-indexeddb", - async save(input) { - return { - id: input.id ?? "assistant-image", - mimeType: input.mimeType ?? "image/png", - storageType: "web-indexeddb", - storageKey: input.id ?? "assistant-image", - fileName: input.fileName, - byteSize: 4, - createdAt: 1, - }; - }, - async encodeBase64() { - return ""; - }, - async resolvePreviewUrl({ attachment }) { - return `blob:${attachment.id}`; - }, - async delete() {}, - async garbageCollect() {}, - }; - __setAttachmentStoreForTests(attachmentStore); - - let readCount = 0; - const connectedClient = { - async readFile(_cwd: string, path: string) { - readCount += 1; - return { - kind: "image" as const, - path, - mime: "image/png", - size: 4, - modifiedAt: "1", - bytes: new Uint8Array([1, 2, 3, 4]), - }; - }, - } as unknown as DaemonClient; - const initialProps: { client: DaemonClient | null } = { client: null }; - const { result, rerender } = renderHook( - ({ client }: { client: DaemonClient | null }) => - useAssistantImage({ - source: "/workspace/reconnect.png", - occurrenceKey: "agent:message:reconnect-image", - client, - workspaceRoot: "/workspace", - serverId: "server", - }), - { initialProps }, - ); - - await waitFor(() => expect(result.current.status).toBe("failed"), { timeout: 3_000 }); - - rerender({ client: connectedClient }); - - await waitFor(() => { - expect(readCount).toBe(1); - expect(result.current.status).toBe("loading"); - expect(result.current.status === "loading" && result.current.binding).not.toBeNull(); - }); - }); -}); diff --git a/packages/app/src/assistant-image/use-assistant-image.ts b/packages/app/src/assistant-image/use-assistant-image.ts index 6dc714e1ff..38090c45f0 100644 --- a/packages/app/src/assistant-image/use-assistant-image.ts +++ b/packages/app/src/assistant-image/use-assistant-image.ts @@ -20,28 +20,28 @@ import { releaseAttachmentPreviewUrl, resolveAttachmentPreviewUrl, } from "@/attachments/service"; -import { - createPreviewAttachmentId, - getFileNameFromPath, - parseImageDataUrl, -} from "@/attachments/utils"; +import { createPreviewAttachmentId, parseImageDataUrl } from "@/attachments/utils"; import { getAssistantImageMetadata, setAssistantImageMetadata, } from "@/utils/assistant-image-metadata"; import { resolveAssistantImageSource } from "@/utils/assistant-image-source"; -import type { AssistantImageSourceResolution } from "@/utils/assistant-image-source"; +import { createAssistantImageAcquisitionCache } from "./acquisition-cache"; import { - createAssistantImageAcquisitionCache, - createAssistantImageFileAcquisitionKey, - createAssistantImageFilePreviewAttachmentId, -} from "./acquisition-cache"; + createAssistantImageFileAcquisition, + type AssistantImageAcquisition, + type AssistantImageFileAcquisitionPort, +} from "./file-acquisition"; import { createAssistantImageLifecycle, transitionAssistantImageLifecycle, type AssistantImageLifecycle, type AssistantImageLifecycleEvent, } from "./lifecycle"; +import { + type AssistantImageRenderedDimensionsReader, + resolveAssistantImageLoadDimensions, +} from "./load-dimensions"; import { runAssistantImageOperationWithRetry } from "./retry"; interface AssistantImageRenderBinding { @@ -51,35 +51,17 @@ interface AssistantImageRenderBinding { onError: () => void; } -function getImageLoadDimensions( - event: ImageLoadEvent, - renderedImage: unknown, -): { width: number; height: number } | null { - const nativeEvent = event.nativeEvent as ImageLoadEvent["nativeEvent"] & { - target?: { naturalWidth?: unknown; naturalHeight?: unknown }; - }; - const source = nativeEvent.source; - if (source && source.width > 0 && source.height > 0) { - return source; - } - const { naturalWidth, naturalHeight } = nativeEvent.target ?? {}; - if ( - typeof naturalWidth === "number" && - typeof naturalHeight === "number" && - naturalWidth > 0 && - naturalHeight > 0 - ) { - return { width: naturalWidth, height: naturalHeight }; - } - - if (isWeb && renderedImage instanceof HTMLElement) { - const image = renderedImage.querySelector("img"); - if (image && image.naturalWidth > 0 && image.naturalHeight > 0) { - return { width: image.naturalWidth, height: image.naturalHeight }; +const renderedDimensions: AssistantImageRenderedDimensionsReader = { + read(renderedImage) { + if (!isWeb || !(renderedImage instanceof HTMLElement)) { + return null; } - } - return null; -} + const image = renderedImage.querySelector("img"); + return image && image.naturalWidth > 0 && image.naturalHeight > 0 + ? { width: image.naturalWidth, height: image.naturalHeight } + : null; + }, +}; export type AssistantImageResult = | { @@ -114,11 +96,6 @@ type AttachmentAcquisitionState = | { status: "loaded"; attachment: AttachmentMetadata } | { status: "failed"; error: unknown }; -interface AttachmentAcquisition { - key: string; - locate: () => Promise; -} - interface DataImage { mimeType: string; base64: string; @@ -169,12 +146,12 @@ function rememberLoadedImage(uri: string, aspectRatio: number): void { } } -function acquireAttachment(acquisition: AttachmentAcquisition): Promise { +function acquireAttachment(acquisition: AssistantImageAcquisition): Promise { return attachmentAcquisitionCache.acquire(acquisition.key, acquisition.locate); } function useAttachmentAcquisition( - acquisition: AttachmentAcquisition | null, + acquisition: AssistantImageAcquisition | null, ): AttachmentAcquisitionState { const acquisitionKey = acquisition?.key ?? null; const [entry, setEntry] = useState<{ @@ -232,54 +209,10 @@ function useAttachmentAcquisition( return entry.key === acquisitionKey ? entry.state : { status: "waiting" }; } -function createFileAcquisition(input: { - client?: DaemonClient | null; - resolution: AssistantImageSourceResolution | null; - serverId?: string; - occurrenceKey: string; - unavailableMessage: string; -}): AttachmentAcquisition | null { - if (input.resolution?.kind !== "file_rpc") { - return null; - } - const { client, resolution } = input; - return { - key: createAssistantImageFileAcquisitionKey({ - serverId: input.serverId, - occurrenceKey: input.occurrenceKey, - cwd: resolution.cwd, - path: resolution.path, - }), - locate: async () => { - if (!client) { - throw new Error(input.unavailableMessage); - } - const file = await client.readFile(resolution.cwd, resolution.path); - if (file.kind !== "image") { - throw new Error(input.unavailableMessage); - } - return await persistAttachmentFromBytes({ - id: createAssistantImageFilePreviewAttachmentId({ - serverId: input.serverId, - occurrenceKey: input.occurrenceKey, - mimeType: file.mime, - path: file.path || resolution.path, - size: file.size, - modifiedAt: file.modifiedAt, - contentLength: file.bytes.byteLength, - }), - bytes: file.bytes, - mimeType: file.mime, - fileName: getFileNameFromPath(file.path || resolution.path), - }); - }, - }; -} - function createDataImageAcquisition(input: { source: string; dataImage: DataImage | null; -}): AttachmentAcquisition | null { +}): AssistantImageAcquisition | null { if (!input.dataImage) { return null; } @@ -436,17 +369,21 @@ export function useAssistantImage({ [source, workspaceRoot], ); const dataImage = useMemo(() => parseImageDataUrl(source), [source]); - const fileAcquisition = useMemo( - () => - createFileAcquisition({ - client, - resolution, - serverId, - occurrenceKey, - unavailableMessage: t("message.attachments.imagePreviewUnavailable"), - }), - [client, occurrenceKey, resolution, serverId, t], - ); + const fileAcquisition = useMemo(() => { + const port: AssistantImageFileAcquisitionPort | null = client + ? { + readFile: async (cwd, path) => await client.readFile(cwd, path), + persist: persistAttachmentFromBytes, + } + : null; + return createAssistantImageFileAcquisition({ + port, + resolution, + serverId, + occurrenceKey, + unavailableMessage: t("message.attachments.imagePreviewUnavailable"), + }); + }, [client, occurrenceKey, resolution, serverId, t]); const dataImageAcquisition = useMemo( () => createDataImageAcquisition({ source, dataImage }), [dataImage, source], @@ -520,7 +457,15 @@ export function useAssistantImage({ if (!uri) { return; } - const dimensions = getImageLoadDimensions(event, renderedImageRef.current); + const nativeEvent = event.nativeEvent as ImageLoadEvent["nativeEvent"] & { + target?: { naturalWidth?: unknown; naturalHeight?: unknown }; + }; + const dimensions = resolveAssistantImageLoadDimensions({ + source: nativeEvent.source, + target: nativeEvent.target, + renderedImage: renderedImageRef.current, + renderedDimensions, + }); const metadata = dimensions ? setAssistantImageMetadata({ source, workspaceRoot, serverId }, dimensions) : null;