From d9178eac7594b4c261f36e769f249a89620f95b4 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 16 Oct 2026 08:55:33 +0200 Subject: [PATCH] fix(markdown): reserve a placeholder box with an image glyph while images load --- docs/native-text-selection.md | 2 +- .../browser/agent-tab-image-stability.spec.ts | 9 ++- .../generate-markdown-image-placeholder.mjs | 27 +++++++ .../src/components/markdown/image-decoding.ts | 16 +++++ .../app/src/components/markdown/images.ts | 34 ++++----- .../src/components/markdown/renderer.test.ts | 72 +++++++++++++++++-- .../app/src/components/markdown/source.ts | 4 +- 7 files changed, 134 insertions(+), 30 deletions(-) create mode 100644 packages/app/scripts/generate-markdown-image-placeholder.mjs diff --git a/docs/native-text-selection.md b/docs/native-text-selection.md index 22e5e0c7e1..44d4df68ca 100644 --- a/docs/native-text-selection.md +++ b/docs/native-text-selection.md @@ -19,6 +19,6 @@ Reopening a list marker or quote prefix on the following fragment cannot place t ## Leading inline images -Native image decoders do not support SVG badges, so the renderer shows their alt text. Images that fail decoding also show alt text. +Native image decoders do not support SVG badges, so the renderer shows their alt text. Pending images share the web placeholder: a reserved image box with a small centred image glyph; confirmed acquisition or decoding failures show alt text. The native library classifies an image from the text preceding it. A raster image at the start of a paragraph uses block sizing even when text follows; the inline size setting only affects images the library recognizes as inline. Correcting that classification requires a native library change and new app builds. JavaScript-only updates retain that classification until native app builds include a library fix. diff --git a/packages/app/e2e/browser/agent-tab-image-stability.spec.ts b/packages/app/e2e/browser/agent-tab-image-stability.spec.ts index 7149e1aa25..97b1c8337c 100644 --- a/packages/app/e2e/browser/agent-tab-image-stability.spec.ts +++ b/packages/app/e2e/browser/agent-tab-image-stability.spec.ts @@ -168,7 +168,7 @@ test("reloading a timeline reserves image height without a load-time scroll jump await page.reload(); await delivery.waitUntilRequested(); const rendered = page.getByRole("button", { name: image.alt }).first(); - await expect(rendered).toHaveAttribute("src", /^data:image\/gif/); + await expect(rendered).toHaveAttribute("src", /^data:image\/png/); await scrollTimelinePromptIntoView( page, "image-history-turn-2: emit 1 coalesced agent stream updates", @@ -251,7 +251,7 @@ test("a later reply reads new bytes at the same workspace image path", async ({ test("pending assistant images reserve their decoded height", async ({ imageWorkspace: workspace, page, -}) => { +}, testInfo) => { const image = await createSmallAssistantPng(workspace, { alt: "Delayed remote image", fileName: "delayed.png", @@ -265,10 +265,13 @@ test("pending assistant images reserve their decoded height", async ({ await emitSettledAssistantImage(workspace.client, agent, image, url); await openAssistantImageTimeline(page, agent); const rendered = page.getByRole("button", { name: image.alt }).first(); - await expect(rendered).toHaveAttribute("src", /^data:image\/gif/); + await expect(rendered).toHaveAttribute("src", /^data:image\/png/); await delivery.waitUntilRequested(); const pendingBox = await rendered.boundingBox(); expect(pendingBox?.height).toBe(320); + await rendered.click(); + await expect(page.getByTestId("attachment-lightbox-image")).toHaveCount(0); + await page.screenshot({ path: testInfo.outputPath("pending-image.png") }); delivery.release(); await expect(rendered).toHaveAttribute("src", url); expect((await rendered.boundingBox())?.height).toBe(pendingBox?.height); diff --git a/packages/app/scripts/generate-markdown-image-placeholder.mjs b/packages/app/scripts/generate-markdown-image-placeholder.mjs new file mode 100644 index 0000000000..4e147161e4 --- /dev/null +++ b/packages/app/scripts/generate-markdown-image-placeholder.mjs @@ -0,0 +1,27 @@ +// Regenerate from the repo root: node packages/app/scripts/generate-markdown-image-placeholder.mjs +// sharp is already provided by the development dependency tree; it is not used at runtime. +import { readFile, writeFile } from "node:fs/promises"; +import sharp from "sharp"; + +// Lucide Image (ISC), lucide-react-native/dist/esm/icons/image.mjs. +const svg = ` + + + + + + + +`; +const png = await sharp(Buffer.from(svg)).png({ compressionLevel: 9 }).toBuffer(); +const uri = `data:image/png;base64,${png.toString("base64")}`; +const sourceFile = new URL("../src/components/markdown/source.ts", import.meta.url); +const source = await readFile(sourceFile, "utf8"); +await writeFile( + sourceFile, + source.replace( + /export const PENDING_MARKDOWN_IMAGE =\s*"[^"]*";/, + `export const PENDING_MARKDOWN_IMAGE =\n "${uri}";`, + ), +); +console.log(`Generated 640×640 Markdown image placeholder: ${png.length} bytes`); diff --git a/packages/app/src/components/markdown/image-decoding.ts b/packages/app/src/components/markdown/image-decoding.ts index 9750d7d6c0..efd1ad3521 100644 --- a/packages/app/src/components/markdown/image-decoding.ts +++ b/packages/app/src/components/markdown/image-decoding.ts @@ -16,3 +16,19 @@ export function decodeMarkdownImage(uri: string, decode: ImageDecoder): Promise< ); }); } + +interface MarkdownImageLoadState { + canAcquire: boolean; + url: string | undefined; + acquisitionFailed: boolean; + decodeFailed: boolean; + decoded: boolean; +} + +export function getMarkdownImageStatus( + state: MarkdownImageLoadState, +): "pending" | "ready" | "unavailable" { + const cannotResolve = !state.canAcquire && state.url === undefined; + if (cannotResolve || state.acquisitionFailed || state.decodeFailed) return "unavailable"; + return state.decoded ? "ready" : "pending"; +} diff --git a/packages/app/src/components/markdown/images.ts b/packages/app/src/components/markdown/images.ts index a707a0b3c1..83b24b481b 100644 --- a/packages/app/src/components/markdown/images.ts +++ b/packages/app/src/components/markdown/images.ts @@ -15,7 +15,7 @@ import type { DaemonClient } from "@getpaseo/client/internal/daemon-client"; import type { AttachmentMetadata } from "@/attachments/types"; import { resolveMarkdownPreviewImageSource } from "./image-source"; import { prepareMarkdownSource } from "./source"; -import { decodeMarkdownImage } from "./image-decoding"; +import { decodeMarkdownImage, getMarkdownImageStatus } from "./image-decoding"; const urls = createAssistantImageAcquisitionCache<{ uri: string; attachment: AttachmentMetadata }>({ capacity: 500, @@ -111,27 +111,21 @@ export function useMarkdownSource(text: string, context: MarkdownImageContext) { }, })), ); - const unavailableImages = new Set( - sources.filter((_source, index) => - isWeb - ? (!acquisitions[index] && displayUrls[index] === undefined) || - results[index]?.isError || - decoded[index]?.isError - : displayUrls[index] !== undefined && decoded[index]?.data !== true, - ), + const statuses = sources.map((_source, index) => + getMarkdownImageStatus({ + canAcquire: acquisitions[index] !== null, + url: displayUrls[index], + acquisitionFailed: results[index].isError, + decodeFailed: decoded[index].isError, + decoded: decoded[index].data === true, + }), ); + const unavailableImages = new Set( + sources.filter((_source, index) => statuses[index] === "unavailable"), + ); + const pendingImages = new Set(sources.filter((_source, index) => statuses[index] === "pending")); return { - markdown: prepared.render( - resolved, - unavailableImages, - new Set( - isWeb - ? sources.filter( - (source, index) => !unavailableImages.has(source) && decoded[index]?.data !== true, - ) - : [], - ), - ), + markdown: prepared.render(resolved, unavailableImages, pendingImages), linkRecognition: prepared.linkRecognition, linkSource: prepared.linkSource, imageSources: Object.fromEntries([...resolved].map(([source, url]) => [url, source])), diff --git a/packages/app/src/components/markdown/renderer.test.ts b/packages/app/src/components/markdown/renderer.test.ts index 1703a4c5fc..10ca97beec 100644 --- a/packages/app/src/components/markdown/renderer.test.ts +++ b/packages/app/src/components/markdown/renderer.test.ts @@ -1,10 +1,11 @@ +import { fromMarkdown } from "mdast-util-from-markdown"; import { toNativeLinkRecognition } from "react-native-enriched-markdown/src/utils/regexParser"; import { QueryClient } from "@tanstack/react-query"; -import { decodeMarkdownImage } from "./image-decoding"; +import { decodeMarkdownImage, getMarkdownImageStatus } from "./image-decoding"; import { describe, expect, it } from "vitest"; import { darkTheme, lightTheme } from "@/styles/theme"; import { createMarkdownStyles } from "./styles"; -import { prepareMarkdownSource } from "./source"; +import { prepareMarkdownSource, PENDING_MARKDOWN_IMAGE } from "./source"; import { normalizeMarkdownHtml } from "./html"; describe("shared Markdown source adaptation", () => { @@ -375,7 +376,6 @@ it("retries transient image decoding failures instead of caching an unavailable }); it("keeps the document's first reference definition across every native Mermaid segment", async () => { - const { fromMarkdown } = await import("mdast-util-from-markdown"); const { splitMermaidFences } = await import("./fence/mermaid/segments"); const before = '[first]: /repo/first.ts "Original"\n\n'; const after = '\n\n[open][first]\n\n[first]: /repo/second.ts "Later"'; @@ -405,7 +405,7 @@ it("reserves pending image geometry until decode succeeds and collapses only con new Set(prepared.images.map((image) => image.source)), ); expect(pending.match(/!\[/g)).toHaveLength(2); - expect(pending).toContain("data:image/gif;base64,"); + expect(pending).toContain("data:image/png;base64,"); expect(prepared.render(new Map(), new Set(prepared.images.map((image) => image.source)))).toBe( "Local\n\nRemote", ); @@ -477,3 +477,67 @@ it("keeps prose and inline-code recognition enabled through the native regex ada isDefault: false, }); }); + +it.each([undefined, "file:///cache/shot.png"])( + "reserves a native workspace image while acquiring or decoding %s", + (url) => { + const status = getMarkdownImageStatus({ + canAcquire: true, + url, + acquisitionFailed: false, + decodeFailed: false, + decoded: false, + }); + expect(status).toBe("pending"); + const prepared = prepareMarkdownSource("Before ![image](shot.png) after", undefined, { + supportsSvg: false, + }); + const rendered = prepared.render(new Map(), new Set(), new Set(["shot.png"])); + expect(rendered).toBe(`Before ![image](<${PENDING_MARKDOWN_IMAGE}>) after`); + expect(fromMarkdown(rendered).children).toMatchObject([ + { + type: "paragraph", + children: [ + { type: "text", value: "Before " }, + { type: "image", url: PENDING_MARKDOWN_IMAGE }, + { type: "text", value: " after" }, + ], + }, + ]); + }, +); + +it.each([ + { acquisitionFailed: true, decodeFailed: false }, + { acquisitionFailed: false, decodeFailed: true }, +])("collapses confirmed native image failures to alt text: %s", (failure) => { + expect( + getMarkdownImageStatus({ + canAcquire: true, + url: "file:///cache/shot.png", + decoded: false, + ...failure, + }), + ).toBe("unavailable"); + const prepared = prepareMarkdownSource("Before ![image](shot.png) after", undefined, { + supportsSvg: false, + }); + expect(prepared.render(new Map(), new Set(["shot.png"]), new Set(["shot.png"]))).toBe( + "Before image after", + ); +}); + +it("replaces the native placeholder only after the resolved URL decodes", () => { + const url = "file:///cache/shot.png"; + expect( + getMarkdownImageStatus({ + canAcquire: true, + url, + acquisitionFailed: false, + decodeFailed: false, + decoded: true, + }), + ).toBe("ready"); + const prepared = prepareMarkdownSource("![image](shot.png)", undefined, { supportsSvg: false }); + expect(prepared.render(new Map([["shot.png", url]]))).toBe(`![image](<${url}>)`); +}); diff --git a/packages/app/src/components/markdown/source.ts b/packages/app/src/components/markdown/source.ts index 166f55a2f5..320864f064 100644 --- a/packages/app/src/components/markdown/source.ts +++ b/packages/app/src/components/markdown/source.ts @@ -45,9 +45,9 @@ export function markdownLinkSource( return { href: url, text: url }; } -// A decoded transparent pixel reserves the library's normal block/inline image box. +// Generated by scripts/generate-markdown-image-placeholder.mjs; reserves the image box. export const PENDING_MARKDOWN_IMAGE = - "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"; + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAoAAAAKACAYAAAAMzckjAAAACXBIWXMAAAsTAAALEwEAmpwYAAAKBklEQVR42u3dXWscVRgH8ElFLK032tiCN1nSEiiowdKAn8DY2rR+AG965Qt4Y24Sm91kd5tN1ogQaGzzFfIl43Nki+s4u+m+ZWfC7weH0M6Z2QPn5s8zc87JMgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgKvm/Px8YXd3d7Hdbi/Ns6UxxHAWzAgAwIxE4LrebDafxd/f9/b2TkvSujGep2lsZggAYIr29/c/iZDVKlHwy7dmryIIAEA2hcpfo9Folzj89YdAlUAAgEml174VCH//tBjrhhkDAMgmXvDxn2/+4t+von0dbXmOC0CW6/X6ehpL/9iiUtnNLAwBAMgmef27mK+ypfCXlac6uZ4fX6fTuWXmAADGD4C1fMCKCtzdsowvjaUgoNbMHADA+AFwuSBgLRsfAIAAmL/vRrSVVqu1mipy0a4JgAAAVzAAbm9v34nrP0a/17l7/oz25PT09H0BEADgigTAnZ2d+3H9eNg2LXHvb9E+FAABACoeAFPl76Lw13f/r9mUtmoRAAEA5hQAe69933nD5uj/UAAEAKhoAOwt+Pgr1+cwFoB8Hn9vx159j+Lvm9xzfhEAAQCqGwBXCvp80d8n/u957vofAiAAQEUDYCz+WC3oczv3nMe5I9tOBEAAgOpWAP93Wkh67fv2+tnZ2Xtp9W+uz74ACABQ3QB4Le3zl+uTvvl73qv8vSh4xvcCIABAhVcB1+v1jRFWAb+OZ3x60RnE0b6Lvj+kFhXFZ3Hu75IACABQkgCYTvgoeM07aAuYx4N+7+jo6GZc/3nI/T9tbm7eFAABAEpwEkg64SOubw6r/EUl79tswCbQW1tbH8XikPZFATL1SX0FQACAEpwFnMJd2uQ57fMX7SjaSVrwkb75G/baNwW6uP5yhI2kX/buEQABAOYcAEd2Qfh7Edd2BoXA2Gx6VQAEAKhQABwU/tLJIrGwZC37d5HJWv60kd7r4K4ACABQkQCYwl88rz3gW8G1fP+o9n1ZFAIFQACACgTAUcPfKCFQAAQAKFkAHDf8vWsIFAABAEoUAId98xftwQjjejAoBHY6nXtmDgCgBAFwWuGvPwSmqmHB874ycwAAcw6A0w5/b8Ur40f5Z8Yr4vtmDgBgjgFwVuHPSSAAACUMgJMu+BAAAQAqFABnHf4EQACAEgXAywh/AiAAQEkC4GWFPwEQAKAEAfAyw58ACAAw5wB42eFPAAQAmE3AquUDVrvdvluG8Ncb372CAFgzcwAA4wesxXzAqtfr62UIf73xfVNwFNwtMwcAML6FCFXdXIXtVQS79fSqNdpn0Q4LqnBv0ikdUS1cmkVLv52e32g0TnK/e5jGbNoAALKJqmxPC6p7pWwx1idmDABg8gB4PcJVswLhb+/4+PgDMwYAMKVvAaO1yhz+0hjNFADAFKXqWnx3txHf3XVLFPwO02tflT8AgNlaSCtt02KMg4ODWlrpGyFsZVYLPoYsBPnYgg8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOBK+hsMQ7RDjzvGwwAAAABJRU5ErkJggg=="; export interface MarkdownImage { source: string;