mirror of
https://github.com/getpaseo/paseo.git
synced 2026-10-24 10:12:44 +00:00
fix(markdown): reserve a placeholder box with an image glyph while images load
This commit is contained in:
Mohamed Boudra
@@ -19,6 +19,6 @@ Reopening a list marker or quote prefix on the following fragment cannot place t
|
||||
|
||||
## Leading inline images
|
||||
|
||||
Native image decoders do not support SVG badges, so the renderer shows their alt text. Images that fail decoding also show alt text.
|
||||
Native image decoders do not support SVG badges, so the renderer shows their alt text. Pending images share the web placeholder: a reserved image box with a small centred image glyph; confirmed acquisition or decoding failures show alt text.
|
||||
|
||||
The native library classifies an image from the text preceding it. A raster image at the start of a paragraph uses block sizing even when text follows; the inline size setting only affects images the library recognizes as inline. Correcting that classification requires a native library change and new app builds. JavaScript-only updates retain that classification until native app builds include a library fix.
|
||||
|
||||
@@ -168,7 +168,7 @@ test("reloading a timeline reserves image height without a load-time scroll jump
|
||||
await page.reload();
|
||||
await delivery.waitUntilRequested();
|
||||
const rendered = page.getByRole("button", { name: image.alt }).first();
|
||||
await expect(rendered).toHaveAttribute("src", /^data:image\/gif/);
|
||||
await expect(rendered).toHaveAttribute("src", /^data:image\/png/);
|
||||
await scrollTimelinePromptIntoView(
|
||||
page,
|
||||
"image-history-turn-2: emit 1 coalesced agent stream updates",
|
||||
@@ -251,7 +251,7 @@ test("a later reply reads new bytes at the same workspace image path", async ({
|
||||
test("pending assistant images reserve their decoded height", async ({
|
||||
imageWorkspace: workspace,
|
||||
page,
|
||||
}) => {
|
||||
}, testInfo) => {
|
||||
const image = await createSmallAssistantPng(workspace, {
|
||||
alt: "Delayed remote image",
|
||||
fileName: "delayed.png",
|
||||
@@ -265,10 +265,13 @@ test("pending assistant images reserve their decoded height", async ({
|
||||
await emitSettledAssistantImage(workspace.client, agent, image, url);
|
||||
await openAssistantImageTimeline(page, agent);
|
||||
const rendered = page.getByRole("button", { name: image.alt }).first();
|
||||
await expect(rendered).toHaveAttribute("src", /^data:image\/gif/);
|
||||
await expect(rendered).toHaveAttribute("src", /^data:image\/png/);
|
||||
await delivery.waitUntilRequested();
|
||||
const pendingBox = await rendered.boundingBox();
|
||||
expect(pendingBox?.height).toBe(320);
|
||||
await rendered.click();
|
||||
await expect(page.getByTestId("attachment-lightbox-image")).toHaveCount(0);
|
||||
await page.screenshot({ path: testInfo.outputPath("pending-image.png") });
|
||||
delivery.release();
|
||||
await expect(rendered).toHaveAttribute("src", url);
|
||||
expect((await rendered.boundingBox())?.height).toBe(pendingBox?.height);
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
// Regenerate from the repo root: node packages/app/scripts/generate-markdown-image-placeholder.mjs
|
||||
// sharp is already provided by the development dependency tree; it is not used at runtime.
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import sharp from "sharp";
|
||||
|
||||
// Lucide Image (ISC), lucide-react-native/dist/esm/icons/image.mjs.
|
||||
const svg = `<svg xmlns="http://www.w3.org/2000/svg" width="640" height="640" viewBox="0 0 640 640">
|
||||
<svg x="293.333333" y="293.333333" width="53.333333" height="53.333333" viewBox="0 0 24 24">
|
||||
<g fill="none" stroke="#808080" opacity="0.6" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||
<rect width="18" height="18" x="3" y="3" rx="2" ry="2"/>
|
||||
<circle cx="9" cy="9" r="2"/>
|
||||
<path d="m21 15-3.086-3.086a2 2 0 0 0-2.828 0L6 21"/>
|
||||
</g>
|
||||
</svg>
|
||||
</svg>`;
|
||||
const png = await sharp(Buffer.from(svg)).png({ compressionLevel: 9 }).toBuffer();
|
||||
const uri = `data:image/png;base64,${png.toString("base64")}`;
|
||||
const sourceFile = new URL("../src/components/markdown/source.ts", import.meta.url);
|
||||
const source = await readFile(sourceFile, "utf8");
|
||||
await writeFile(
|
||||
sourceFile,
|
||||
source.replace(
|
||||
/export const PENDING_MARKDOWN_IMAGE =\s*"[^"]*";/,
|
||||
`export const PENDING_MARKDOWN_IMAGE =\n "${uri}";`,
|
||||
),
|
||||
);
|
||||
console.log(`Generated 640×640 Markdown image placeholder: ${png.length} bytes`);
|
||||
@@ -16,3 +16,19 @@ export function decodeMarkdownImage(uri: string, decode: ImageDecoder): Promise<
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
interface MarkdownImageLoadState {
|
||||
canAcquire: boolean;
|
||||
url: string | undefined;
|
||||
acquisitionFailed: boolean;
|
||||
decodeFailed: boolean;
|
||||
decoded: boolean;
|
||||
}
|
||||
|
||||
export function getMarkdownImageStatus(
|
||||
state: MarkdownImageLoadState,
|
||||
): "pending" | "ready" | "unavailable" {
|
||||
const cannotResolve = !state.canAcquire && state.url === undefined;
|
||||
if (cannotResolve || state.acquisitionFailed || state.decodeFailed) return "unavailable";
|
||||
return state.decoded ? "ready" : "pending";
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import type { DaemonClient } from "@getpaseo/client/internal/daemon-client";
|
||||
import type { AttachmentMetadata } from "@/attachments/types";
|
||||
import { resolveMarkdownPreviewImageSource } from "./image-source";
|
||||
import { prepareMarkdownSource } from "./source";
|
||||
import { decodeMarkdownImage } from "./image-decoding";
|
||||
import { decodeMarkdownImage, getMarkdownImageStatus } from "./image-decoding";
|
||||
|
||||
const urls = createAssistantImageAcquisitionCache<{ uri: string; attachment: AttachmentMetadata }>({
|
||||
capacity: 500,
|
||||
@@ -111,27 +111,21 @@ export function useMarkdownSource(text: string, context: MarkdownImageContext) {
|
||||
},
|
||||
})),
|
||||
);
|
||||
const unavailableImages = new Set(
|
||||
sources.filter((_source, index) =>
|
||||
isWeb
|
||||
? (!acquisitions[index] && displayUrls[index] === undefined) ||
|
||||
results[index]?.isError ||
|
||||
decoded[index]?.isError
|
||||
: displayUrls[index] !== undefined && decoded[index]?.data !== true,
|
||||
),
|
||||
const statuses = sources.map((_source, index) =>
|
||||
getMarkdownImageStatus({
|
||||
canAcquire: acquisitions[index] !== null,
|
||||
url: displayUrls[index],
|
||||
acquisitionFailed: results[index].isError,
|
||||
decodeFailed: decoded[index].isError,
|
||||
decoded: decoded[index].data === true,
|
||||
}),
|
||||
);
|
||||
const unavailableImages = new Set(
|
||||
sources.filter((_source, index) => statuses[index] === "unavailable"),
|
||||
);
|
||||
const pendingImages = new Set(sources.filter((_source, index) => statuses[index] === "pending"));
|
||||
return {
|
||||
markdown: prepared.render(
|
||||
resolved,
|
||||
unavailableImages,
|
||||
new Set(
|
||||
isWeb
|
||||
? sources.filter(
|
||||
(source, index) => !unavailableImages.has(source) && decoded[index]?.data !== true,
|
||||
)
|
||||
: [],
|
||||
),
|
||||
),
|
||||
markdown: prepared.render(resolved, unavailableImages, pendingImages),
|
||||
linkRecognition: prepared.linkRecognition,
|
||||
linkSource: prepared.linkSource,
|
||||
imageSources: Object.fromEntries([...resolved].map(([source, url]) => [url, source])),
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import { fromMarkdown } from "mdast-util-from-markdown";
|
||||
import { toNativeLinkRecognition } from "react-native-enriched-markdown/src/utils/regexParser";
|
||||
import { QueryClient } from "@tanstack/react-query";
|
||||
import { decodeMarkdownImage } from "./image-decoding";
|
||||
import { decodeMarkdownImage, getMarkdownImageStatus } from "./image-decoding";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { darkTheme, lightTheme } from "@/styles/theme";
|
||||
import { createMarkdownStyles } from "./styles";
|
||||
import { prepareMarkdownSource } from "./source";
|
||||
import { prepareMarkdownSource, PENDING_MARKDOWN_IMAGE } from "./source";
|
||||
import { normalizeMarkdownHtml } from "./html";
|
||||
|
||||
describe("shared Markdown source adaptation", () => {
|
||||
@@ -375,7 +376,6 @@ it("retries transient image decoding failures instead of caching an unavailable
|
||||
});
|
||||
|
||||
it("keeps the document's first reference definition across every native Mermaid segment", async () => {
|
||||
const { fromMarkdown } = await import("mdast-util-from-markdown");
|
||||
const { splitMermaidFences } = await import("./fence/mermaid/segments");
|
||||
const before = '[first]: /repo/first.ts "Original"\n\n';
|
||||
const after = '\n\n[open][first]\n\n[first]: /repo/second.ts "Later"';
|
||||
@@ -405,7 +405,7 @@ it("reserves pending image geometry until decode succeeds and collapses only con
|
||||
new Set(prepared.images.map((image) => image.source)),
|
||||
);
|
||||
expect(pending.match(/!\[/g)).toHaveLength(2);
|
||||
expect(pending).toContain("data:image/gif;base64,");
|
||||
expect(pending).toContain("data:image/png;base64,");
|
||||
expect(prepared.render(new Map(), new Set(prepared.images.map((image) => image.source)))).toBe(
|
||||
"Local\n\nRemote",
|
||||
);
|
||||
@@ -477,3 +477,67 @@ it("keeps prose and inline-code recognition enabled through the native regex ada
|
||||
isDefault: false,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([undefined, "file:///cache/shot.png"])(
|
||||
"reserves a native workspace image while acquiring or decoding %s",
|
||||
(url) => {
|
||||
const status = getMarkdownImageStatus({
|
||||
canAcquire: true,
|
||||
url,
|
||||
acquisitionFailed: false,
|
||||
decodeFailed: false,
|
||||
decoded: false,
|
||||
});
|
||||
expect(status).toBe("pending");
|
||||
const prepared = prepareMarkdownSource("Before  after", undefined, {
|
||||
supportsSvg: false,
|
||||
});
|
||||
const rendered = prepared.render(new Map(), new Set(), new Set(["shot.png"]));
|
||||
expect(rendered).toBe(`Before  after`);
|
||||
expect(fromMarkdown(rendered).children).toMatchObject([
|
||||
{
|
||||
type: "paragraph",
|
||||
children: [
|
||||
{ type: "text", value: "Before " },
|
||||
{ type: "image", url: PENDING_MARKDOWN_IMAGE },
|
||||
{ type: "text", value: " after" },
|
||||
],
|
||||
},
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ acquisitionFailed: true, decodeFailed: false },
|
||||
{ acquisitionFailed: false, decodeFailed: true },
|
||||
])("collapses confirmed native image failures to alt text: %s", (failure) => {
|
||||
expect(
|
||||
getMarkdownImageStatus({
|
||||
canAcquire: true,
|
||||
url: "file:///cache/shot.png",
|
||||
decoded: false,
|
||||
...failure,
|
||||
}),
|
||||
).toBe("unavailable");
|
||||
const prepared = prepareMarkdownSource("Before  after", undefined, {
|
||||
supportsSvg: false,
|
||||
});
|
||||
expect(prepared.render(new Map(), new Set(["shot.png"]), new Set(["shot.png"]))).toBe(
|
||||
"Before image after",
|
||||
);
|
||||
});
|
||||
|
||||
it("replaces the native placeholder only after the resolved URL decodes", () => {
|
||||
const url = "file:///cache/shot.png";
|
||||
expect(
|
||||
getMarkdownImageStatus({
|
||||
canAcquire: true,
|
||||
url,
|
||||
acquisitionFailed: false,
|
||||
decodeFailed: false,
|
||||
decoded: true,
|
||||
}),
|
||||
).toBe("ready");
|
||||
const prepared = prepareMarkdownSource("", undefined, { supportsSvg: false });
|
||||
expect(prepared.render(new Map([["shot.png", url]]))).toBe(``);
|
||||
});
|
||||
|
||||
@@ -45,9 +45,9 @@ export function markdownLinkSource(
|
||||
return { href: url, text: url };
|
||||
}
|
||||
|
||||
// A decoded transparent pixel reserves the library's normal block/inline image box.
|
||||
// Generated by scripts/generate-markdown-image-placeholder.mjs; reserves the image box.
|
||||
export const PENDING_MARKDOWN_IMAGE =
|
||||
"data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7";
|
||||
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAoAAAAKACAYAAAAMzckjAAAACXBIWXMAAAsTAAALEwEAmpwYAAAKBklEQVR42u3dXWscVRgH8ElFLK032tiCN1nSEiiowdKAn8DY2rR+AG965Qt4Y24Sm91kd5tN1ogQaGzzFfIl43Nki+s4u+m+ZWfC7weH0M6Z2QPn5s8zc87JMgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgKvm/Px8YXd3d7Hdbi/Ns6UxxHAWzAgAwIxE4LrebDafxd/f9/b2TkvSujGep2lsZggAYIr29/c/iZDVKlHwy7dmryIIAEA2hcpfo9Folzj89YdAlUAAgEml174VCH//tBjrhhkDAMgmXvDxn2/+4t+von0dbXmOC0CW6/X6ehpL/9iiUtnNLAwBAMgmef27mK+ypfCXlac6uZ4fX6fTuWXmAADGD4C1fMCKCtzdsowvjaUgoNbMHADA+AFwuSBgLRsfAIAAmL/vRrSVVqu1mipy0a4JgAAAVzAAbm9v34nrP0a/17l7/oz25PT09H0BEADgigTAnZ2d+3H9eNg2LXHvb9E+FAABACoeAFPl76Lw13f/r9mUtmoRAAEA5hQAe69933nD5uj/UAAEAKhoAOwt+Pgr1+cwFoB8Hn9vx159j+Lvm9xzfhEAAQCqGwBXCvp80d8n/u957vofAiAAQEUDYCz+WC3oczv3nMe5I9tOBEAAgOpWAP93Wkh67fv2+tnZ2Xtp9W+uz74ACABQ3QB4Le3zl+uTvvl73qv8vSh4xvcCIABAhVcB1+v1jRFWAb+OZ3x60RnE0b6Lvj+kFhXFZ3Hu75IACABQkgCYTvgoeM07aAuYx4N+7+jo6GZc/3nI/T9tbm7eFAABAEpwEkg64SOubw6r/EUl79tswCbQW1tbH8XikPZFATL1SX0FQACAEpwFnMJd2uQ57fMX7SjaSVrwkb75G/baNwW6uP5yhI2kX/buEQABAOYcAEd2Qfh7Edd2BoXA2Gx6VQAEAKhQABwU/tLJIrGwZC37d5HJWv60kd7r4K4ACABQkQCYwl88rz3gW8G1fP+o9n1ZFAIFQACACgTAUcPfKCFQAAQAKFkAHDf8vWsIFAABAEoUAId98xftwQjjejAoBHY6nXtmDgCgBAFwWuGvPwSmqmHB874ycwAAcw6A0w5/b8Ur40f5Z8Yr4vtmDgBgjgFwVuHPSSAAACUMgJMu+BAAAQAqFABnHf4EQACAEgXAywh/AiAAQEkC4GWFPwEQAKAEAfAyw58ACAAw5wB42eFPAAQAmE3AquUDVrvdvluG8Ncb372CAFgzcwAA4wesxXzAqtfr62UIf73xfVNwFNwtMwcAML6FCFXdXIXtVQS79fSqNdpn0Q4LqnBv0ikdUS1cmkVLv52e32g0TnK/e5jGbNoAALKJqmxPC6p7pWwx1idmDABg8gB4PcJVswLhb+/4+PgDMwYAMKVvAaO1yhz+0hjNFADAFKXqWnx3txHf3XVLFPwO02tflT8AgNlaSCtt02KMg4ODWlrpGyFsZVYLPoYsBPnYgg8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOBK+hsMQ7RDjzvGwwAAAABJRU5ErkJggg==";
|
||||
|
||||
export interface MarkdownImage {
|
||||
source: string;
|
||||
|
||||
Reference in New Issue
Block a user