From 94107c70c51549a910f7d0e14ba302fa3a2ffc05 Mon Sep 17 00:00:00 2001 From: gengjiawen Date: Wed, 11 Nov 2026 19:50:17 +0800 Subject: [PATCH] fix(app): lay out HTML image rows side by side in Markdown A README screenshot table (an HTML block of cells) rendered as a column of full-width images: the Markdown engine has no per-image size and treats every image on its own line as a full-width block. GitHub shows the cells side by side at their declared width. Top-level HTML blocks that hold only images (tables, centered badge rows, ) now render outside the engine as rows of images at their declared width and height, wrapping when the pane is narrower than the row. Linked images open their link; others open the lightbox. Local images load through the same daemon path as Markdown images. Blocks with any text, headings or other elements, and blocks nested in lists or quotes, stay with the engine. --- docs/markdown-renderer.md | 6 +- packages/app/e2e/browser/file-editing.spec.ts | 23 ++- .../src/components/markdown/definitions.ts | 24 +++ .../markdown/fence/mermaid/segments.ts | 18 +-- .../components/markdown/html-image-blocks.ts | 145 ++++++++++++++++++ .../src/components/markdown/html-images.tsx | 127 +++++++++++++++ .../src/components/markdown/image-decoding.ts | 9 +- .../app/src/components/markdown/images.ts | 49 ++++-- .../src/components/markdown/renderer.test.ts | 97 +++++++++++- .../app/src/components/markdown/renderer.tsx | 62 +++++--- .../app/src/components/markdown/source.ts | 10 +- 11 files changed, 510 insertions(+), 60 deletions(-) create mode 100644 packages/app/src/components/markdown/definitions.ts create mode 100644 packages/app/src/components/markdown/html-image-blocks.ts create mode 100644 packages/app/src/components/markdown/html-images.tsx diff --git a/docs/markdown-renderer.md b/docs/markdown-renderer.md index 11b9f09405..37dc3d8910 100644 --- a/docs/markdown-renderer.md +++ b/docs/markdown-renderer.md @@ -6,7 +6,7 @@ Native selection belongs to the library's text view. Keep an assistant message i ## Block spacing -Every block is separated by one gap, `density.blockGap`: 12 on compact layouts, 16 wider. Inside the library's text, every element style ends with that gap and the library drops it after the last block. Views stacked outside the library (Mermaid segments, streaming head and tail) use the same gap through `MarkdownBlockStack`. Do not give a block type its own margins; change the gap. +Every block is separated by one gap, `density.blockGap`: 12 on compact layouts, 16 wider. Inside the library's text, every element style ends with that gap and the library drops it after the last block. Views stacked outside the library (Mermaid segments, HTML image rows, streaming head and tail) use the same gap through `MarkdownBlockStack`. Do not give a block type its own margins; change the gap. ## Links and images @@ -14,13 +14,15 @@ Workspace file links use the shared file-link resolver. The workspace supplies i Resolve workspace image paths over the existing session RPC, then give the library a cached native file URL or web blob URL. HTTP(S) images pass through unchanged. This preserves relay, SSH and local socket connections, which do not expose a daemon HTTP endpoint. Each mounted renderer owns its image acquisition identity, so a later reply reads a rewritten path again while the earlier mounted reply retains its snapshot. Image taps open the attachment lightbox. File previews resolve relative images against the Markdown file's directory. An image alone on its line becomes its own paragraph before the library sees it, so it renders as a block on every platform; the library only treats sole-child paragraph images as blocks, and with hard soft breaks every newline is a break node, which left own-line images inline on iOS and web. Images that share a line with text stay inline at line height and keep their aspect ratio. A linked image counts as an image, so review-bot badges such as `[![Retrigger](badge.svg)](url) Confidence` stay inline on web; the library's web renderer only checked direct paragraph children, so it rendered them as full-width block images (local patch in `patches/react-native-enriched-markdown+*.patch`). +A top-level HTML block that holds only images, such as a README screenshot table or a centered badge row, renders outside the library on every platform: rows of images side by side at their declared `width` and `height`, wrapping when the pane is narrower than the row. The library has no per-image size and lays every block image out at full width, so these rows stacked into a column. Any text, heading or other element in the block leaves it to the library, and so does nesting it in a list or quote. Native selection cannot cross the row. + ## Engine constraints The pinned nightly has the same React Native package and core source trees as upstream commit `73ac5085eb1fdc743973b3473d4dba65abad5261`. Compatibility was checked with React Native 0.81.5. Native dependencies must be rebuilt when changing the highlighting configuration; JavaScript refresh cannot enable compiled grammars. Math and video are disabled. Task lists and admonitions are enabled; task checkboxes are read-only. Source adaptation spaces adjacent empty GFM table delimiters before escaping prose spoiler delimiters and decodes entity text before the engine can discard it. Code spans and fences keep their literal source. Comment HTML is normalized to Markdown before parsing; details show their summary and body expanded. -Mermaid fences use the existing isolated diagram runtime and fullscreen viewer. Web dispatches them through the library’s code-block override. Native has no block-renderer hook, so the renderer extracts only Mermaid fences and places the existing WebView host between library-rendered segments. This is the only block exception; see [native selection limitations](native-text-selection.md) for diagram grouping and leading inline images. Streaming keeps the last valid diagram visible while the next revision renders. +Mermaid fences use the existing isolated diagram runtime and fullscreen viewer. Web dispatches them through the library’s code-block override. Native has no block-renderer hook, so the renderer extracts only Mermaid fences and places the existing WebView host between library-rendered segments. Apart from HTML image rows, this is the only block exception; see [native selection limitations](native-text-selection.md) for diagram grouping and leading inline images. Streaming keeps the last valid diagram visible while the next revision renders. Native code blocks use the library's copy control and compiled grammars. Web code blocks use `HighlightedCodeBlock` through the library's `renderers.codeBlock` override. The patch also keeps the library's block spacing around the web code block override, preserves web list numbering and link titles, marks pending parses with `aria-busy` so Find waits for rendered text, and carries link syntax through native and web tap events. Web inline code inside a link inherits only the link colour; its code font, size and background stay intact. Keep link variants colour-only: native pills replace text runs and prevent paths from wrapping; Android pills also use the surrounding block’s font size. The package patch is tracked by [upstream PR #997](https://github.com/software-mansion/enriched-markdown/pull/997); remove the patch when the pinned release includes it. diff --git a/packages/app/e2e/browser/file-editing.spec.ts b/packages/app/e2e/browser/file-editing.spec.ts index 7634b57746..2242e2744b 100644 --- a/packages/app/e2e/browser/file-editing.spec.ts +++ b/packages/app/e2e/browser/file-editing.spec.ts @@ -110,8 +110,15 @@ function fileReadCount( async function expectMarkdownImageLoaded(page: Page, name: string): Promise { const image = page.getByRole("button", { name, exact: true }); await expect(image).toBeVisible(); + // Markdown images are the itself; HTML image rows wrap one in a pressable. await expect - .poll(() => image.evaluate((element: HTMLImageElement) => element.naturalWidth)) + .poll(() => + image.evaluate( + (element) => + (element instanceof HTMLImageElement ? element : element.querySelector("img")) + ?.naturalWidth ?? 0, + ), + ) .toBeGreaterThan(0); } @@ -792,7 +799,9 @@ test.describe("CodeMirror workspace file editing", () => { "", "![hero](../data/cache/hero.png?v=2#preview)", "", - // README screenshot rows indent their cells; GitHub ignores that indentation. + "- Screenshots", + "", + // A README screenshot row: GitHub shows the cells side by side at their declared width. "", " ", ' ', @@ -815,6 +824,16 @@ test.describe("CodeMirror workspace file editing", () => { await expectMarkdownImageLoaded(page, "hero"); await expectMarkdownImageLoaded(page, "table sibling"); await expectMarkdownImageLoaded(page, "table hero"); + const tableSibling = await page + .getByRole("button", { name: "table sibling", exact: true }) + .boundingBox(); + const tableHero = await page + .getByRole("button", { name: "table hero", exact: true }) + .boundingBox(); + expect(tableSibling!.width).toBeCloseTo(120, 0); + expect(tableHero!.width).toBeCloseTo(120, 0); + expect(tableHero!.y).toBeCloseTo(tableSibling!.y, 0); + expect(tableHero!.x).toBeGreaterThan(tableSibling!.x + tableSibling!.width); await visibleFilePane.screenshot({ path: test.info().outputPath("markdown-relative-images.png"), }); diff --git a/packages/app/src/components/markdown/definitions.ts b/packages/app/src/components/markdown/definitions.ts new file mode 100644 index 0000000000..3458c0d2bf --- /dev/null +++ b/packages/app/src/components/markdown/definitions.ts @@ -0,0 +1,24 @@ +import type { Definition, Nodes, Root } from "mdast"; + +/** + * The document's link reference definitions as Markdown, for fragments that render + * on their own. Prepend it to each fragment: the first definition of a label wins, + * so fragment-local duplicates cannot override the document's. + */ +export function referenceDefinitionSource(tree: Root): string { + const definitions = new Map(); + function visit(node: Nodes): void { + if (node.type === "definition" && !definitions.has(node.identifier)) + definitions.set(node.identifier, node); + if ("children" in node) node.children.forEach(visit); + } + visit(tree); + return [...definitions.values()] + .map((node) => { + const label = node.identifier.replace(/[\\[\]]/g, "\\$&"); + const url = node.url.replace(/[\\<>]/g, "\\$&"); + const title = node.title ? ` "${node.title.replace(/[\\"]/g, "\\$&")}"` : ""; + return `[${label}]: <${url}>${title}`; + }) + .join("\n"); +} diff --git a/packages/app/src/components/markdown/fence/mermaid/segments.ts b/packages/app/src/components/markdown/fence/mermaid/segments.ts index be2f229051..200caed8c2 100644 --- a/packages/app/src/components/markdown/fence/mermaid/segments.ts +++ b/packages/app/src/components/markdown/fence/mermaid/segments.ts @@ -1,5 +1,6 @@ import { fromMarkdown } from "mdast-util-from-markdown"; -import type { Definition, RootContent } from "mdast"; +import type { RootContent } from "mdast"; +import { referenceDefinitionSource } from "../../definitions"; interface Segment { kind: "markdown" | "mermaid"; @@ -11,11 +12,8 @@ interface Segment { export function splitMermaidFences(markdown: string): Segment[] { if (!/mermaid/i.test(markdown)) return [{ kind: "markdown", id: "0", source: markdown }]; const segments: Segment[] = []; - const definitions = new Map(); let cursor = 0; function visit(node: RootContent): void { - if (node.type === "definition" && !definitions.has(node.identifier)) - definitions.set(node.identifier, node); if (node.type === "code" && node.lang?.toLowerCase() === "mermaid" && node.position) { const start = node.position.start.offset!; const end = node.position.end.offset!; @@ -37,17 +35,9 @@ export function splitMermaidFences(markdown: string): Segment[] { id: String(segments.length), source: markdown.slice(cursor), }); - // Prepend the document's first definitions after splitting, so fragment-local - // duplicates cannot override them. Original source slices and fence offsets stay intact. + // Original source slices and fence offsets stay intact. if (segments.some((segment) => segment.kind === "mermaid")) { - const referenceSource = [...definitions.values()] - .map((node) => { - const label = node.identifier.replace(/[\\[\]]/g, "\\$&"); - const url = node.url.replace(/[\\<>]/g, "\\$&"); - const title = node.title ? ` "${node.title.replace(/[\\"]/g, "\\$&")}"` : ""; - return `[${label}]: <${url}>${title}`; - }) - .join("\n"); + const referenceSource = referenceDefinitionSource(tree); if (referenceSource) for (const segment of segments) { if (segment.kind === "markdown") segment.source = `${referenceSource}\n\n${segment.source}`; diff --git a/packages/app/src/components/markdown/html-image-blocks.ts b/packages/app/src/components/markdown/html-image-blocks.ts new file mode 100644 index 0000000000..90bf0d1875 --- /dev/null +++ b/packages/app/src/components/markdown/html-image-blocks.ts @@ -0,0 +1,145 @@ +import { Parser } from "htmlparser2"; +import { fromMarkdown } from "mdast-util-from-markdown"; +import { referenceDefinitionSource } from "./definitions"; + +export interface HtmlImage { + /** Position in its row; images keep their source order. */ + id: string; + src: string; + alt: string; + href?: string; + width?: number; + height?: number; +} + +export interface HtmlImageRow { + id: string; + align: "left" | "center" | "right"; + images: HtmlImage[]; +} + +export type MarkdownBlock = + | { kind: "markdown"; id: string; source: string } + | { kind: "images"; id: string; rows: HtmlImageRow[] }; + +// An HTML block that holds nothing but images, like a README screenshot table or a +// centered row of badges. GitHub lays these out side by side at their declared size, +// which the Markdown engine cannot do, so they render outside it. +const ROW_TAGS = new Set(["table", "tr", "p", "div", "center", "br"]); +const CELL_TAGS = new Set(["thead", "tbody", "tfoot", "td", "th", "a", "picture", "source"]); +const VOID_TAGS = new Set(["img", "br", "source"]); + +/** Split top-level HTML image blocks out of a Markdown document. */ +export function splitHtmlImageBlocks(source: string): MarkdownBlock[] { + if (!/ { + const text = source.slice(cursor, end); + if (text.trim()) blocks.push({ kind: "markdown", id: String(blocks.length), source: text }); + }; + for (const node of tree.children) { + if (node.type !== "html" || !node.position) continue; + const rows = parseHtmlImageRows(node.value); + if (!rows) continue; + pushMarkdown(node.position.start.offset!); + blocks.push({ kind: "images", id: String(blocks.length), rows }); + cursor = node.position.end.offset!; + } + if (blocks.length === 0) return [{ kind: "markdown", id: "0", source }]; + pushMarkdown(source.length); + // Each fragment parses on its own, so give every one the document's link definitions. + const definitions = referenceDefinitionSource(tree); + if (definitions) + for (const block of blocks) { + if (block.kind === "markdown") block.source = `${definitions}\n\n${block.source}`; + } + return blocks; +} + +function parseHtmlImageRows(html: string): HtmlImageRow[] | null { + const rows: HtmlImageRow[] = []; + const aligns: (HtmlImageRow["align"] | undefined)[] = []; + const links: (string | undefined)[] = []; + let row: HtmlImageRow | null = null; + let valid = true; + const parser = new Parser( + { + onopentag(name, attributes) { + if (name === "img") { + if (!attributes.src) { + valid = false; + return; + } + if (!row) { + row = { + id: String(rows.length), + align: aligns.findLast(Boolean) ?? "left", + images: [], + }; + rows.push(row); + } + row.images.push({ + id: String(row.images.length), + src: attributes.src, + alt: attributes.alt ?? "", + ...(links.at(-1) ? { href: links.at(-1) } : {}), + ...dimension("width", attributes.width), + ...dimension("height", attributes.height), + }); + return; + } + if (!ROW_TAGS.has(name) && !CELL_TAGS.has(name)) valid = false; + if (ROW_TAGS.has(name)) row = null; + if (VOID_TAGS.has(name)) return; + aligns.push(name === "center" ? "center" : alignment(attributes.align)); + links.push(name === "a" ? attributes.href : links.at(-1)); + }, + onclosetag(name) { + if (VOID_TAGS.has(name)) return; + if (ROW_TAGS.has(name)) row = null; + aligns.pop(); + links.pop(); + }, + ontext(text) { + if (text.trim()) valid = false; + }, + }, + { decodeEntities: true, lowerCaseTags: true, lowerCaseAttributeNames: true }, + ); + parser.end(html); + return valid && rows.length > 0 ? rows : null; +} + +function alignment(value: string | undefined): HtmlImageRow["align"] | undefined { + const align = value?.trim().toLowerCase(); + return align === "center" || align === "right" || align === "left" ? align : undefined; +} + +function dimension(key: "width" | "height", value: string | undefined) { + const match = value ? /^\s*(\d+(?:\.\d+)?)(?:px)?\s*$/i.exec(value) : null; + const parsed = match ? Number(match[1]) : 0; + return parsed > 0 ? { [key]: parsed } : {}; +} + +// Block images in the Markdown engine are 320 tall; undeclared images stay within that. +const UNDECLARED_MAX_HEIGHT = 320; + +/** Width and aspect ratio of an HTML image, from its attributes and, once decoded, its pixels. */ +export function htmlImageBox( + image: Pick, + natural: { width: number; height: number } | undefined, +): { width: number; aspectRatio: number } { + const ratio = natural ? natural.width / natural.height : undefined; + if (image.width && image.height) + return { width: image.width, aspectRatio: image.width / image.height }; + if (image.width) return { width: image.width, aspectRatio: ratio ?? 1 }; + if (image.height) return { width: image.height * (ratio ?? 1), aspectRatio: ratio ?? 1 }; + if (natural && ratio) + return { + width: natural.width * Math.min(1, UNDECLARED_MAX_HEIGHT / natural.height), + aspectRatio: ratio, + }; + return { width: 0, aspectRatio: 1 }; +} diff --git a/packages/app/src/components/markdown/html-images.tsx b/packages/app/src/components/markdown/html-images.tsx new file mode 100644 index 0000000000..2f65aa0b20 --- /dev/null +++ b/packages/app/src/components/markdown/html-images.tsx @@ -0,0 +1,127 @@ +import { useMemo, useState } from "react"; +import { Image, Text, View } from "react-native"; +import { StyleSheet } from "react-native-unistyles"; +import { useAssistantFileLinkActions } from "@/assistant-file-links"; +import { markdownCopyImageSourcesDataSet } from "@/assistant-selection-copy/markup"; +import { AttachmentLightbox, type ImageLightboxSource } from "@/components/attachment-lightbox"; +import { PressHighlight } from "@/components/ui/press-highlight"; +import { isWeb } from "@/constants/platform"; +import { useStableEvent } from "@/hooks/use-stable-event"; +import { htmlImageBox, type HtmlImage, type HtmlImageRow } from "./html-image-blocks"; +import type { MarkdownImageSize } from "./image-decoding"; +import { useMarkdownImageContext, useMarkdownImages, type MarkdownImageContext } from "./images"; +import { isSvgImageSource, markdownLinkSource } from "./source"; + +/** Images from an HTML block, side by side at their declared size, as GitHub shows them. */ +export function MarkdownHtmlImages({ + rows, + onLinkPress, + ...context +}: MarkdownImageContext & { + rows: HtmlImageRow[]; + onLinkPress?: (url: string) => unknown; +}) { + // Native renders no SVG, so those images show their alt text, as they do in Markdown. + const sources = useMemo( + () => + [...new Set(rows.flatMap((row) => row.images.map((image) => image.src)))].filter( + (source) => isWeb || !isSvgImageSource(source), + ), + [rows], + ); + const imageContext = useMarkdownImageContext(context); + const images = useMarkdownImages(sources, imageContext); + const actions = useAssistantFileLinkActions(); + const [lightbox, setLightbox] = useState(null); + const pressImage = useStableEvent((image: HtmlImage, uri: string) => { + if (!image.href) { + setLightbox({ type: "uri", uri }); + return; + } + const link = markdownLinkSource(image.href); + if (onLinkPress && !actions.canResolveFile(link)) onLinkPress(image.href); + else actions.open(link, "preferred"); + }); + const closeLightbox = useStableEvent(() => setLightbox(null)); + const copySources = Object.fromEntries( + [...images.resolved].map(([source, url]) => [url, source]), + ); + return ( + + {rows.map((row) => ( + + {row.images.map((image) => { + const index = sources.indexOf(image.src); + return ( + + ); + })} + + ))} + {lightbox ? : null} + + ); +} + +function HtmlImageView({ + image, + status, + uri, + size, + onPress, +}: { + image: HtmlImage; + status: "pending" | "ready" | "unavailable"; + uri: string | undefined; + size: MarkdownImageSize | undefined; + onPress: (image: HtmlImage, uri: string) => void; +}) { + const { width, aspectRatio } = htmlImageBox(image, size); + const box = useMemo(() => ({ width, aspectRatio }), [width, aspectRatio]); + const source = useMemo(() => (uri ? { uri } : undefined), [uri]); + const press = useStableEvent(() => { + if (uri) onPress(image, uri); + }); + if (status === "unavailable") + return image.alt ? {image.alt} : null; + if (status === "pending" || !source) return ; + return ( + + + + ); +} + +const styles = StyleSheet.create((theme) => ({ + rows: { gap: theme.spacing[2] }, + row: { flexDirection: "row", flexWrap: "wrap", alignItems: "flex-start", gap: theme.spacing[2] }, + alignLeft: { justifyContent: "flex-start" }, + alignCenter: { justifyContent: "center" }, + alignRight: { justifyContent: "flex-end" }, + // Narrow panes shrink an image to the row; its aspect ratio keeps the height in step. + frame: { maxWidth: "100%", borderRadius: theme.borderRadius.lg, overflow: "hidden" }, + pending: { backgroundColor: theme.colors.surface2 }, + image: { width: "100%", height: "100%" }, + alt: { color: theme.colors.foregroundMuted, fontSize: theme.fontSize.base }, +})); + +const ALIGN = { + left: styles.alignLeft, + center: styles.alignCenter, + right: styles.alignRight, +} as const; diff --git a/packages/app/src/components/markdown/image-decoding.ts b/packages/app/src/components/markdown/image-decoding.ts index efd1ad3521..ad820dcc28 100644 --- a/packages/app/src/components/markdown/image-decoding.ts +++ b/packages/app/src/components/markdown/image-decoding.ts @@ -4,12 +4,17 @@ type ImageDecoder = ( failure: (error: unknown) => void, ) => void; -export function decodeMarkdownImage(uri: string, decode: ImageDecoder): Promise { +export interface MarkdownImageSize { + width: number; + height: number; +} + +export function decodeMarkdownImage(uri: string, decode: ImageDecoder): Promise { return new Promise((resolve, reject) => { decode( uri, (width, height) => { - if (width > 0 && height > 0) resolve(true); + if (width > 0 && height > 0) resolve({ width, height }); else reject(new Error("Image has no decodable dimensions")); }, reject, diff --git a/packages/app/src/components/markdown/images.ts b/packages/app/src/components/markdown/images.ts index 9a52491125..f25808acd6 100644 --- a/packages/app/src/components/markdown/images.ts +++ b/packages/app/src/components/markdown/images.ts @@ -2,6 +2,7 @@ import { Image } from "react-native"; import { isWeb } from "@/constants/platform"; import { useMemo, useEffect, useId } from "react"; import { useFetchQueries } from "@/data/query"; +import { useAssistantFileLinkEnvironment } from "@/assistant-file-links"; import { createAssistantImageFileAcquisition } from "@/assistant-image/file-acquisition"; import { persistAttachmentFromBytes, @@ -35,12 +36,27 @@ export interface MarkdownImageContext { filePath?: string; } +/** Fill in the workspace, host, and file reader of the surrounding assistant view. */ +export function useMarkdownImageContext(context: MarkdownImageContext): MarkdownImageContext { + const environment = useAssistantFileLinkEnvironment(); + const client = + context.client ?? + (environment.client?.readFile + ? { readFile: environment.client.readFile.bind(environment.client) } + : undefined); + return { + workspaceRoot: environment.workspaceRoot, + serverId: environment.serverId, + ...context, + client, + }; +} + export function useMarkdownSource( text: string, context: MarkdownImageContext, links: { commandNames?: ReadonlySet; forgeReferencePattern?: string } = {}, ) { - const occurrenceKey = useId(); const { commandNames, forgeReferencePattern } = links; const prepared = useMemo( () => @@ -52,6 +68,22 @@ export function useMarkdownSource( [text, context.workspaceRoot, commandNames, forgeReferencePattern], ); const sources = [...new Set(prepared.images.map((image) => image.source))]; + const { resolved, statuses } = useMarkdownImages(sources, context); + const unavailableImages = new Set( + sources.filter((_source, index) => statuses[index] === "unavailable"), + ); + const pendingImages = new Set(sources.filter((_source, index) => statuses[index] === "pending")); + return { + markdown: prepared.render(resolved, unavailableImages, pendingImages), + linkRecognition: prepared.linkRecognition, + linkSource: prepared.linkSource, + imageSources: Object.fromEntries([...resolved].map(([source, url]) => [url, source])), + }; +} + +/** Load Markdown image sources: local files through the daemon, then decode each one. */ +export function useMarkdownImages(sources: readonly string[], context: MarkdownImageContext) { + const occurrenceKey = useId(); const acquisitions = sources.map((source) => { const resolved = resolveMarkdownPreviewImageSource({ source, @@ -127,17 +159,14 @@ export function useMarkdownSource( url: displayUrls[index], acquisitionFailed: results[index].isError, decodeFailed: decoded[index].isError, - decoded: decoded[index].data === true, + decoded: decoded[index].data !== undefined, }), ); - const unavailableImages = new Set( - sources.filter((_source, index) => statuses[index] === "unavailable"), - ); - const pendingImages = new Set(sources.filter((_source, index) => statuses[index] === "pending")); return { - markdown: prepared.render(resolved, unavailableImages, pendingImages), - linkRecognition: prepared.linkRecognition, - linkSource: prepared.linkSource, - imageSources: Object.fromEntries([...resolved].map(([source, url]) => [url, source])), + /** Daemon-read sources and the local URL each one displays from. */ + resolved, + statuses, + urls: displayUrls, + sizes: decoded.map((result) => result.data), }; } diff --git a/packages/app/src/components/markdown/renderer.test.ts b/packages/app/src/components/markdown/renderer.test.ts index 3dee3561c6..2905fe3087 100644 --- a/packages/app/src/components/markdown/renderer.test.ts +++ b/packages/app/src/components/markdown/renderer.test.ts @@ -15,6 +15,7 @@ import { darkTheme, lightTheme } from "@/styles/theme"; import { createMarkdownStyles } from "./styles"; import { prepareMarkdownSource, PENDING_MARKDOWN_IMAGE } from "./source"; import { normalizeMarkdownHtml } from "./html"; +import { htmlImageBox, splitHtmlImageBlocks } from "./html-image-blocks"; it("recognizes complete known commands in inline code", () => { const names = new Set(["code-review", "review:fast"]); @@ -779,6 +780,100 @@ describe("GitHub comment HTML normalization", () => { }); }); +describe("HTML image blocks", () => { + // A README screenshot row; GitHub shows its cells side by side at the declared width. + const gallery = [ + "
table sibling
", + " ", + ' ', + ' ', + " ", + "
OneTwo
", + ].join("\n"); + + it("lays out a README screenshot table as one row of images", () => { + expect(splitHtmlImageBlocks(`- Feature\n\n${gallery}\n\n## Next`)).toEqual([ + { kind: "markdown", id: "0", source: "- Feature\n\n" }, + { + kind: "images", + id: "1", + rows: [ + { + id: "0", + align: "left", + images: [ + { id: "0", src: "shots/one.png", alt: "One", width: 120 }, + { id: "1", src: "shots/two.png", alt: "Two", width: 120 }, + ], + }, + ], + }, + { kind: "markdown", id: "2", source: "\n\n## Next" }, + ]); + }); + + it("keeps the alignment, links, and rows of a centered badge block", () => { + const badges = [ + '

', + ' CI ', + ' Docs
', + ' ', + "

", + ].join("\n"); + expect(splitHtmlImageBlocks(badges)).toEqual([ + { + kind: "images", + id: "0", + rows: [ + { + id: "0", + align: "center", + images: [ + { id: "0", src: "ci.svg", alt: "CI", href: "https://example.com/ci" }, + { id: "1", src: "docs.png", alt: "Docs", href: "docs/guide.md", height: 20 }, + ], + }, + { id: "1", align: "center", images: [{ id: "0", src: "light.png", alt: "", width: 64 }] }, + ], + }, + ]); + }); + + it.each([ + ["text beside the images", '
Caption
'], + ["a heading", '

Title

'], + ["a collapsed section", '
Shots
'], + ["an image without a source", '

Missing

'], + ["inline HTML in a paragraph", 'See here'], + ["a table inside a list item", `- Shots\n\n${gallery.replace(/^/gm, " ")}`], + ])("leaves %s to the Markdown engine", (_name, source) => { + expect(splitHtmlImageBlocks(source)).toEqual([{ kind: "markdown", id: "0", source }]); + }); + + it("gives every Markdown fragment the document's link definitions", () => { + const blocks = splitHtmlImageBlocks( + 'See [guide].\n\n

\n\n[More][guide]\n\n[guide]: /docs/guide.md', + ); + expect(blocks.map((block) => (block.kind === "markdown" ? block.source : "images"))).toEqual([ + "[guide]: \n\nSee [guide].\n\n", + "images", + "[guide]: \n\n\n\n[More][guide]\n\n[guide]: /docs/guide.md", + ]); + }); + + it.each([ + ["both dimensions", { width: 120, height: 60 }, { width: 200, height: 200 }, 120, 2], + ["a width", { width: 120 }, { width: 200, height: 100 }, 120, 2], + ["a width before decoding", { width: 120 }, undefined, 120, 1], + ["a height", { height: 20 }, { width: 100, height: 20 }, 100, 5], + ["no dimensions", {}, { width: 1600, height: 800 }, 640, 2], + ["no dimensions on a small image", {}, { width: 100, height: 50 }, 100, 2], + ["no dimensions before decoding", {}, undefined, 0, 1], + ])("sizes an image with %s", (_name, declared, natural, width, aspectRatio) => { + expect(htmlImageBox(declared, natural)).toEqual({ width, aspectRatio }); + }); +}); + describe("indented HTML blocks", () => { // A README screenshot row: GitHub ignores the indentation of the nested tags. const gallery = [ @@ -946,7 +1041,7 @@ it("retries transient image decoding failures instead of caching an unavailable else success(20, 20); }), }); - expect(decoded).toBe(true); + expect(decoded).toEqual({ width: 20, height: 20 }); expect(attempts).toBe(2); client.clear(); }); diff --git a/packages/app/src/components/markdown/renderer.tsx b/packages/app/src/components/markdown/renderer.tsx index 19d3134eec..43ceda8f45 100644 --- a/packages/app/src/components/markdown/renderer.tsx +++ b/packages/app/src/components/markdown/renderer.tsx @@ -1,10 +1,7 @@ import { memo, useMemo, useState } from "react"; import { EnrichedMarkdownText, type LinkPressEvent } from "react-native-enriched-markdown"; import { withUnistyles } from "react-native-unistyles"; -import { - useAssistantFileLinkActions, - useAssistantFileLinkEnvironment, -} from "@/assistant-file-links"; +import { useAssistantFileLinkActions } from "@/assistant-file-links"; import { useStableEvent } from "@/hooks/use-stable-event"; import { matchCommandLink, useAgentCommandHints, type CommandHints } from "@/agent-commands"; import { useForgeShorthand } from "@/git/forge-shorthand"; @@ -18,7 +15,9 @@ import { WEB_RENDERERS } from "./renderers"; import { createMarkdownStyles } from "./styles"; import { PENDING_MARKDOWN_IMAGE } from "./source"; import { splitStreamingMarkdown } from "./streaming"; -import { useMarkdownSource, type MarkdownImageContext } from "./images"; +import { useMarkdownImageContext, useMarkdownSource, type MarkdownImageContext } from "./images"; +import { splitHtmlImageBlocks } from "./html-image-blocks"; +import { MarkdownHtmlImages } from "./html-images"; import { MermaidFence, MermaidStreamingContext, splitMermaidFences } from "./fence/mermaid"; @@ -78,7 +77,33 @@ const CommandMarkdownPart = memo(function CommandMarkdownPart(props: MarkdownRen return ; }); -const MarkdownPart = memo(function MarkdownPart({ +type MarkdownPartProps = MarkdownRendererProps & { commandHints?: CommandHints | null }; + +const MarkdownPart = memo(function MarkdownPart(props: MarkdownPartProps) { + const blocks = useMemo(() => splitHtmlImageBlocks(props.text), [props.text]); + if (blocks.length === 1 && blocks[0]!.kind === "markdown") return ; + return ( + + {blocks.map((block) => + block.kind === "markdown" ? ( + + ) : ( + + ), + )} + + ); +}); + +function MarkdownText({ text, compact = false, muted = false, @@ -88,25 +113,14 @@ const MarkdownPart = memo(function MarkdownPart({ agentCommands: _agentCommands, commandHints = null, ...imageContext -}: MarkdownRendererProps & { commandHints?: CommandHints | null }) { - const environment = useAssistantFileLinkEnvironment(); +}: MarkdownPartProps) { const actions = useAssistantFileLinkActions(); const forgeShorthand = useForgeShorthand(); - const client = - imageContext.client ?? - (environment.client?.readFile - ? { readFile: environment.client.readFile.bind(environment.client) } - : undefined); - const { markdown, imageSources, linkRecognition, linkSource } = useMarkdownSource( - text, - { - workspaceRoot: environment.workspaceRoot, - serverId: environment.serverId, - ...imageContext, - client, - }, - { commandNames: commandHints?.names, forgeReferencePattern: forgeShorthand?.pattern }, - ); + const context = useMarkdownImageContext(imageContext); + const { markdown, imageSources, linkRecognition, linkSource } = useMarkdownSource(text, context, { + commandNames: commandHints?.names, + forgeReferencePattern: forgeShorthand?.pattern, + }); const [image, setImage] = useState(null); const pressLink = useStableEvent(({ url, sourceType }: LinkPressEvent) => { const command = commandHints && matchCommandLink(url, sourceType, commandHints.names); @@ -162,4 +176,4 @@ const MarkdownPart = memo(function MarkdownPart({ {image ? : null} ); -}); +} diff --git a/packages/app/src/components/markdown/source.ts b/packages/app/src/components/markdown/source.ts index 8b15b8ff98..577f3a8eb1 100644 --- a/packages/app/src/components/markdown/source.ts +++ b/packages/app/src/components/markdown/source.ts @@ -130,6 +130,10 @@ function workspaceHoverTitle(link: Link, markup: string, workspaceRoot?: string) return link.url.slice(workspaceRoot.length + 1); } +export function isSvgImageSource(url: string) { + return /(?:\.svg(?:[?#]|$)|^data:image\/svg\+xml)/i.test(url); +} + /** Preserve source spelling; adapt workspace images, spoiler delimiters, and entity text. */ export function prepareMarkdownSource( source: string, @@ -180,11 +184,7 @@ export function prepareMarkdownSource( end: number, ) { const target = node.type === "image" ? node : definitions.get(node.identifier); - if ( - target && - options.supportsSvg === false && - /(?:\.svg(?:[?#]|$)|^data:image\/svg\+xml)/i.test(target.url) - ) { + if (target && options.supportsSvg === false && isSvgImageSource(target.url)) { edits.push({ start, end, text: escapeMarkdownText(node.alt ?? "") }); return; }