From a8deb51086f1baa9daa0da2c8b3689b7e362e10f Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 16 Oct 2026 17:23:43 +0200 Subject: [PATCH] fix(server): keep provider images separate from adjacent prose Generated image Markdown was concatenated into the preceding paragraph and rendered at inline size. Preserve paragraph boundaries around image output while retaining intentional inline image behavior. --- .../claude/agent.image-rendering.test.ts | 4 +- .../providers/codex-app-server-agent.test.ts | 6 +- .../providers/opencode/v2/timeline.test.ts | 2 +- .../providers/provider-image-output.test.ts | 61 +++++++++++++++++-- .../agent/providers/provider-image-output.ts | 7 ++- 5 files changed, 66 insertions(+), 14 deletions(-) diff --git a/packages/server/src/server/agent/providers/claude/agent.image-rendering.test.ts b/packages/server/src/server/agent/providers/claude/agent.image-rendering.test.ts index 66891e01bf..5a06fea7fd 100644 --- a/packages/server/src/server/agent/providers/claude/agent.image-rendering.test.ts +++ b/packages/server/src/server/agent/providers/claude/agent.image-rendering.test.ts @@ -140,11 +140,11 @@ function imageMessages(items: AgentTimelineItem[]): string[] { return items .filter((item) => item.type === "assistant_message") .map((item) => (item as { text: string }).text) - .filter((text) => text.startsWith("![")); + .filter((text) => text.trimStart().startsWith("![")); } function markdownImageSource(markdown: string): string { - const match = markdown.match(/^!\[[^\]]*]\((.*)\)$/); + const match = markdown.trim().match(/^!\[[^\]]*]\((.*)\)$/); if (!match) { throw new Error(`Expected markdown image, got: ${markdown}`); } diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts index c8269722a7..8238a80131 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts @@ -502,7 +502,7 @@ function asInternals(session: CodexTestSession): CodexSessionTestAccess { } function markdownImageSource(markdown: string): string { - const match = markdown.match(/^!\[[^\]]*]\((.*)\)$/); + const match = markdown.trim().match(/^!\[[^\]]*]\((.*)\)$/); if (!match) { throw new Error(`Expected markdown image, got: ${markdown}`); } @@ -6000,7 +6000,7 @@ describe("Codex app-server provider", () => { turnId: "test-turn", item: { type: "assistant_message", - text: "![Image](file:///tmp/paseo%20image.png)", + text: "\n\n![Image](file:///tmp/paseo%20image.png)\n\n", }, }, ]); @@ -6032,7 +6032,7 @@ describe("Codex app-server provider", () => { turnId: "test-turn", item: { type: "assistant_message", - text: `![Image](${expectedPath})`, + text: `\n\n![Image](${expectedPath})\n\n`, }, }, ]); diff --git a/packages/server/src/server/agent/providers/opencode/v2/timeline.test.ts b/packages/server/src/server/agent/providers/opencode/v2/timeline.test.ts index 82236de2b8..d86b9a7dcb 100644 --- a/packages/server/src/server/agent/providers/opencode/v2/timeline.test.ts +++ b/packages/server/src/server/agent/providers/opencode/v2/timeline.test.ts @@ -92,7 +92,7 @@ test("a tool's image output is shown as an image, not as base64 in the tool outp }, { type: "assistant_message", - text: expect.stringMatching(/^!\[Image\]\(file:\/\/.*paseo-attachments.*\.png\)$/), + text: expect.stringMatching(/^\n\n!\[Image\]\(file:\/\/.*paseo-attachments.*\.png\)\n\n$/), }, ]); expect(timeline.messages([screenshot])).toEqual([]); diff --git a/packages/server/src/server/agent/providers/provider-image-output.test.ts b/packages/server/src/server/agent/providers/provider-image-output.test.ts index 0c32af04fe..b4f4784a66 100644 --- a/packages/server/src/server/agent/providers/provider-image-output.test.ts +++ b/packages/server/src/server/agent/providers/provider-image-output.test.ts @@ -1,5 +1,7 @@ import { existsSync, rmSync } from "node:fs"; import path from "node:path"; +import MarkdownIt from "markdown-it"; +import { projectTimelineRows } from "../timeline-projection.js"; import { describe, expect, test } from "vitest"; import { @@ -44,7 +46,7 @@ describe("isProviderImageMarkdown", () => { ); expect(markdown).toBe( - `![Image](file:///C:/Users/me/AppData/Local/Temp/paseo-attachments/${HASH}.png)`, + `\n\n![Image](file:///C:/Users/me/AppData/Local/Temp/paseo-attachments/${HASH}.png)\n\n`, ); expect(isProviderImageMarkdown(markdown)).toBe(true); }); @@ -53,20 +55,20 @@ describe("isProviderImageMarkdown", () => { const markdown = renderImageMarkdown("/home/user/Projects/Project With Spaces/screenshot.png"); expect(markdown).toBe( - "![Image](file:///home/user/Projects/Project%20With%20Spaces/screenshot.png)", + "\n\n![Image](file:///home/user/Projects/Project%20With%20Spaces/screenshot.png)\n\n", ); }); test("encodes URI-significant characters in POSIX file paths", () => { const markdown = renderImageMarkdown("/tmp/screenshot#1?draft.png"); - expect(markdown).toBe("![Image](file:///tmp/screenshot%231%3Fdraft.png)"); + expect(markdown).toBe("\n\n![Image](file:///tmp/screenshot%231%3Fdraft.png)\n\n"); }); test("preserves double-leading slashes in POSIX file paths", () => { const markdown = renderImageMarkdown("//tmp/screenshot#1.png"); - expect(markdown).toBe("![Image](file:////tmp/screenshot%231.png)"); + expect(markdown).toBe("\n\n![Image](file:////tmp/screenshot%231.png)\n\n"); }); test.each([ @@ -77,7 +79,7 @@ describe("isProviderImageMarkdown", () => { "file://server/share/shot%3Fdraft.png", ], ])("encodes %s image paths as file URIs", (_label, imagePath, expectedSource) => { - expect(renderImageMarkdown(imagePath)).toBe(`![Image](${expectedSource})`); + expect(renderImageMarkdown(imagePath)).toBe(`\n\n![Image](${expectedSource})\n\n`); }); test("rejects user-authored markdown that is not a materialized attachment", () => { @@ -113,3 +115,52 @@ describe("materializeProviderImage", () => { } }); }); + +describe("provider image paragraph boundaries", () => { + test.each(["", "\n"])("keeps image outputs separate from prose ending in %j", (ending) => { + const image = renderProviderImageOutputAsAssistantMarkdown({ + url: "https://example.com/chart.png", + }); + if (!image) throw new Error("Expected image output"); + const projected = projectTimelineRows({ + mode: "projected", + rows: [ + { + seq: 1, + timestamp: "2026-10-16T15:08:23Z", + item: { type: "assistant_message", text: `Before${ending}`, messageId: "commentary" }, + }, + { seq: 2, timestamp: "2026-10-16T15:08:24Z", item: image }, + { + seq: 3, + timestamp: "2026-10-16T15:08:25Z", + item: { type: "assistant_message", text: "After" }, + }, + ], + }); + expect(projected).toHaveLength(1); + const item = projected[0]!.item; + if (item.type !== "assistant_message") throw new Error("Expected assistant message"); + const paragraphs = new MarkdownIt() + .parse(item.text, {}) + .filter((token) => token.type === "inline"); + expect(paragraphs.map((token) => token.content)).toEqual([ + "Before", + "![Image](https://example.com/chart.png)", + "After", + ]); + }); +}); + +test("materialized image output retains paragraph boundaries and attachment recognition", () => { + const item = renderProviderImageOutputAsAssistantMarkdown( + { data: "YWJjMTIz", mimeType: "image/png" }, + { materialize: () => ({ path: `/tmp/paseo-attachments/${HASH}.png` }) }, + ); + expect(item).toEqual({ + type: "assistant_message", + text: `\n\n![Image](file:///tmp/paseo-attachments/${HASH}.png)\n\n`, + }); + if (!item || item.type !== "assistant_message") throw new Error("Expected image output"); + expect(isProviderImageMarkdown(item.text)).toBe(true); +}); diff --git a/packages/server/src/server/agent/providers/provider-image-output.ts b/packages/server/src/server/agent/providers/provider-image-output.ts index 387158b32f..ba8843918b 100644 --- a/packages/server/src/server/agent/providers/provider-image-output.ts +++ b/packages/server/src/server/agent/providers/provider-image-output.ts @@ -108,7 +108,7 @@ const PROVIDER_IMAGE_MARKDOWN = new RegExp( ); export function isProviderImageMarkdown(text: string): boolean { - return PROVIDER_IMAGE_MARKDOWN.test(text); + return PROVIDER_IMAGE_MARKDOWN.test(text.trimStart()); } interface RenderProviderImageOutputOptions { @@ -178,7 +178,8 @@ export function renderProviderImageOutputAsAssistantMarkdown( const altText = escapeMarkdownImageAlt(nonEmptyString(image.altText) ?? "Image"); return { type: "assistant_message", - text: `![${altText}](${escapeMarkdownImageSource(source)})`, + // Image outputs remain standalone paragraphs when adjacent assistant chunks merge. + text: `\n\n![${altText}](${escapeMarkdownImageSource(source)})\n\n`, }; } @@ -208,6 +209,6 @@ export function renderProviderImageOutputAsAssistantMarkdown( const altText = escapeMarkdownImageAlt(nonEmptyString(image.altText) ?? "Image"); return { type: "assistant_message", - text: `![${altText}](${escapeMarkdownImageSource(materialized.path)})`, + text: `\n\n![${altText}](${escapeMarkdownImageSource(materialized.path)})\n\n`, }; }