From 99fc204c55c8c1666477282eeba562ba87a3135f Mon Sep 17 00:00:00 2001 From: "paseo-bot[bot]" <266920839+paseo-bot[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 01:01:21 +0200 Subject: [PATCH] fix(opencode): browser screenshots reach OpenCode v2 agents (#6330) * fix(opencode): return browser screenshots to OpenCode v2 agents The v2 bridge plugin passed a Paseo tool's MCP content straight to OpenCode. OpenCode v2 tool content is text or a file, so the image block from browser_screenshot was not valid output: the tool call never completed and the turn failed. Convert image blocks to file content with a data URI and keep text blocks as they are. Closes #6318 * fix(opencode): show v2 tool images as images, not base64 output With the bridge now returning screenshots to OpenCode v2 as data URI file content, the v2 timeline copied each file's URI into the tool call's output, so the screenshot card showed the whole base64 string. Keep data URI images out of the tool output, leave an "[image]" placeholder, and emit each image after the tool call as image markdown, the way Claude and Codex tool images are shown. History replay uses the same mapping. --------- Co-authored-by: paseo-bot[bot] <266920839+paseo-bot[bot]@users.noreply.github.com> --- .../agent/providers/opencode/bridge.test.ts | 76 +++++++++++++++++++ .../providers/opencode/v2/bridge-plugin.mjs | 13 +++- .../providers/opencode/v2/timeline.test.ts | 41 ++++++++++ .../agent/providers/opencode/v2/timeline.ts | 45 +++++++++-- 4 files changed, 168 insertions(+), 7 deletions(-) diff --git a/packages/server/src/server/agent/providers/opencode/bridge.test.ts b/packages/server/src/server/agent/providers/opencode/bridge.test.ts index 13cd6aa4d8..7130ac13b9 100644 --- a/packages/server/src/server/agent/providers/opencode/bridge.test.ts +++ b/packages/server/src/server/agent/providers/opencode/bridge.test.ts @@ -300,6 +300,82 @@ describe("OpenCodeBridge", () => { } }); + test("v2 plugin returns a tool image as OpenCode file content", async () => { + const paseoHome = await mkdtemp(path.join(tmpdir(), "paseo-opencode-v2-image-")); + temporaryDirectories.push(paseoHome); + const png = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; + const screenshot = { + name: "browser_screenshot", + title: "Browser screenshot", + description: "Captures the page.", + inputSchema: {}, + async handler() { + return { + content: [ + { type: "text", text: "Captured browser screenshot." }, + { type: "image", data: png, mimeType: "image/png" }, + ], + }; + }, + }; + const tools = new Map([[screenshot.name, screenshot]]); + const catalog: PaseoToolCatalog = { + tools, + getTool: (name) => tools.get(name), + executeTool: async (name, input, context) => + await tools.get(name)!.handler(input, context ?? {}), + }; + const bridge = new OpenCodeBridge({ paseoHome, logger: createTestLogger() }); + bridge.setManifestCatalog(catalog); + await bridge.start(); + const release = bridge.bindSession({ sessionId: "session", env: {}, tools: catalog }); + try { + const env = await bridge.decorateV2ServerEnv({}); + const plugin = z + .object({ + plugins: z.array( + z.object({ + package: z.string(), + options: z.object({ baseUrl: z.string(), token: z.string() }), + }), + ), + }) + .parse(JSON.parse(env.OPENCODE_CONFIG_CONTENT)).plugins[0]!; + const pluginTools = new Map(); + const module: { + default: { setup(context: V2TestPluginContext): Promise<() => Promise> }; + } = await import(pathToFileURL(path.join(fileURLToPath(plugin.package), "server.js")).href); + const dispose = await module.default.setup({ + options: plugin.options, + tool: { + transform: async (transform) => { + transform({ add: (tool) => pluginTools.set(tool.name, tool) }); + return { dispose: async () => undefined }; + }, + }, + session: { + context: async () => [], + get: async () => ({ parentID: undefined }), + hook: async () => ({ dispose: async () => undefined }), + }, + }); + const result = await pluginTools + .get("paseo_browser_screenshot")! + .execute({}, { sessionID: "session" }); + expect(result).toMatchObject({ + content: [ + { type: "text", text: "Captured browser screenshot." }, + { type: "file", mime: "image/png", uri: `data:image/png;base64,${png}` }, + ], + }); + await dispose(); + } finally { + release(); + await bridge.close(); + } + }); + test("v2 structured output validates values and clears its tool on ordinary turns", async () => { const root = await mkdtemp(path.join(tmpdir(), "paseo-opencode-v2-structured-")); temporaryDirectories.push(root); diff --git a/packages/server/src/server/agent/providers/opencode/v2/bridge-plugin.mjs b/packages/server/src/server/agent/providers/opencode/v2/bridge-plugin.mjs index b14540a1c3..190e015d8e 100644 --- a/packages/server/src/server/agent/providers/opencode/v2/bridge-plugin.mjs +++ b/packages/server/src/server/agent/providers/opencode/v2/bridge-plugin.mjs @@ -45,7 +45,10 @@ export default { `/sessions/${encodeURIComponent(binding.sessionID)}/tools/${encodeURIComponent(definition.name)}`, input, ); - return { content: result.content, metadata: { paseoTool: definition.name } }; + return { + content: result.content.map(toOpenCodeContent), + metadata: { paseoTool: definition.name }, + }; }, }); } @@ -65,3 +68,11 @@ export default { }; }, }; + +// OpenCode tool content is text or a file; MCP images arrive as base64 data. +function toOpenCodeContent(part) { + if (part.type === "image") { + return { type: "file", uri: `data:${part.mimeType};base64,${part.data}`, mime: part.mimeType }; + } + return part; +} diff --git a/packages/server/src/server/agent/providers/opencode/v2/timeline.test.ts b/packages/server/src/server/agent/providers/opencode/v2/timeline.test.ts index 1bbd86490a..9189f5d3c5 100644 --- a/packages/server/src/server/agent/providers/opencode/v2/timeline.test.ts +++ b/packages/server/src/server/agent/providers/opencode/v2/timeline.test.ts @@ -54,3 +54,44 @@ test("a completed edit carries the replaced and replacement text", () => { }, ]); }); + +test("a tool's image output is shown as an image, not as base64 in the tool output", () => { + const png = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; + const screenshot = assistant([ + { + type: "tool", + id: "call-screenshot", + name: "paseo_browser_screenshot", + executed: false, + state: { + status: "completed", + input: {}, + content: [ + { type: "text", text: "Captured browser screenshot (1x1)." }, + { type: "file", uri: `data:image/png;base64,${png}`, mime: "image/png" }, + ], + metadata: {}, + }, + time: { created: 2, ran: 2, completed: 2 }, + }, + ]); + const timeline = new V2Timeline(); + + const events = timeline.messages([screenshot]); + + expect(JSON.stringify(events)).not.toContain(png); + expect(events.map((event) => event.type === "timeline" && event.item)).toMatchObject([ + { + type: "tool_call", + name: "paseo_browser_screenshot", + status: "completed", + detail: { output: "Captured browser screenshot (1x1).\n[image]" }, + }, + { + type: "assistant_message", + text: expect.stringMatching(/^!\[Image\]\(file:\/\/.*paseo-attachments.*\.png\)$/), + }, + ]); + expect(timeline.messages([screenshot])).toEqual([]); +}); diff --git a/packages/server/src/server/agent/providers/opencode/v2/timeline.ts b/packages/server/src/server/agent/providers/opencode/v2/timeline.ts index 3bc0a5bf76..c58316df9e 100644 --- a/packages/server/src/server/agent/providers/opencode/v2/timeline.ts +++ b/packages/server/src/server/agent/providers/opencode/v2/timeline.ts @@ -1,4 +1,8 @@ import { mapOpencodeToolCall } from "../tool-call-mapper.js"; +import { + materializeProviderImage, + renderProviderImageOutputAsAssistantMarkdown, +} from "../../provider-image-output.js"; import type { SessionMessageAssistantTool } from "@opencode/client"; import { STRUCTURED_OUTPUT_TOOL } from "./structured-output.js"; import type { SessionMessageInfo } from "@opencode/client"; @@ -158,8 +162,21 @@ export class V2Timeline { const serialized = JSON.stringify(part); if (serialized === this.content.get(key)) return; this.content.set(key, serialized); - const item = toolFromV2(part); + const { output, images } = splitToolContent(part.state); + const item = toolFromV2(part, output); if (item) push(item); + // Images follow their tool call as markdown, as Claude and Codex emit them, + // so base64 never reaches the tool output. + images.forEach((image, index) => { + const imageKey = `${key}:image:${index}`; + if (this.content.has(imageKey)) return; + this.content.set(imageKey, ""); + const imageItem = renderProviderImageOutputAsAssistantMarkdown( + { url: image.uri, mimeType: image.mime }, + { materialize: materializeProviderImage }, + ); + if (imageItem) push(imageItem); + }); } private compactionMessage( @@ -187,12 +204,28 @@ interface TextPartIdentity { interface TextPartDelta extends TextPartIdentity { delta: string; } -function toolFromV2(tool: SessionMessageAssistantTool): AgentTimelineItem | null { +function splitToolContent(state: SessionMessageAssistantTool["state"]): { + output: string | undefined; + images: { uri: string; mime: string }[]; +} { + if (!("content" in state) || !state.content) return { output: undefined, images: [] }; + const images: { uri: string; mime: string }[] = []; + const output = state.content + .map((part) => { + if (part.type === "text") return part.text; + if (!part.uri.startsWith("data:image/")) return part.uri; + images.push(part); + return "[image]"; + }) + .join("\n"); + return { output, images }; +} + +function toolFromV2( + tool: SessionMessageAssistantTool, + output: string | undefined, +): AgentTimelineItem | null { const state = tool.state; - const output = - "content" in state - ? state.content?.map((part) => (part.type === "text" ? part.text : part.uri)).join("\n") - : undefined; return mapOpencodeToolCall({ toolName: tool.name, callId: tool.id,