fix(opencode): browser screenshots reach OpenCode v2 agents (#6330)

* fix(opencode): return browser screenshots to OpenCode v2 agents

The v2 bridge plugin passed a Paseo tool's MCP content straight to
OpenCode. OpenCode v2 tool content is text or a file, so the image block
from browser_screenshot was not valid output: the tool call never
completed and the turn failed.

Convert image blocks to file content with a data URI and keep text
blocks as they are.

Closes #6318

* fix(opencode): show v2 tool images as images, not base64 output

With the bridge now returning screenshots to OpenCode v2 as data URI
file content, the v2 timeline copied each file's URI into the tool
call's output, so the screenshot card showed the whole base64 string.

Keep data URI images out of the tool output, leave an "[image]"
placeholder, and emit each image after the tool call as image markdown,
the way Claude and Codex tool images are shown. History replay uses the
same mapping.

---------

Co-authored-by: paseo-bot[bot] <266920839+paseo-bot[bot]@users.noreply.github.com>
This commit is contained in:
paseo-bot[bot]
2026-10-08 01:01:21 +02:00
committed by GitHub
parent 33eb36c021
commit 99fc204c55
4 changed files with 168 additions and 7 deletions
@@ -300,6 +300,82 @@ describe("OpenCodeBridge", () => {
}
});
test("v2 plugin returns a tool image as OpenCode file content", async () => {
const paseoHome = await mkdtemp(path.join(tmpdir(), "paseo-opencode-v2-image-"));
temporaryDirectories.push(paseoHome);
const png =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
const screenshot = {
name: "browser_screenshot",
title: "Browser screenshot",
description: "Captures the page.",
inputSchema: {},
async handler() {
return {
content: [
{ type: "text", text: "Captured browser screenshot." },
{ type: "image", data: png, mimeType: "image/png" },
],
};
},
};
const tools = new Map([[screenshot.name, screenshot]]);
const catalog: PaseoToolCatalog = {
tools,
getTool: (name) => tools.get(name),
executeTool: async (name, input, context) =>
await tools.get(name)!.handler(input, context ?? {}),
};
const bridge = new OpenCodeBridge({ paseoHome, logger: createTestLogger() });
bridge.setManifestCatalog(catalog);
await bridge.start();
const release = bridge.bindSession({ sessionId: "session", env: {}, tools: catalog });
try {
const env = await bridge.decorateV2ServerEnv({});
const plugin = z
.object({
plugins: z.array(
z.object({
package: z.string(),
options: z.object({ baseUrl: z.string(), token: z.string() }),
}),
),
})
.parse(JSON.parse(env.OPENCODE_CONFIG_CONTENT)).plugins[0]!;
const pluginTools = new Map<string, V2TestTool>();
const module: {
default: { setup(context: V2TestPluginContext): Promise<() => Promise<void>> };
} = await import(pathToFileURL(path.join(fileURLToPath(plugin.package), "server.js")).href);
const dispose = await module.default.setup({
options: plugin.options,
tool: {
transform: async (transform) => {
transform({ add: (tool) => pluginTools.set(tool.name, tool) });
return { dispose: async () => undefined };
},
},
session: {
context: async () => [],
get: async () => ({ parentID: undefined }),
hook: async () => ({ dispose: async () => undefined }),
},
});
const result = await pluginTools
.get("paseo_browser_screenshot")!
.execute({}, { sessionID: "session" });
expect(result).toMatchObject({
content: [
{ type: "text", text: "Captured browser screenshot." },
{ type: "file", mime: "image/png", uri: `data:image/png;base64,${png}` },
],
});
await dispose();
} finally {
release();
await bridge.close();
}
});
test("v2 structured output validates values and clears its tool on ordinary turns", async () => {
const root = await mkdtemp(path.join(tmpdir(), "paseo-opencode-v2-structured-"));
temporaryDirectories.push(root);
@@ -45,7 +45,10 @@ export default {
`/sessions/${encodeURIComponent(binding.sessionID)}/tools/${encodeURIComponent(definition.name)}`,
input,
);
return { content: result.content, metadata: { paseoTool: definition.name } };
return {
content: result.content.map(toOpenCodeContent),
metadata: { paseoTool: definition.name },
};
},
});
}
@@ -65,3 +68,11 @@ export default {
};
},
};
// OpenCode tool content is text or a file; MCP images arrive as base64 data.
function toOpenCodeContent(part) {
if (part.type === "image") {
return { type: "file", uri: `data:${part.mimeType};base64,${part.data}`, mime: part.mimeType };
}
return part;
}
@@ -54,3 +54,44 @@ test("a completed edit carries the replaced and replacement text", () => {
},
]);
});
test("a tool's image output is shown as an image, not as base64 in the tool output", () => {
const png =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==";
const screenshot = assistant([
{
type: "tool",
id: "call-screenshot",
name: "paseo_browser_screenshot",
executed: false,
state: {
status: "completed",
input: {},
content: [
{ type: "text", text: "Captured browser screenshot (1x1)." },
{ type: "file", uri: `data:image/png;base64,${png}`, mime: "image/png" },
],
metadata: {},
},
time: { created: 2, ran: 2, completed: 2 },
},
]);
const timeline = new V2Timeline();
const events = timeline.messages([screenshot]);
expect(JSON.stringify(events)).not.toContain(png);
expect(events.map((event) => event.type === "timeline" && event.item)).toMatchObject([
{
type: "tool_call",
name: "paseo_browser_screenshot",
status: "completed",
detail: { output: "Captured browser screenshot (1x1).\n[image]" },
},
{
type: "assistant_message",
text: expect.stringMatching(/^!\[Image\]\(file:\/\/.*paseo-attachments.*\.png\)$/),
},
]);
expect(timeline.messages([screenshot])).toEqual([]);
});
@@ -1,4 +1,8 @@
import { mapOpencodeToolCall } from "../tool-call-mapper.js";
import {
materializeProviderImage,
renderProviderImageOutputAsAssistantMarkdown,
} from "../../provider-image-output.js";
import type { SessionMessageAssistantTool } from "@opencode/client";
import { STRUCTURED_OUTPUT_TOOL } from "./structured-output.js";
import type { SessionMessageInfo } from "@opencode/client";
@@ -158,8 +162,21 @@ export class V2Timeline {
const serialized = JSON.stringify(part);
if (serialized === this.content.get(key)) return;
this.content.set(key, serialized);
const item = toolFromV2(part);
const { output, images } = splitToolContent(part.state);
const item = toolFromV2(part, output);
if (item) push(item);
// Images follow their tool call as markdown, as Claude and Codex emit them,
// so base64 never reaches the tool output.
images.forEach((image, index) => {
const imageKey = `${key}:image:${index}`;
if (this.content.has(imageKey)) return;
this.content.set(imageKey, "");
const imageItem = renderProviderImageOutputAsAssistantMarkdown(
{ url: image.uri, mimeType: image.mime },
{ materialize: materializeProviderImage },
);
if (imageItem) push(imageItem);
});
}
private compactionMessage(
@@ -187,12 +204,28 @@ interface TextPartIdentity {
interface TextPartDelta extends TextPartIdentity {
delta: string;
}
function toolFromV2(tool: SessionMessageAssistantTool): AgentTimelineItem | null {
function splitToolContent(state: SessionMessageAssistantTool["state"]): {
output: string | undefined;
images: { uri: string; mime: string }[];
} {
if (!("content" in state) || !state.content) return { output: undefined, images: [] };
const images: { uri: string; mime: string }[] = [];
const output = state.content
.map((part) => {
if (part.type === "text") return part.text;
if (!part.uri.startsWith("data:image/")) return part.uri;
images.push(part);
return "[image]";
})
.join("\n");
return { output, images };
}
function toolFromV2(
tool: SessionMessageAssistantTool,
output: string | undefined,
): AgentTimelineItem | null {
const state = tool.state;
const output =
"content" in state
? state.content?.map((part) => (part.type === "text" ? part.text : part.uri)).join("\n")
: undefined;
return mapOpencodeToolCall({
toolName: tool.name,
callId: tool.id,