diff --git a/packages/server/src/server/agent/providers/opencode/bridge.test.ts b/packages/server/src/server/agent/providers/opencode/bridge.test.ts index 13cd6aa4d8..7130ac13b9 100644 --- a/packages/server/src/server/agent/providers/opencode/bridge.test.ts +++ b/packages/server/src/server/agent/providers/opencode/bridge.test.ts @@ -300,6 +300,82 @@ describe("OpenCodeBridge", () => { } }); + test("v2 plugin returns a tool image as OpenCode file content", async () => { + const paseoHome = await mkdtemp(path.join(tmpdir(), "paseo-opencode-v2-image-")); + temporaryDirectories.push(paseoHome); + const png = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; + const screenshot = { + name: "browser_screenshot", + title: "Browser screenshot", + description: "Captures the page.", + inputSchema: {}, + async handler() { + return { + content: [ + { type: "text", text: "Captured browser screenshot." }, + { type: "image", data: png, mimeType: "image/png" }, + ], + }; + }, + }; + const tools = new Map([[screenshot.name, screenshot]]); + const catalog: PaseoToolCatalog = { + tools, + getTool: (name) => tools.get(name), + executeTool: async (name, input, context) => + await tools.get(name)!.handler(input, context ?? {}), + }; + const bridge = new OpenCodeBridge({ paseoHome, logger: createTestLogger() }); + bridge.setManifestCatalog(catalog); + await bridge.start(); + const release = bridge.bindSession({ sessionId: "session", env: {}, tools: catalog }); + try { + const env = await bridge.decorateV2ServerEnv({}); + const plugin = z + .object({ + plugins: z.array( + z.object({ + package: z.string(), + options: z.object({ baseUrl: z.string(), token: z.string() }), + }), + ), + }) + .parse(JSON.parse(env.OPENCODE_CONFIG_CONTENT)).plugins[0]!; + const pluginTools = new Map(); + const module: { + default: { setup(context: V2TestPluginContext): Promise<() => Promise> }; + } = await import(pathToFileURL(path.join(fileURLToPath(plugin.package), "server.js")).href); + const dispose = await module.default.setup({ + options: plugin.options, + tool: { + transform: async (transform) => { + transform({ add: (tool) => pluginTools.set(tool.name, tool) }); + return { dispose: async () => undefined }; + }, + }, + session: { + context: async () => [], + get: async () => ({ parentID: undefined }), + hook: async () => ({ dispose: async () => undefined }), + }, + }); + const result = await pluginTools + .get("paseo_browser_screenshot")! + .execute({}, { sessionID: "session" }); + expect(result).toMatchObject({ + content: [ + { type: "text", text: "Captured browser screenshot." }, + { type: "file", mime: "image/png", uri: `data:image/png;base64,${png}` }, + ], + }); + await dispose(); + } finally { + release(); + await bridge.close(); + } + }); + test("v2 structured output validates values and clears its tool on ordinary turns", async () => { const root = await mkdtemp(path.join(tmpdir(), "paseo-opencode-v2-structured-")); temporaryDirectories.push(root); diff --git a/packages/server/src/server/agent/providers/opencode/v2/bridge-plugin.mjs b/packages/server/src/server/agent/providers/opencode/v2/bridge-plugin.mjs index b14540a1c3..190e015d8e 100644 --- a/packages/server/src/server/agent/providers/opencode/v2/bridge-plugin.mjs +++ b/packages/server/src/server/agent/providers/opencode/v2/bridge-plugin.mjs @@ -45,7 +45,10 @@ export default { `/sessions/${encodeURIComponent(binding.sessionID)}/tools/${encodeURIComponent(definition.name)}`, input, ); - return { content: result.content, metadata: { paseoTool: definition.name } }; + return { + content: result.content.map(toOpenCodeContent), + metadata: { paseoTool: definition.name }, + }; }, }); } @@ -65,3 +68,11 @@ export default { }; }, }; + +// OpenCode tool content is text or a file; MCP images arrive as base64 data. +function toOpenCodeContent(part) { + if (part.type === "image") { + return { type: "file", uri: `data:${part.mimeType};base64,${part.data}`, mime: part.mimeType }; + } + return part; +} diff --git a/packages/server/src/server/agent/providers/opencode/v2/timeline.test.ts b/packages/server/src/server/agent/providers/opencode/v2/timeline.test.ts index 1bbd86490a..9189f5d3c5 100644 --- a/packages/server/src/server/agent/providers/opencode/v2/timeline.test.ts +++ b/packages/server/src/server/agent/providers/opencode/v2/timeline.test.ts @@ -54,3 +54,44 @@ test("a completed edit carries the replaced and replacement text", () => { }, ]); }); + +test("a tool's image output is shown as an image, not as base64 in the tool output", () => { + const png = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; + const screenshot = assistant([ + { + type: "tool", + id: "call-screenshot", + name: "paseo_browser_screenshot", + executed: false, + state: { + status: "completed", + input: {}, + content: [ + { type: "text", text: "Captured browser screenshot (1x1)." }, + { type: "file", uri: `data:image/png;base64,${png}`, mime: "image/png" }, + ], + metadata: {}, + }, + time: { created: 2, ran: 2, completed: 2 }, + }, + ]); + const timeline = new V2Timeline(); + + const events = timeline.messages([screenshot]); + + expect(JSON.stringify(events)).not.toContain(png); + expect(events.map((event) => event.type === "timeline" && event.item)).toMatchObject([ + { + type: "tool_call", + name: "paseo_browser_screenshot", + status: "completed", + detail: { output: "Captured browser screenshot (1x1).\n[image]" }, + }, + { + type: "assistant_message", + text: expect.stringMatching(/^!\[Image\]\(file:\/\/.*paseo-attachments.*\.png\)$/), + }, + ]); + expect(timeline.messages([screenshot])).toEqual([]); +}); diff --git a/packages/server/src/server/agent/providers/opencode/v2/timeline.ts b/packages/server/src/server/agent/providers/opencode/v2/timeline.ts index 3bc0a5bf76..c58316df9e 100644 --- a/packages/server/src/server/agent/providers/opencode/v2/timeline.ts +++ b/packages/server/src/server/agent/providers/opencode/v2/timeline.ts @@ -1,4 +1,8 @@ import { mapOpencodeToolCall } from "../tool-call-mapper.js"; +import { + materializeProviderImage, + renderProviderImageOutputAsAssistantMarkdown, +} from "../../provider-image-output.js"; import type { SessionMessageAssistantTool } from "@opencode/client"; import { STRUCTURED_OUTPUT_TOOL } from "./structured-output.js"; import type { SessionMessageInfo } from "@opencode/client"; @@ -158,8 +162,21 @@ export class V2Timeline { const serialized = JSON.stringify(part); if (serialized === this.content.get(key)) return; this.content.set(key, serialized); - const item = toolFromV2(part); + const { output, images } = splitToolContent(part.state); + const item = toolFromV2(part, output); if (item) push(item); + // Images follow their tool call as markdown, as Claude and Codex emit them, + // so base64 never reaches the tool output. + images.forEach((image, index) => { + const imageKey = `${key}:image:${index}`; + if (this.content.has(imageKey)) return; + this.content.set(imageKey, ""); + const imageItem = renderProviderImageOutputAsAssistantMarkdown( + { url: image.uri, mimeType: image.mime }, + { materialize: materializeProviderImage }, + ); + if (imageItem) push(imageItem); + }); } private compactionMessage( @@ -187,12 +204,28 @@ interface TextPartIdentity { interface TextPartDelta extends TextPartIdentity { delta: string; } -function toolFromV2(tool: SessionMessageAssistantTool): AgentTimelineItem | null { +function splitToolContent(state: SessionMessageAssistantTool["state"]): { + output: string | undefined; + images: { uri: string; mime: string }[]; +} { + if (!("content" in state) || !state.content) return { output: undefined, images: [] }; + const images: { uri: string; mime: string }[] = []; + const output = state.content + .map((part) => { + if (part.type === "text") return part.text; + if (!part.uri.startsWith("data:image/")) return part.uri; + images.push(part); + return "[image]"; + }) + .join("\n"); + return { output, images }; +} + +function toolFromV2( + tool: SessionMessageAssistantTool, + output: string | undefined, +): AgentTimelineItem | null { const state = tool.state; - const output = - "content" in state - ? state.content?.map((part) => (part.type === "text" ? part.text : part.uri)).join("\n") - : undefined; return mapOpencodeToolCall({ toolName: tool.name, callId: tool.id,