puter/paseo
1
0
Fork 0
mirror of https://github.com/getpaseo/paseo.git synced 2026-07-29 12:01:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Test Codex MCP images through app-server boundary

Browse Source
This commit is contained in:
Mohamed Boudra
2026-07-03 01:16:57 +02:00
parent 4ca89f21f9
commit 6781521c8e
1 changed files with 109 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

180
packages/server/src/server/agent/providers/codex-app-server-agent.test.ts
View File

@@ -2359,86 +2359,124 @@ describe("Codex app-server provider", () => {
rmSync(source, { force: true });
});
test("emits mcpToolCall image content as a completed tool call plus assistant markdown image", () => {
const session = createSession();
test("mcpToolCall image content emits a completed tool call plus assistant markdown image", async () => {
const appServer = createFakeCodexAppServer();
const session = new CodexAppServerAgentSession(
createConfig({ cwd: "/workspace/project" }),
null,
createTestLogger(),
async () => appServer.child,
);
const events: AgentStreamEvent[] = [];
session.subscribe((event) => events.push(event));
asInternals(session).handleNotification("item/completed", {
item: {
id: "mcp-browser-screenshot",
type: "mcpToolCall",
status: "completed",
server: "paseo",
tool: "browser_screenshot",
arguments: { browserId: "11111111-1111-4111-8111-111111111111" },
result: {
content: [
{ type: "text", text: "Captured browser screenshot (1x1)." },
{ type: "image", data: ONE_BY_ONE_PNG_BASE64, mimeType: "image/png" },
],
structuredContent: {
ok: true,
result: {
command: "screenshot",
browserId: "11111111-1111-4111-8111-111111111111",
mimeType: "image/png",
width: 1,
height: 1,
},
},
},
},
const timelineEvents: Array<Extract<AgentStreamEvent, { type: "timeline" }>> = [];
const timelineItemsReceived = new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
unsubscribe();
reject(new Error("Timed out waiting for MCP image timeline events"));
}, 1000);
const unsubscribe = session.subscribe((event) => {
events.push(event);
if (event.type !== "timeline") {
return;
}
timelineEvents.push(event);
if (timelineEvents.length === 2) {
clearTimeout(timeout);
unsubscribe();
resolve();
}
});
});
expect(events.length).toBe(2);
expect(events[0]).toEqual({
type: "timeline",
provider: "codex",
turnId: "test-turn",
item: {
type: "tool_call",
callId: "mcp-browser-screenshot",
name: "paseo.browser_screenshot",
status: "completed",
error: null,
detail: {
type: "unknown",
input: { browserId: "11111111-1111-4111-8111-111111111111" },
output: {
content: [
{ type: "text", text: "Captured browser screenshot (1x1)." },
{ type: "text", text: "[image]" },
],
structuredContent: {
ok: true,
try {
const { turnId } = await session.startTurn("capture a browser screenshot");
appServer.child.stdout.write(
`${JSON.stringify({
method: "item/completed",
params: {
item: {
id: "mcp-browser-screenshot",
type: "mcpToolCall",
status: "completed",
server: "paseo",
tool: "browser_screenshot",
arguments: { browserId: "11111111-1111-4111-8111-111111111111" },
result: {
command: "screenshot",
browserId: "11111111-1111-4111-8111-111111111111",
mimeType: "image/png",
width: 1,
height: 1,
content: [
{ type: "text", text: "Captured browser screenshot (1x1)." },
{ type: "image", data: ONE_BY_ONE_PNG_BASE64, mimeType: "image/png" },
],
structuredContent: {
ok: true,
result: {
command: "screenshot",
browserId: "11111111-1111-4111-8111-111111111111",
mimeType: "image/png",
width: 1,
height: 1,
},
},
},
},
},
})}\n`,
);
await timelineItemsReceived;
expect(timelineEvents).toEqual([
{
type: "timeline",
provider: "codex",
turnId,
item: {
type: "tool_call",
callId: "mcp-browser-screenshot",
name: "paseo.browser_screenshot",
status: "completed",
error: null,
detail: {
type: "unknown",
input: { browserId: "11111111-1111-4111-8111-111111111111" },
output: {
content: [
{ type: "text", text: "Captured browser screenshot (1x1)." },
{ type: "text", text: "[image]" },
],
structuredContent: {
ok: true,
result: {
command: "screenshot",
browserId: "11111111-1111-4111-8111-111111111111",
mimeType: "image/png",
width: 1,
height: 1,
},
},
},
},
},
},
},
});
expect(events[1]).toMatchObject({
type: "timeline",
provider: "codex",
turnId: "test-turn",
item: { type: "assistant_message" },
});
const imageEvent = events[1];
if (imageEvent?.type !== "timeline" || imageEvent.item.type !== "assistant_message") {
throw new Error("Expected assistant image timeline event");
{
type: "timeline",
provider: "codex",
turnId,
item: expect.objectContaining({ type: "assistant_message" }),
},
]);
const imageEvent = timelineEvents[1];
if (imageEvent.item.type !== "assistant_message") {
throw new Error("Expected assistant image timeline event");
}
expect(JSON.stringify(events)).not.toContain(ONE_BY_ONE_PNG_BASE64);
const source = markdownImageSource(imageEvent.item.text);
expect(source).toMatch(/paseo-attachments[\\/].+\.png$/);
expect(existsSync(source)).toBe(true);
rmSync(source, { force: true });
appServer.assertNoErrors();
} finally {
await session.close();
}
expect(JSON.stringify(events)).not.toContain(ONE_BY_ONE_PNG_BASE64);
const source = markdownImageSource(imageEvent.item.text);
expect(source).toMatch(/paseo-attachments[\\/].+\.png$/);
expect(existsSync(source)).toBe(true);
rmSync(source, { force: true });
});
test("ignores incomplete imageGeneration thread items without failing the turn", () => {