diff --git a/packages/server/src/server/agent/providers/codex-mcp-agent.ts b/packages/server/src/server/agent/providers/codex-mcp-agent.ts index 0c8a64e43..f9a5455f2 100644 --- a/packages/server/src/server/agent/providers/codex-mcp-agent.ts +++ b/packages/server/src/server/agent/providers/codex-mcp-agent.ts @@ -66,6 +66,7 @@ type CodexToolArguments = { [key: string]: unknown }; const DEFAULT_TIMEOUT_MS = 14 * 24 * 60 * 60 * 1000; const CODEX_PROVIDER = "codex" as const; +const CODEX_IMAGE_ATTACHMENT_DIR = "paseo-attachments"; const CODEX_MCP_CAPABILITIES: AgentCapabilityFlags = { supportsStreaming: true, @@ -2310,7 +2311,36 @@ function normalizeEvent(raw: unknown): CodexEvent { return CodexEventSchema.parse(eventRecord); } -function toPromptText(prompt: AgentPromptInput): string { +function getImageExtension(mimeType: string): string { + switch (mimeType) { + case "image/jpeg": + return "jpg"; + case "image/png": + return "png"; + case "image/webp": + return "webp"; + case "image/gif": + return "gif"; + case "image/bmp": + return "bmp"; + case "image/tiff": + return "tiff"; + default: + return "bin"; + } +} + +async function writeImageAttachment(mimeType: string, data: string): Promise { + const attachmentsDir = path.join(os.tmpdir(), CODEX_IMAGE_ATTACHMENT_DIR); + await fs.mkdir(attachmentsDir, { recursive: true }); + const extension = getImageExtension(mimeType); + const filename = `${randomUUID()}.${extension}`; + const filePath = path.join(attachmentsDir, filename); + await fs.writeFile(filePath, Buffer.from(data, "base64")); + return filePath; +} + +async function toPromptText(prompt: AgentPromptInput): Promise { if (typeof prompt === "string") { return prompt; } @@ -2321,8 +2351,13 @@ function toPromptText(prompt: AgentPromptInput): string { continue; } if (chunk.type === "image") { - const dataUrl = `data:${chunk.mimeType};base64,${chunk.data}`; - parts.push(`![user image](${dataUrl})`); + try { + const filePath = await writeImageAttachment(chunk.mimeType, chunk.data); + parts.push(`User attached image: ${filePath}`); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + parts.push(`User attached image (failed to write temp file): ${message}`); + } } } return parts.join("\n\n"); @@ -2846,7 +2881,7 @@ class CodexMcpAgentSession implements AgentSession { const abortController = new AbortController(); this.currentAbortController = abortController; - const promptText = toPromptText(prompt); + const promptText = await toPromptText(prompt); // NOTE: user_message is NOT emitted here because the agent-manager's // recordUserMessage() already handles emitting the user message timeline // event before calling stream(). Emitting here would cause duplicates. diff --git a/plan.md b/plan.md index 8c7ad1ae6..eb3d51422 100644 --- a/plan.md +++ b/plan.md @@ -152,3 +152,23 @@ Improvements to the new agent screen in the app. - Implement Claude-specific image content blocks - Implement Codex/OpenAI image handling - **Done (2025-12-29 09:14)**: WHAT: `packages/server/src/server/agent/agent-sdk-types.ts:39-43` adds `AgentPromptContentBlock` and extends `AgentPromptInput` to include image blocks; `packages/server/src/server/session.ts:317-333` now builds prompt blocks with text + image data; `packages/server/src/server/agent/providers/claude-agent.ts:790-820` maps prompt blocks to Claude SDK image/text content; `packages/server/src/server/agent/providers/codex-mcp-agent.ts:2313-2328` renders image blocks as data URLs in prompt text. RESULT: server prompts now carry image data for Claude and include image payloads for Codex/OpenAI flows instead of a text-only summary. EVIDENCE: Not run (not requested). + +- [x] agent=claude **Retest**: Verify image attachment fixes via Playwright MCP. + + - Create a solid color image using ImageMagick (e.g., `convert -size 100x100 xc:red /tmp/red.png`). + - Navigate to `http://localhost:8081/agent/new`. + - Upload the red image and ask "what color is this image?". + - Test with Claude agent: verify it correctly identifies the color as red. + - Test with Codex agent: verify it correctly identifies the color as red. + - If still broken, add fix tasks with debugging details. + - **Done (2025-12-29 02:53)**: WHAT: Tested via Playwright MCP with `/tmp/red.png` (100x100 solid red image). RESULT: Both agents correctly identify the image as red. EVIDENCE: (1) Claude responded "The image is **red** - a bright, pure red color (appears to be full red with RGB values around 255, 0, 0)." (2) Codex decoded the base64 image via Python/PIL and responded "Solid red (#FF0000)." Server logs confirm image blocks are passed correctly: `[ClaudeAgentSession] Block 1: image, mimeType=image/png, dataLength=428`. NOTE: Codex works but embeds full base64 in context - needs fix below. + +- [x] **Fix**: Codex image attachments - save to temp file instead of base64 in prompt. + + - Current implementation in `codex-mcp-agent.ts:2313-2328` embeds base64 data URL in prompt text via `toPromptText()`. + - This stuffs the entire image into the LLM context (e.g., 1MB image = 1.3MB base64 in prompt). + - The user message UI shows original text, but the actual prompt to Codex contains the embedded base64. + - Fix: Write image to temp file (e.g., `/tmp/paseo-attachments/{uuid}.png`). + - Pass file path in prompt text (e.g., "User attached image: /tmp/paseo-attachments/abc123.png"). + - Codex can then use its Read tool to view the image without bloating context. + - **Done (2025-12-29 10:12)**: WHAT: `packages/server/src/server/agent/providers/codex-mcp-agent.ts:2314-2363` adds temp file helpers and writes image attachments under `/tmp/paseo-attachments` before emitting prompt text; `packages/server/src/server/agent/providers/codex-mcp-agent.ts:2884` now awaits async prompt building. RESULT: Codex prompts reference temp file paths instead of base64 data URLs, avoiding context bloat. EVIDENCE: Not run (not requested).