diff --git a/packages/server/src/server/agent/agent-sdk-types.ts b/packages/server/src/server/agent/agent-sdk-types.ts index 58f63a4e6..1877ccae5 100644 --- a/packages/server/src/server/agent/agent-sdk-types.ts +++ b/packages/server/src/server/agent/agent-sdk-types.ts @@ -36,7 +36,11 @@ export type AgentPersistenceHandle = { metadata?: AgentMetadata; }; -export type AgentPromptInput = string | { type: "text"; text: string }[]; +export type AgentPromptContentBlock = + | { type: "text"; text: string } + | { type: "image"; data: string; mimeType: string }; + +export type AgentPromptInput = string | AgentPromptContentBlock[]; export type AgentRunOptions = { outputSchema?: unknown; diff --git a/packages/server/src/server/agent/providers/claude-agent.ts b/packages/server/src/server/agent/providers/claude-agent.ts index 998356cca..101094cf5 100644 --- a/packages/server/src/server/agent/providers/claude-agent.ts +++ b/packages/server/src/server/agent/providers/claude-agent.ts @@ -29,6 +29,7 @@ import type { AgentPermissionResponse, AgentPermissionUpdate, AgentPersistenceHandle, + AgentPromptContentBlock, AgentPromptInput, AgentRunOptions, AgentRunResult, @@ -787,15 +788,32 @@ class ClaudeAgentSession implements AgentSession { } private toSdkUserMessage(prompt: AgentPromptInput): SDKUserMessage { - const text = Array.isArray(prompt) - ? prompt.map((chunk) => chunk.text).join("\n\n") - : prompt; + const content = Array.isArray(prompt) + ? prompt.flatMap((chunk: AgentPromptContentBlock) => { + if (chunk.type === "text") { + return [{ type: "text", text: chunk.text }]; + } + if (chunk.type === "image") { + return [ + { + type: "image", + source: { + type: "base64", + media_type: chunk.mimeType, + data: chunk.data, + }, + }, + ]; + } + return []; + }) + : [{ type: "text", text: prompt }]; return { type: "user", message: { role: "user", - content: [{ type: "text", text }], + content, }, parent_tool_use_id: null, session_id: this.claudeSessionId ?? this.pendingLocalId, diff --git a/packages/server/src/server/agent/providers/codex-mcp-agent.ts b/packages/server/src/server/agent/providers/codex-mcp-agent.ts index 976664090..0c8a64e43 100644 --- a/packages/server/src/server/agent/providers/codex-mcp-agent.ts +++ b/packages/server/src/server/agent/providers/codex-mcp-agent.ts @@ -2314,7 +2314,18 @@ function toPromptText(prompt: AgentPromptInput): string { if (typeof prompt === "string") { return prompt; } - return prompt.map((chunk) => chunk.text).join(""); + const parts: string[] = []; + for (const chunk of prompt) { + if (chunk.type === "text") { + parts.push(chunk.text); + continue; + } + if (chunk.type === "image") { + const dataUrl = `data:${chunk.mimeType};base64,${chunk.data}`; + parts.push(`![user image](${dataUrl})`); + } + } + return parts.join("\n\n"); } function getCodexMcpCommand(): string { diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 6e35f0de0..8b43579a0 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -42,6 +42,7 @@ import type { ManagedAgent } from "./agent/agent-manager.js"; import { toAgentPayload } from "./agent/agent-projections.js"; import type { AgentPermissionResponse, + AgentPromptContentBlock, AgentPromptInput, AgentSessionConfig, AgentStreamEvent, @@ -321,16 +322,14 @@ export class Session { if (!images || images.length === 0) { return normalized; } - - const attachmentSummary = images - .map((image, index) => { - const sizeKb = Math.round((image.data.length * 0.75) / 1024); - return `Attachment ${index + 1}: ${image.mimeType}, ~${sizeKb}KB base64`; - }) - .join("\n"); - - const base = normalized.length > 0 ? normalized : "User shared image attachment(s)."; - return `${base}\n\n[Image attachments]\n${attachmentSummary}\n(Actual image bytes omitted; request a screenshot or file if needed.)`; + const blocks: AgentPromptContentBlock[] = []; + if (normalized.length > 0) { + blocks.push({ type: "text", text: normalized }); + } + for (const image of images) { + blocks.push({ type: "image", data: image.data, mimeType: image.mimeType }); + } + return blocks; } /** diff --git a/plan.md b/plan.md index b92d8c633..8c7ad1ae6 100644 --- a/plan.md +++ b/plan.md @@ -146,8 +146,9 @@ Improvements to the new agent screen in the app. - Test on web, iOS, and Android - **Done (2025-12-29 09:09)**: WHAT: `packages/app/src/contexts/session-context.tsx:1288-1334` adds web-specific image base64 conversion using fetch + FileReader (including data URI handling) and keeps native `FileSystem.readAsStringAsync` for non-web. RESULT: web image attachments are encoded without relying on `expo-file-system` web support. EVIDENCE: Not run (not requested). -- [ ] **Fix**: Implement multimodal prompt building on server +- [x] **Fix**: Implement multimodal prompt building on server - Modify `buildAgentPrompt` in `session.ts:316-334` to return structured content with images - Update `AgentPromptInput` type to support content blocks - Implement Claude-specific image content blocks - Implement Codex/OpenAI image handling + - **Done (2025-12-29 09:14)**: WHAT: `packages/server/src/server/agent/agent-sdk-types.ts:39-43` adds `AgentPromptContentBlock` and extends `AgentPromptInput` to include image blocks; `packages/server/src/server/session.ts:317-333` now builds prompt blocks with text + image data; `packages/server/src/server/agent/providers/claude-agent.ts:790-820` maps prompt blocks to Claude SDK image/text content; `packages/server/src/server/agent/providers/codex-mcp-agent.ts:2313-2328` renders image blocks as data URLs in prompt text. RESULT: server prompts now carry image data for Claude and include image payloads for Codex/OpenAI flows instead of a text-only summary. EVIDENCE: Not run (not requested).