Add multimodal prompt blocks for agent prompts

packages/server/src/server/agent/agent-sdk-types.ts

@@ -36,7 +36,11 @@ export type AgentPersistenceHandle = {
   metadata?: AgentMetadata;
 };
 
-export type AgentPromptInput = string | { type: "text"; text: string }[];
+export type AgentPromptContentBlock =
+  | { type: "text"; text: string }
+  | { type: "image"; data: string; mimeType: string };
+
+export type AgentPromptInput = string | AgentPromptContentBlock[];
 
 export type AgentRunOptions = {
   outputSchema?: unknown;

packages/server/src/server/agent/providers/claude-agent.ts

@@ -29,6 +29,7 @@ import type {
   AgentPermissionResponse,
   AgentPermissionUpdate,
   AgentPersistenceHandle,
+  AgentPromptContentBlock,
   AgentPromptInput,
   AgentRunOptions,
   AgentRunResult,
@@ -787,15 +788,32 @@ class ClaudeAgentSession implements AgentSession {
   }
 
   private toSdkUserMessage(prompt: AgentPromptInput): SDKUserMessage {
-    const text = Array.isArray(prompt)
-      ? prompt.map((chunk) => chunk.text).join("\n\n")
-      : prompt;
+    const content = Array.isArray(prompt)
+      ? prompt.flatMap((chunk: AgentPromptContentBlock) => {
+          if (chunk.type === "text") {
+            return [{ type: "text", text: chunk.text }];
+          }
+          if (chunk.type === "image") {
+            return [
+              {
+                type: "image",
+                source: {
+                  type: "base64",
+                  media_type: chunk.mimeType,
+                  data: chunk.data,
+                },
+              },
+            ];
+          }
+          return [];
+        })
+      : [{ type: "text", text: prompt }];
 
     return {
       type: "user",
       message: {
         role: "user",
-        content: [{ type: "text", text }],
+        content,
       },
       parent_tool_use_id: null,
       session_id: this.claudeSessionId ?? this.pendingLocalId,

packages/server/src/server/agent/providers/codex-mcp-agent.ts

@@ -2314,7 +2314,18 @@ function toPromptText(prompt: AgentPromptInput): string {
   if (typeof prompt === "string") {
     return prompt;
   }
-  return prompt.map((chunk) => chunk.text).join("");
+  const parts: string[] = [];
+  for (const chunk of prompt) {
+    if (chunk.type === "text") {
+      parts.push(chunk.text);
+      continue;
+    }
+    if (chunk.type === "image") {
+      const dataUrl = `data:${chunk.mimeType};base64,${chunk.data}`;
+      parts.push(`![user image](${dataUrl})`);
+    }
+  }
+  return parts.join("\n\n");
 }
 
 function getCodexMcpCommand(): string {

packages/server/src/server/session.ts

@@ -42,6 +42,7 @@ import type { ManagedAgent } from "./agent/agent-manager.js";
 import { toAgentPayload } from "./agent/agent-projections.js";
 import type {
   AgentPermissionResponse,
+  AgentPromptContentBlock,
   AgentPromptInput,
   AgentSessionConfig,
   AgentStreamEvent,
@@ -321,16 +322,14 @@ export class Session {
     if (!images || images.length === 0) {
       return normalized;
     }
-
-    const attachmentSummary = images
-      .map((image, index) => {
-        const sizeKb = Math.round((image.data.length * 0.75) / 1024);
-        return `Attachment ${index + 1}: ${image.mimeType}, ~${sizeKb}KB base64`;
-      })
-      .join("\n");
-
-    const base = normalized.length > 0 ? normalized : "User shared image attachment(s).";
-    return `${base}\n\n[Image attachments]\n${attachmentSummary}\n(Actual image bytes omitted; request a screenshot or file if needed.)`;
+    const blocks: AgentPromptContentBlock[] = [];
+    if (normalized.length > 0) {
+      blocks.push({ type: "text", text: normalized });
+    }
+    for (const image of images) {
+      blocks.push({ type: "image", data: image.data, mimeType: image.mimeType });
+    }
+    return blocks;
   }
 
   /**

plan.md

@@ -146,8 +146,9 @@ Improvements to the new agent screen in the app.
     - Test on web, iOS, and Android
     - **Done (2025-12-29 09:09)**: WHAT: `packages/app/src/contexts/session-context.tsx:1288-1334` adds web-specific image base64 conversion using fetch + FileReader (including data URI handling) and keeps native `FileSystem.readAsStringAsync` for non-web. RESULT: web image attachments are encoded without relying on `expo-file-system` web support. EVIDENCE: Not run (not requested).
 
-- [ ] **Fix**: Implement multimodal prompt building on server
+- [x] **Fix**: Implement multimodal prompt building on server
     - Modify `buildAgentPrompt` in `session.ts:316-334` to return structured content with images
     - Update `AgentPromptInput` type to support content blocks
     - Implement Claude-specific image content blocks
     - Implement Codex/OpenAI image handling
+    - **Done (2025-12-29 09:14)**: WHAT: `packages/server/src/server/agent/agent-sdk-types.ts:39-43` adds `AgentPromptContentBlock` and extends `AgentPromptInput` to include image blocks; `packages/server/src/server/session.ts:317-333` now builds prompt blocks with text + image data; `packages/server/src/server/agent/providers/claude-agent.ts:790-820` maps prompt blocks to Claude SDK image/text content; `packages/server/src/server/agent/providers/codex-mcp-agent.ts:2313-2328` renders image blocks as data URLs in prompt text. RESULT: server prompts now carry image data for Claude and include image payloads for Codex/OpenAI flows instead of a text-only summary. EVIDENCE: Not run (not requested).