From d5f5351d37c0feb875f3e843ef93a4989feff95e Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Mon, 29 Dec 2025 09:07:54 +0700 Subject: [PATCH] Add image attachments to agent creation --- packages/app/src/app/agent/new.tsx | 7 +- packages/app/src/contexts/session-context.tsx | 67 +++++++++++-------- packages/app/src/stores/session-store.ts | 3 +- packages/server/src/server/messages.ts | 4 ++ packages/server/src/server/session.ts | 5 +- plan.md | 3 +- 6 files changed, 51 insertions(+), 38 deletions(-) diff --git a/packages/app/src/app/agent/new.tsx b/packages/app/src/app/agent/new.tsx index 0e8abf666..9f08d48d1 100644 --- a/packages/app/src/app/agent/new.tsx +++ b/packages/app/src/app/agent/new.tsx @@ -573,12 +573,6 @@ export default function DraftAgentScreen() { ...(modeId ? { modeId } : {}), ...(trimmedModel ? { model: trimmedModel } : {}), }; - // TODO: Images in initial agent creation are not yet supported by the server API. - // For now we log a warning. Images can be sent after agent creation via sendAgentMessage. - if (images && images.length > 0) { - console.warn("[DraftAgentScreen] Image attachments on agent creation not yet supported"); - } - const trimmedBaseBranch = baseBranch.trim(); const shouldIncludeBase = trimmedBaseBranch.length > 0 || @@ -608,6 +602,7 @@ export default function DraftAgentScreen() { createAgent({ config, initialPrompt: trimmedPrompt, + images, git: gitOptions, requestId, }); diff --git a/packages/app/src/contexts/session-context.tsx b/packages/app/src/contexts/session-context.tsx index 63ec10987..84f3c92d4 100644 --- a/packages/app/src/contexts/session-context.tsx +++ b/packages/app/src/contexts/session-context.tsx @@ -1284,6 +1284,34 @@ export function SessionProvider({ children, serverUrl, serverId }: SessionProvid ws.send(msg); }, [serverId, ws, setProviderModels]); + const encodeImages = useCallback(async ( + images?: Array<{ uri: string; mimeType?: string }> + ) => { + if (!images || images.length === 0) { + return undefined; + } + const encodedImages = await Promise.all( + images.map(async ({ uri, mimeType }) => { + try { + const data = await FileSystem.readAsStringAsync(uri, { + encoding: "base64", + }); + return { + data, + mimeType: mimeType ?? "image/jpeg", + }; + } catch (error) { + console.error("[Session] Failed to convert image:", error); + return null; + } + }) + ); + const validImages = encodedImages.filter( + (entry): entry is { data: string; mimeType: string } => entry !== null + ); + return validImages.length > 0 ? validImages : undefined; + }, []); + const sendAgentMessage = useCallback(async ( agentId: string, message: string, @@ -1304,31 +1332,7 @@ export function SessionProvider({ children, serverUrl, serverId }: SessionProvid return updated; }); - let imagesData: Array<{ data: string; mimeType: string }> | undefined; - if (images && images.length > 0) { - const encodedImages = await Promise.all( - images.map(async ({ uri, mimeType }) => { - try { - const data = await FileSystem.readAsStringAsync(uri, { - encoding: "base64", - }); - return { - data, - mimeType: mimeType ?? "image/jpeg", - }; - } catch (error) { - console.error("[Session] Failed to convert image:", error); - return null; - } - }) - ); - const validImages = encodedImages.filter( - (entry): entry is { data: string; mimeType: string } => entry !== null - ); - if (validImages.length > 0) { - imagesData = validImages; - } - } + const imagesData = await encodeImages(images); const msg: WSInboundMessage = { type: "session", @@ -1341,7 +1345,7 @@ export function SessionProvider({ children, serverUrl, serverId }: SessionProvid }, }; ws.send(msg); - }, [serverId, ws, setAgentStreamState]); + }, [encodeImages, serverId, ws, setAgentStreamState]); const cancelAgentRun = useCallback((agentId: string) => { const msg: WSInboundMessage = { @@ -1431,21 +1435,28 @@ export function SessionProvider({ children, serverUrl, serverId }: SessionProvid } }, [ws]); - const createAgent = useCallback(({ config, initialPrompt, git, worktreeName, requestId }: { config: any; initialPrompt: string; git?: any; worktreeName?: string; requestId?: string }) => { + const createAgent = useCallback(async ({ config, initialPrompt, images, git, worktreeName, requestId }: { config: any; initialPrompt: string; images?: Array<{ uri: string; mimeType?: string }>; git?: any; worktreeName?: string; requestId?: string }) => { const trimmedPrompt = initialPrompt.trim(); + let imagesData: Array<{ data: string; mimeType: string }> | undefined; + try { + imagesData = await encodeImages(images); + } catch (error) { + console.error("[Session] Failed to prepare images for agent creation:", error); + } const msg: WSInboundMessage = { type: "session", message: { type: "create_agent_request", config, ...(trimmedPrompt ? { initialPrompt: trimmedPrompt } : {}), + ...(imagesData && imagesData.length > 0 ? { images: imagesData } : {}), ...(git ? { git } : {}), ...(worktreeName ? { worktreeName } : {}), ...(requestId ? { requestId } : {}), }, }; ws.send(msg); - }, [ws]); + }, [encodeImages, ws]); const resumeAgent = useCallback(({ handle, overrides, requestId }: { handle: any; overrides?: any; requestId?: string }) => { const msg: WSInboundMessage = { diff --git a/packages/app/src/stores/session-store.ts b/packages/app/src/stores/session-store.ts index 462d113ab..964ac6a35 100644 --- a/packages/app/src/stores/session-store.ts +++ b/packages/app/src/stores/session-store.ts @@ -199,10 +199,11 @@ export interface SessionState { createAgent: (options: { config: any; initialPrompt: string; + images?: Array<{ uri: string; mimeType?: string }>; git?: any; worktreeName?: string; requestId?: string; - }) => void; + }) => Promise; resumeAgent: (options: { handle: any; overrides?: any; requestId?: string }) => void; setAgentMode: (agentId: string, modeId: string) => void; respondToPermission: (agentId: string, requestId: string, response: any) => void; diff --git a/packages/server/src/server/messages.ts b/packages/server/src/server/messages.ts index 19c7b113e..61c1a0be9 100644 --- a/packages/server/src/server/messages.ts +++ b/packages/server/src/server/messages.ts @@ -365,6 +365,10 @@ export const CreateAgentRequestMessageSchema = z.object({ config: AgentSessionConfigSchema, worktreeName: z.string().optional(), initialPrompt: z.string().optional(), + images: z.array(z.object({ + data: z.string(), // base64 encoded image + mimeType: z.string(), // e.g., "image/jpeg", "image/png" + })).optional(), git: GitSetupOptionsSchema.optional(), requestId: z.string().optional(), }); diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 5b3d68a67..6e35f0de0 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -1265,7 +1265,7 @@ export class Session { private async handleCreateAgentRequest( msg: Extract ): Promise { - const { config, worktreeName, requestId, initialPrompt, git } = msg; + const { config, worktreeName, requestId, initialPrompt, git, images } = msg; console.log( `[Session ${this.clientId}] Creating agent in ${config.cwd} (${config.provider})${ worktreeName ? ` with worktree ${worktreeName}` : "" @@ -1306,7 +1306,8 @@ export class Session { await this.handleSendAgentMessage( snapshot.id, trimmedPrompt, - uuidv4() + uuidv4(), + images ); } catch (promptError) { console.error( diff --git a/plan.md b/plan.md index dfd26ff10..b85430a1e 100644 --- a/plan.md +++ b/plan.md @@ -134,10 +134,11 @@ Improvements to the new agent screen in the app. - If broken, add fix tasks with debugging details. - **Done (2025-12-29 09:13)**: WHAT: Tested image attachments via Playwright MCP; traced code in `packages/app/src/app/agent/new.tsx:576-580`, `packages/app/src/contexts/session-context.tsx:1307-1331`, `packages/server/src/server/session.ts:316-334`. RESULT: Image attachments are BROKEN - three issues found: (1) Agent creation silently drops images with console warning, (2) Web platform fails to base64 encode images due to `expo-file-system.readAsStringAsync` not working, (3) Server `buildAgentPrompt` only creates text summary, never passes actual image bytes to LLM. EVIDENCE: Console logs `[WARNING] [DraftAgentScreen] Image attachments on agent creation not yet supported` and `[ERROR] [Session] Failed to convert image: Error: Method readAsStringAsync imported from "expo-file-system"...`; agent responses consistently state "I don't see any image attached". Full report in `REPORT-image-attachment-investigation.md`. -- [ ] **Fix**: Add image support to agent creation API +- [x] **Fix**: Add image support to agent creation API - Extend `create_agent_request` to accept images array - Pass images to `handleSendAgentMessage` after agent creation - Remove warning in `new.tsx:576-580` once implemented + - **Done (2025-12-29 09:07)**: WHAT: `packages/server/src/server/messages.ts:363-373` adds optional `images` to `create_agent_request`; `packages/server/src/server/session.ts:1268-1311` forwards images into `handleSendAgentMessage`; `packages/app/src/contexts/session-context.tsx:1287-1459` encodes image attachments and includes them in create-agent websocket payloads; `packages/app/src/stores/session-store.ts:175-206` updates createAgent type to accept images/async; `packages/app/src/app/agent/new.tsx:560-606` forwards images to createAgent and removes the warning. RESULT: agent creation requests now carry image attachments through to the initial prompt path. EVIDENCE: Not run (not requested). - [ ] **Fix**: Fix web platform image base64 encoding - Replace `FileSystem.readAsStringAsync` in `session-context.tsx:1312` with cross-platform solution