Handle embedded resource blobs in prompts (#202)

Author: benbrandt

src/CodexAcpClient.ts

@@ -686,14 +686,24 @@ function buildPromptItems(prompt: acp.ContentBlock[]): UserInput[] {
                     const context = `<context ref="${resource.uri}">\n${resource.text}\n</context>`;
                     return {type: "text", text: `${link}\n${context}`, text_elements: []};
                 }
-                return null;
+                if (isImageMimeType(resource.mimeType)) {
+                    return {type: "image", url: `data:${resource.mimeType};base64,${resource.blob}`};
+                }
+                const link = formatUriAsLink(null, resource.uri);
+                const mimeType = resource.mimeType ?? "application/octet-stream";
+                const context = `<context ref="${resource.uri}" mimeType="${mimeType}" encoding="base64">\n${resource.blob}\n</context>`;
+                return {type: "text", text: `${link}\n${context}`, text_elements: []};
             }
             case "audio":
                 return null;
         }
     }).filter((block): block is UserInput => block !== null);
 }
 
+function isImageMimeType(mimeType: string | null | undefined): mimeType is string {
+    return mimeType?.startsWith("image/") ?? false;
+}
+
 function formatUriAsLink(name: string | null | undefined, uri: string): string {
     if (name && name.length > 0) {
         return `[@${name}](${uri})`;

src/__tests__/CodexACPAgent/CodexAcpClient.test.ts

@@ -730,6 +730,8 @@ describe('ACP server test', { timeout: 40_000 }, () => {
             { type: "image", mimeType: "image/png", data: "abc123", uri: "https://example.com/image.png" },
             { type: "resource_link", name: "report.txt", uri: "file:///tmp/report.txt" },
             { type: "resource", resource: { uri: "file:///tmp/notes.txt", text: "Notes body" } as acp.EmbeddedResourceResource },
+            { type: "resource", resource: { uri: "file:///tmp/pixel.png", mimeType: "image/png", blob: "iVBORw0KGgo=" } as acp.EmbeddedResourceResource },
+            { type: "resource", resource: { uri: "file:///tmp/archive.bin", mimeType: "application/octet-stream", blob: "AAEC" } as acp.EmbeddedResourceResource },
         ];
 
         await codexAcpAgent.prompt({ sessionId: "session-id", prompt });

src/__tests__/CodexACPAgent/data/send-attachments-turn-start.json

@@ -35,6 +35,15 @@
         "type": "text",
         "text": "[@notes.txt](file:///tmp/notes.txt)\n<context ref=\"file:///tmp/notes.txt\">\nNotes body\n</context>",
         "text_elements": []
+      },
+      {
+        "type": "image",
+        "url": "data:image/png;base64,iVBORw0KGgo="
+      },
+      {
+        "type": "text",
+        "text": "[@archive.bin](file:///tmp/archive.bin)\n<context ref=\"file:///tmp/archive.bin\" mimeType=\"application/octet-stream\" encoding=\"base64\">\nAAEC\n</context>",
+        "text_elements": []
       }
     ],
     "approvalPolicy": "on-request",