diff --git a/src/CodexAcpServer.ts b/src/CodexAcpServer.ts index 3f8440f4..99256eb4 100644 --- a/src/CodexAcpServer.ts +++ b/src/CodexAcpServer.ts @@ -38,6 +38,8 @@ import { createCommandExecutionUpdate, createDynamicToolCallUpdate, createFileChangeUpdate, + createImageGenerationUpdate, + createImageViewUpdate, createMcpToolCallUpdate, formatWebSearchTitle, } from "./CodexToolCallMapper"; @@ -837,9 +839,9 @@ export class CodexAcpServer implements acp.Agent { case "webSearch": return [this.createWebSearchUpdate(item)]; case "imageView": - return [this.createImageViewUpdate(item)]; + return [createImageViewUpdate(item)]; case "imageGeneration": - return []; + return [createImageGenerationUpdate(item)]; case "enteredReviewMode": return [this.createReviewModeUpdate(item, true)]; case "exitedReviewMode": @@ -908,22 +910,6 @@ export class CodexAcpServer implements acp.Agent { }; } - private createImageViewUpdate( - item: ThreadItem & { type: "imageView" } - ): UpdateSessionEvent { - return { - sessionUpdate: "tool_call", - toolCallId: item.id, - kind: "read", - title: "View image", - status: "completed", - locations: [{ path: item.path }], - rawInput: { - path: item.path, - }, - }; - } - private createReviewModeUpdate( item: ThreadItem & { type: "enteredReviewMode" | "exitedReviewMode" }, entered: boolean diff --git a/src/CodexEventHandler.ts b/src/CodexEventHandler.ts index 32fb19c8..ab1615fd 100644 --- a/src/CodexEventHandler.ts +++ b/src/CodexEventHandler.ts @@ -34,6 +34,10 @@ import { createFileChangeUpdate, createGuardianApprovalReviewToolCall, createGuardianApprovalReviewToolCallUpdate, + createImageGenerationCompleteUpdate, + createImageGenerationStartUpdate, + createImageGenerationUpdate, + createImageViewUpdate, createMcpRawInput, createMcpRawOutput, createFuzzyFileSearchComplete, @@ -54,6 +58,8 @@ export class CodexEventHandler { private failure: RequestError | null = null; private readonly activeFuzzyFileSearchSessions = new Set(); private readonly activeGuardianApprovalReviews = new Set(); + private readonly activeImageGenerationItems = new Set(); + private readonly emittedImageViewItems = new Set(); constructor(connection: acp.AgentSideConnection, sessionState: SessionState) { this.connection = connection; @@ -296,13 +302,17 @@ export class CodexEventHandler { return await createDynamicToolCallUpdate(event.item); case "webSearch": return createWebSearchStartUpdate(event.item); + case "imageView": + this.emittedImageViewItems.add(event.item.id); + return createImageViewUpdate(event.item); + case "imageGeneration": + this.activeImageGenerationItems.add(event.item.id); + return createImageGenerationStartUpdate(event.item); case "collabAgentToolCall": case "userMessage": case "hookPrompt": case "agentMessage": case "reasoning": - case "imageView": - case "imageGeneration": case "enteredReviewMode": case "exitedReviewMode": case "contextCompaction": @@ -330,6 +340,16 @@ export class CodexEventHandler { } case "commandExecution": return this.completeCommandExecutionEvent(event.item); + case "imageView": + if (this.emittedImageViewItems.delete(event.item.id)) { + return null; + } + return createImageViewUpdate(event.item); + case "imageGeneration": + if (this.activeImageGenerationItems.delete(event.item.id)) { + return createImageGenerationCompleteUpdate(event.item); + } + return createImageGenerationUpdate(event.item); case "reasoning": const summary = event.item.summary[0]; if (!summary) return null; @@ -346,8 +366,6 @@ export class CodexEventHandler { case "userMessage": case "hookPrompt": case "agentMessage": - case "imageView": - case "imageGeneration": case "enteredReviewMode": case "plan": return null; diff --git a/src/CodexToolCallMapper.ts b/src/CodexToolCallMapper.ts index 6f524690..d467c61c 100644 --- a/src/CodexToolCallMapper.ts +++ b/src/CodexToolCallMapper.ts @@ -1,4 +1,4 @@ -import type { ToolCallContent } from "@agentclientprotocol/sdk"; +import type { ContentBlock, ToolCallContent } from "@agentclientprotocol/sdk"; import { applyPatch, parsePatch, reversePatch } from "diff"; import { readFile } from "node:fs/promises"; import path from "node:path"; @@ -114,6 +114,69 @@ export async function createDynamicToolCallUpdate( return createExecuteToolCallUpdate(item, item.tool, { arguments: item.arguments }) } +export function createImageViewUpdate( + item: ThreadItem & { type: "imageView" } +): UpdateSessionEvent { + const displayPath = item.path; + return { + sessionUpdate: "tool_call", + toolCallId: item.id, + kind: "read", + title: `View Image ${displayPath}`, + status: "completed", + content: [createContent({ + type: "resource_link", + name: displayPath, + uri: displayPath, + })], + locations: [{ path: item.path }], + rawInput: { + path: item.path, + }, + }; +} + +export function createImageGenerationStartUpdate( + item: ThreadItem & { type: "imageGeneration" } +): UpdateSessionEvent { + return { + sessionUpdate: "tool_call", + toolCallId: item.id, + kind: "other", + title: "Image generation", + status: "in_progress", + rawInput: { + id: item.id, + }, + }; +} + +export function createImageGenerationCompleteUpdate( + item: ThreadItem & { type: "imageGeneration" } +): UpdateSessionEvent { + return { + sessionUpdate: "tool_call_update", + toolCallId: item.id, + status: imageGenerationToolStatus(item.status), + content: imageGenerationContent(item), + rawOutput: imageGenerationRawOutput(item), + }; +} + +export function createImageGenerationUpdate( + item: ThreadItem & { type: "imageGeneration" } +): UpdateSessionEvent { + return { + sessionUpdate: "tool_call", + toolCallId: item.id, + kind: "other", + title: "Image generation", + status: imageGenerationToolStatus(item.status), + content: imageGenerationContent(item), + rawOutput: imageGenerationRawOutput(item), + }; +} + export async function createExecuteToolCallUpdate( item: ThreadItem & ({ type: "mcpToolCall" } | { type: "dynamicToolCall" }), title: string, @@ -450,6 +513,74 @@ function shellQuote(arg: string): string { return `'${arg.replace(/'/g, `'\\''`)}'`; } +function imageGenerationToolStatus(status: string): AcpToolCallStatus { + switch (status) { + case "completed": + return "completed"; + case "generating": + case "in_progress": + case "inProgress": + case "incomplete": + return "in_progress"; + case "failed": + return "failed"; + default: + return "completed"; + } +} + +function imageGenerationContent( + item: ThreadItem & { type: "imageGeneration" } +): ToolCallContent[] { + const content: ToolCallContent[] = []; + + if (item.revisedPrompt && item.revisedPrompt.trim() !== "") { + content.push(createContent({ + type: "text", + text: `Revised prompt: ${item.revisedPrompt}`, + })); + } + + if (item.result.trim() !== "") { + const image: ContentBlock = item.savedPath && item.savedPath.trim() !== "" + ? { + type: "image", + data: item.result, + mimeType: "image/png", + uri: item.savedPath, + } + : { + type: "image", + data: item.result, + mimeType: "image/png", + }; + content.push(createContent(image)); + } + + return content; +} + +function imageGenerationRawOutput( + item: ThreadItem & { type: "imageGeneration" } +): Record { + const output: Record = { + status: item.status, + revisedPrompt: item.revisedPrompt, + result: item.result, + }; + if ("savedPath" in item) { + output["savedPath"] = item.savedPath ?? null; + } + return output; +} + +function createContent(content: ContentBlock): ToolCallContent { + return { + type: "content", + content, + }; +} + async function createPatchContent(change: FileUpdateChange): Promise { try { switch (change.kind.type) { diff --git a/src/__tests__/CodexACPAgent/data/image-generation-completed-only.json b/src/__tests__/CodexACPAgent/data/image-generation-completed-only.json new file mode 100644 index 00000000..d20df233 --- /dev/null +++ b/src/__tests__/CodexACPAgent/data/image-generation-completed-only.json @@ -0,0 +1,30 @@ +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "test-session-id", + "update": { + "sessionUpdate": "tool_call", + "toolCallId": "image-generation-completed-only", + "kind": "other", + "title": "Image generation", + "status": "completed", + "content": [ + { + "type": "content", + "content": { + "type": "image", + "data": "iVBORw0KGgo=", + "mimeType": "image/png" + } + } + ], + "rawOutput": { + "status": "completed", + "revisedPrompt": null, + "result": "iVBORw0KGgo=" + } + } + } + ] +} \ No newline at end of file diff --git a/src/__tests__/CodexACPAgent/data/image-generation-flow.json b/src/__tests__/CodexACPAgent/data/image-generation-flow.json new file mode 100644 index 00000000..204fe7f2 --- /dev/null +++ b/src/__tests__/CodexACPAgent/data/image-generation-flow.json @@ -0,0 +1,55 @@ +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "test-session-id", + "update": { + "sessionUpdate": "tool_call", + "toolCallId": "image-generation-1", + "kind": "other", + "title": "Image generation", + "status": "in_progress", + "rawInput": { + "id": "image-generation-1" + } + } + } + ] +} +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "test-session-id", + "update": { + "sessionUpdate": "tool_call_update", + "toolCallId": "image-generation-1", + "status": "completed", + "content": [ + { + "type": "content", + "content": { + "type": "text", + "text": "Revised prompt: A tiny blue square" + } + }, + { + "type": "content", + "content": { + "type": "image", + "data": "iVBORw0KGgo=", + "mimeType": "image/png", + "uri": "/tmp/codex/generated-blue-square.png" + } + } + ], + "rawOutput": { + "status": "completed", + "revisedPrompt": "A tiny blue square", + "result": "iVBORw0KGgo=", + "savedPath": "/tmp/codex/generated-blue-square.png" + } + } + } + ] +} \ No newline at end of file diff --git a/src/__tests__/CodexACPAgent/data/load-session-history.json b/src/__tests__/CodexACPAgent/data/load-session-history.json index 44753465..0a41c1cb 100644 --- a/src/__tests__/CodexACPAgent/data/load-session-history.json +++ b/src/__tests__/CodexACPAgent/data/load-session-history.json @@ -216,4 +216,76 @@ } } ] +} +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "session-1", + "update": { + "sessionUpdate": "tool_call", + "toolCallId": "item-image-view-1", + "kind": "read", + "title": "View Image /test/project/input.png", + "status": "completed", + "content": [ + { + "type": "content", + "content": { + "type": "resource_link", + "name": "/test/project/input.png", + "uri": "/test/project/input.png" + } + } + ], + "locations": [ + { + "path": "/test/project/input.png" + } + ], + "rawInput": { + "path": "/test/project/input.png" + } + } + } + ] +} +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "session-1", + "update": { + "sessionUpdate": "tool_call", + "toolCallId": "item-image-generation-1", + "kind": "other", + "title": "Image generation", + "status": "completed", + "content": [ + { + "type": "content", + "content": { + "type": "text", + "text": "Revised prompt: A tiny blue square" + } + }, + { + "type": "content", + "content": { + "type": "image", + "data": "iVBORw0KGgo=", + "mimeType": "image/png", + "uri": "/test/project/generated-blue-square.png" + } + } + ], + "rawOutput": { + "status": "completed", + "revisedPrompt": "A tiny blue square", + "result": "iVBORw0KGgo=", + "savedPath": "/test/project/generated-blue-square.png" + } + } + } + ] } \ No newline at end of file diff --git a/src/__tests__/CodexACPAgent/data/view-image-flow.json b/src/__tests__/CodexACPAgent/data/view-image-flow.json new file mode 100644 index 00000000..6930a7f8 --- /dev/null +++ b/src/__tests__/CodexACPAgent/data/view-image-flow.json @@ -0,0 +1,33 @@ +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "test-session-id", + "update": { + "sessionUpdate": "tool_call", + "toolCallId": "view-image-1", + "kind": "read", + "title": "View Image /tmp/codex/input.png", + "status": "completed", + "content": [ + { + "type": "content", + "content": { + "type": "resource_link", + "name": "/tmp/codex/input.png", + "uri": "/tmp/codex/input.png" + } + } + ], + "locations": [ + { + "path": "/tmp/codex/input.png" + } + ], + "rawInput": { + "path": "/tmp/codex/input.png" + } + } + } + ] +} \ No newline at end of file diff --git a/src/__tests__/CodexACPAgent/image-events.test.ts b/src/__tests__/CodexACPAgent/image-events.test.ts new file mode 100644 index 00000000..bb7718aa --- /dev/null +++ b/src/__tests__/CodexACPAgent/image-events.test.ts @@ -0,0 +1,121 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import type { SessionState } from "../../CodexAcpServer"; +import type { ServerNotification } from "../../app-server"; +import { createCodexMockTestFixture, createTestSessionState, setupPromptAndSendNotifications, type CodexMockTestFixture } from "../acp-test-utils"; +import { AgentMode } from "../../AgentMode"; + +describe("CodexEventHandler - image events", () => { + let mockFixture: CodexMockTestFixture; + const sessionId = "test-session-id"; + + beforeEach(() => { + mockFixture = createCodexMockTestFixture(); + vi.clearAllMocks(); + }); + + const sessionState: SessionState = createTestSessionState({ + sessionId, + currentModelId: "model-id[effort]", + agentMode: AgentMode.DEFAULT_AGENT_MODE, + }); + + it("maps image generation start and completion as an image tool call flow", async () => { + const notifications: ServerNotification[] = [ + { + method: "item/started", + params: { + threadId: sessionId, + turnId: "turn-1", + startedAtMs: 0, + item: { + type: "imageGeneration", + id: "image-generation-1", + status: "generating", + revisedPrompt: null, + result: "", + }, + }, + }, + { + method: "item/completed", + params: { + threadId: sessionId, + turnId: "turn-1", + completedAtMs: 0, + item: { + type: "imageGeneration", + id: "image-generation-1", + status: "completed", + revisedPrompt: "A tiny blue square", + result: "iVBORw0KGgo=", + savedPath: "/tmp/codex/generated-blue-square.png", + }, + }, + }, + ]; + + await setupPromptAndSendNotifications(mockFixture, sessionId, sessionState, notifications); + + await expect(mockFixture.getAcpConnectionDump([])).toMatchFileSnapshot( + "data/image-generation-flow.json" + ); + }); + + it("maps completed-only image generation as a full completed tool call", async () => { + const completed: ServerNotification = { + method: "item/completed", + params: { + threadId: sessionId, + turnId: "turn-1", + completedAtMs: 0, + item: { + type: "imageGeneration", + id: "image-generation-completed-only", + status: "completed", + revisedPrompt: null, + result: "iVBORw0KGgo=", + }, + }, + }; + + await setupPromptAndSendNotifications(mockFixture, sessionId, sessionState, [completed]); + + await expect(mockFixture.getAcpConnectionDump([])).toMatchFileSnapshot( + "data/image-generation-completed-only.json" + ); + }); + + it("maps view-image start and completion as one completed read tool call", async () => { + const item = { + type: "imageView" as const, + id: "view-image-1", + path: "/tmp/codex/input.png", + }; + const notifications: ServerNotification[] = [ + { + method: "item/started", + params: { + threadId: sessionId, + turnId: "turn-1", + startedAtMs: 0, + item, + }, + }, + { + method: "item/completed", + params: { + threadId: sessionId, + turnId: "turn-1", + completedAtMs: 0, + item, + }, + }, + ]; + + await setupPromptAndSendNotifications(mockFixture, sessionId, sessionState, notifications); + + await expect(mockFixture.getAcpConnectionDump([])).toMatchFileSnapshot( + "data/view-image-flow.json" + ); + }); +}); diff --git a/src/__tests__/CodexACPAgent/load-session.test.ts b/src/__tests__/CodexACPAgent/load-session.test.ts index 9cee5f09..112dbf3e 100644 --- a/src/__tests__/CodexACPAgent/load-session.test.ts +++ b/src/__tests__/CodexACPAgent/load-session.test.ts @@ -143,6 +143,19 @@ describe("CodexACPAgent - loadSession", () => { success: true, durationMs: 3, }, + { + type: "imageView", + id: "item-image-view-1", + path: "/test/project/input.png", + }, + { + type: "imageGeneration", + id: "item-image-generation-1", + status: "completed", + revisedPrompt: "A tiny blue square", + result: "iVBORw0KGgo=", + savedPath: "/test/project/generated-blue-square.png", + }, ], }, ],