diff --git a/src/CodexAcpServer.ts b/src/CodexAcpServer.ts index 447721c9..906ae12e 100644 --- a/src/CodexAcpServer.ts +++ b/src/CodexAcpServer.ts @@ -1,5 +1,8 @@ import * as acp from "@agentclientprotocol/sdk"; import {RequestError, type SessionId, type SessionModeState} from "@agentclientprotocol/sdk"; +import {readFile} from "node:fs/promises"; +import {extname} from "node:path"; +import {fileURLToPath, pathToFileURL} from "node:url"; import {CodexEventHandler} from "./CodexEventHandler"; import {CodexApprovalHandler} from "./CodexApprovalHandler"; import {CodexElicitationHandler} from "./CodexElicitationHandler"; @@ -80,6 +83,7 @@ import { createAgentTextMessageChunk, createAgentTextThoughtChunk, createUserMessageChunk, + visibleUserMessageText, } from "./ContentChunks"; import { sameThreadGoalSnapshot, @@ -1232,13 +1236,11 @@ export class CodexAcpServer { } } - private createUserMessageUpdates(item: ThreadItem & { type: "userMessage" }): UpdateSessionEvent[] { + private async createUserMessageUpdates(item: ThreadItem & { type: "userMessage" }): Promise { const updates: UpdateSessionEvent[] = []; - const messageId = item.id; for (const input of item.content) { - const blocks = this.userInputToContentBlocks(input); - for (const block of blocks) { - updates.push(createUserMessageChunk(block, messageId)); + for (const block of await this.userInputToContentBlocks(input)) { + updates.push(createUserMessageChunk(block, item.id)); } } return updates; @@ -1291,20 +1293,39 @@ export class CodexAcpServer { }; } - private userInputToContentBlocks(input: UserInput): acp.ContentBlock[] { + private async userInputToContentBlocks(input: UserInput): Promise { switch (input.type) { - case "text": - return input.text.length > 0 ? [{ type: "text", text: input.text }] : []; - case "image": + case "text": { + const visibleText = visibleUserMessageText(input.text); + return visibleText.length > 0 ? [{ type: "text", text: visibleText }] : []; + } + case "image": { + const match = /^data:(image\/[^;]+);base64,(.+)$/.exec(input.url); + const mimeType = match?.[1]; + const data = match?.[2]; + if (mimeType && data) { + return [{ type: "image", mimeType, data }]; + } return [{ type: "text", text: this.formatUriAsLink("image", input.url) }]; + } case "localImage": { - const uri = input.path.startsWith("file://") ? input.path : `file://${input.path}`; + const path = input.path.startsWith("file://") ? fileURLToPath(input.path) : input.path; + const uri = pathToFileURL(path).href; + const extension = extname(path).slice(1).toLowerCase(); + const data = extension ? await readFile(path).catch(() => null) : null; + if (data) { + const mimeType = `image/${extension === "jpg" ? "jpeg" : extension}`; + return [{ type: "image", data: data.toString("base64"), mimeType, uri }]; + } return [{ type: "text", text: this.formatUriAsLink(null, uri) }]; } case "skill": return [{ type: "text", text: `skill:${input.name} (${input.path})` }]; + case "mention": { + const uri = input.path.startsWith("file://") ? input.path : pathToFileURL(input.path).href; + return [{ type: "resource_link", name: input.name, uri }]; + } } - return []; } private formatUriAsLink(name: string | null, uri: string): string { diff --git a/src/ContentChunks.ts b/src/ContentChunks.ts index 2bef82e8..4a6a915a 100644 --- a/src/ContentChunks.ts +++ b/src/ContentChunks.ts @@ -3,6 +3,20 @@ import type {UpdateSessionEvent} from "./ACPSessionConnection"; type AcpMeta = Record; +const FILES_MENTIONED_HEADER = "# Files mentioned by the user:\n"; +const REQUEST_MARKER = "\n## My request for Codex:\n"; + +export function visibleUserMessageText(text: string): string { + const normalized = text.trimStart(); + const requestIndex = normalized.indexOf(REQUEST_MARKER); + + if (normalized.startsWith(FILES_MENTIONED_HEADER) && requestIndex !== -1) { + return normalized.slice(requestIndex + REQUEST_MARKER.length); + } + + return text; +} + export function createCodexMessagePhaseMeta(phase: string | null | undefined): AcpMeta | undefined { if (!phase) { return undefined; diff --git a/src/ResponseItemHistoryFallback.ts b/src/ResponseItemHistoryFallback.ts index 7292df20..56619627 100644 --- a/src/ResponseItemHistoryFallback.ts +++ b/src/ResponseItemHistoryFallback.ts @@ -6,7 +6,7 @@ import { stripShellPrefix } from "./CommandUtils"; import type { CommandAction, Thread, ThreadItem } from "./app-server/v2"; import { createCommandActionEvent } from "./CodexToolCallMapper"; import { createTerminalOutputMeta, type TerminalOutputMode } from "./TerminalOutputMode"; -import { createAgentMessageChunk, createCodexMessagePhaseMeta } from "./ContentChunks"; +import { createAgentMessageChunk, createCodexMessagePhaseMeta, createUserMessageChunk, visibleUserMessageText } from "./ContentChunks"; type JsonRecord = Record; type AcpToolCallEvent = Extract; @@ -262,18 +262,12 @@ function createEventMsgUpdates(record: JsonRecord): UpdateSessionEvent[] | null } function createUserMessageEventUpdates(payload: JsonRecord): UpdateSessionEvent[] { - const blocks: ContentBlock[] = []; - const message = stringValue(payload["message"]); - if (message !== null && message.length > 0) { - blocks.push({ type: "text", text: message }); + const text = visibleUserMessageText(stringValue(payload["message"]) ?? ""); + if (text.length > 0) { + return [createUserMessageChunk({ type: "text", text })]; } - blocks.push(...imageBlocks(payload["images"])); - blocks.push(...imageBlocks(payload["local_images"])); - return blocks.map((content) => ({ - sessionUpdate: "user_message_chunk", - content, - })); + return []; } function createAgentReasoningEventUpdates(payload: JsonRecord): UpdateSessionEvent[] { @@ -288,22 +282,6 @@ function createAgentReasoningEventUpdates(payload: JsonRecord): UpdateSessionEve }]; } -function imageBlocks(images: unknown): ContentBlock[] { - if (!Array.isArray(images)) { - return []; - } - - return images.flatMap((image): ContentBlock[] => { - if (typeof image === "string") { - return [{ type: "text", text: `[@image](${image})` }]; - } - - const record = asRecord(image); - const path = record ? stringValue(record["path"]) ?? stringValue(record["url"]) : null; - return path ? [{ type: "text", text: `[@image](${path})` }] : []; - }); -} - function contentBlocksFromResponseContent(content: unknown): ContentBlock[] { if (!Array.isArray(content)) { return []; diff --git a/src/__tests__/CodexACPAgent/data/load-session-history.json b/src/__tests__/CodexACPAgent/data/load-session-history.json index 0018e170..6f6068e1 100644 --- a/src/__tests__/CodexACPAgent/data/load-session-history.json +++ b/src/__tests__/CodexACPAgent/data/load-session-history.json @@ -151,6 +151,58 @@ } ] } +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "session-1", + "update": { + "sessionUpdate": "user_message_chunk", + "messageId": "item-user-1", + "content": { + "type": "image", + "mimeType": "image/png", + "data": "dGVzdCBpbWFnZQ==" + } + } + } + ] +} +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "session-1", + "update": { + "sessionUpdate": "user_message_chunk", + "messageId": "item-user-1", + "content": { + "type": "image", + "data": "dGVzdCBpbWFnZQ==", + "mimeType": "image/png", + "uri": "file:///tmp/codex-acp-load-session-image.png" + } + } + } + ] +} +{ + "method": "sessionUpdate", + "args": [ + { + "sessionId": "session-1", + "update": { + "sessionUpdate": "user_message_chunk", + "messageId": "item-user-1", + "content": { + "type": "resource_link", + "name": "notes.txt", + "uri": "file:///test/project/notes.txt" + } + } + } + ] +} { "method": "sessionUpdate", "args": [ @@ -429,4 +481,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/__tests__/CodexACPAgent/load-session.test.ts b/src/__tests__/CodexACPAgent/load-session.test.ts index a64089e1..0513b2d9 100644 --- a/src/__tests__/CodexACPAgent/load-session.test.ts +++ b/src/__tests__/CodexACPAgent/load-session.test.ts @@ -3,11 +3,16 @@ import type * as acp from "@agentclientprotocol/sdk"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { pathToFileURL } from "node:url"; import { createCodexMockTestFixture, createTestModel } from "../acp-test-utils"; import type { Model, Thread, ThreadGoal } from "../../app-server/v2"; describe("CodexACPAgent - loadSession", () => { it("should replay history during loadSession", async () => { + const localImageDirectory = await mkdtemp(join(tmpdir(), "codex-acp-load-session-")); + const localImagePath = join(localImageDirectory, "image.png"); + await writeFile(localImagePath, "test image"); + const fixture = createCodexMockTestFixture(); const codexAcpAgent = fixture.getCodexAcpAgent(); const codexAcpClient = fixture.getCodexAcpClient(); @@ -82,8 +87,15 @@ describe("CodexACPAgent - loadSession", () => { id: "item-user-1", clientId: null, content: [ - { type: "text", text: "Hi", text_elements: [] }, + { + type: "text", + text: `\n# Files mentioned by the user:\n\n## image.png: ${localImagePath}\n\n## My request for Codex:\nHi`, + text_elements: [], + }, { type: "image", url: "https://example.com/image.png" }, + { type: "image", url: "data:image/png;base64,dGVzdCBpbWFnZQ==" }, + { type: "localImage", path: localImagePath }, + { type: "mention", name: "notes.txt", path: "/test/project/notes.txt" }, ], }, { @@ -223,9 +235,14 @@ describe("CodexACPAgent - loadSession", () => { includeTurns: true, }); expect(codexAppServerClient.threadGoalGet).toHaveBeenCalledWith({ threadId: thread.id }); - await expect(fixture.getAcpConnectionDump([])).toMatchFileSnapshot( + const replay = fixture.getAcpConnectionDump([]).replaceAll( + pathToFileURL(localImagePath).href, + "file:///tmp/codex-acp-load-session-image.png", + ); + await expect(`${replay}\n`).toMatchFileSnapshot( "data/load-session-history.json" ); + await rm(localImageDirectory, { recursive: true }); }); it("should not recover session mcp servers during loadSession when request omits them", async () => { diff --git a/src/__tests__/CodexACPAgent/response-item-history-fallback.test.ts b/src/__tests__/CodexACPAgent/response-item-history-fallback.test.ts index aaeef097..a5286353 100644 --- a/src/__tests__/CodexACPAgent/response-item-history-fallback.test.ts +++ b/src/__tests__/CodexACPAgent/response-item-history-fallback.test.ts @@ -55,6 +55,24 @@ describe("ResponseItemHistoryFallback", () => { expect(thoughtTexts(updates)).toEqual(["Need to inspect the directory."]); }); + it("leaves user attachments to thread history", () => { + const updates = parseResponseItemHistoryFallback(jsonl([ + { + type: "event_msg", + payload: { + type: "user_message", + message: "\n# Files mentioned by the user:\n\n## screenshot.png: /tmp/screenshot.png\n\n## My request for Codex:\nInspect the screenshot", + images: [], + local_images: ["/tmp/screenshot.png"], + }, + }, + functionCall("call-missing", "ls"), + functionCallOutput("call-missing", "Chunk ID: missing\nProcess exited with code 0\nOutput:\nREADME.md\n"), + ]), "terminal_output"); + + expect(userMessageTexts(updates)).toEqual(["Inspect the screenshot"]); + }); + it("preserves assistant message phase metadata from response items", () => { const updates = parseResponseItemHistoryFallback(jsonl([ { @@ -154,6 +172,14 @@ function thoughtTexts(updates: UpdateSessionEvent[] | null): string[] { .flatMap((update) => update.content.type === "text" ? [update.content.text] : []); } +function userMessageTexts(updates: UpdateSessionEvent[] | null): string[] { + return (updates ?? []) + .filter((update): update is Extract => ( + update.sessionUpdate === "user_message_chunk" + )) + .flatMap((update) => update.content.type === "text" ? [update.content.text] : []); +} + function agentMessageMetas(updates: UpdateSessionEvent[] | null): unknown[] { return (updates ?? []) .filter((update): update is Extract => (