|
| 1 | +import {afterEach, describe, expect, it, vi} from 'vitest'; |
| 2 | +import * as acp from '@agentclientprotocol/sdk'; |
| 3 | +import {type ChildProcessWithoutNullStreams, spawn} from 'node:child_process'; |
| 4 | +import {Readable, Writable} from 'node:stream'; |
| 5 | +import fs from "node:fs"; |
| 6 | +import os from "node:os"; |
| 7 | +import path from "node:path"; |
| 8 | +import {removeDirectoryWithRetry} from "../acp-test-utils"; |
| 9 | + |
| 10 | +const RUN_E2E_TESTS = process.env["RUN_E2E_TESTS"] === "true"; |
| 11 | +const TEST_TIMEOUT_MS = 90_000; |
| 12 | + |
| 13 | +class RecordingClient implements acp.Client { |
| 14 | + private readonly textBySessionId = new Map<string, string>(); |
| 15 | + |
| 16 | + async requestPermission(_params: acp.RequestPermissionRequest): Promise<acp.RequestPermissionResponse> { |
| 17 | + return { |
| 18 | + outcome: {outcome: "cancelled"}, |
| 19 | + }; |
| 20 | + } |
| 21 | + |
| 22 | + async sessionUpdate(params: acp.SessionNotification): Promise<void> { |
| 23 | + if (params.update.sessionUpdate !== "agent_message_chunk" || params.update.content.type !== "text") { |
| 24 | + return; |
| 25 | + } |
| 26 | + |
| 27 | + const nextText = `${this.textBySessionId.get(params.sessionId) ?? ""}${params.update.content.text}`; |
| 28 | + this.textBySessionId.set(params.sessionId, nextText); |
| 29 | + } |
| 30 | + |
| 31 | + readText(sessionId: string): string { |
| 32 | + return this.textBySessionId.get(sessionId) ?? ""; |
| 33 | + } |
| 34 | +} |
| 35 | + |
| 36 | +interface SpawnedAgentFixture { |
| 37 | + expectPromptText(promptText: string, assertText: (text: string) => void, timeoutMs?: number): Promise<void>; |
| 38 | + dispose(): Promise<void>; |
| 39 | +} |
| 40 | + |
| 41 | +interface TestSkill { |
| 42 | + readonly name: string; |
| 43 | + readonly description: string; |
| 44 | + readonly body: string; |
| 45 | +} |
| 46 | + |
| 47 | +interface RuntimePaths { |
| 48 | + readonly rootDir: string; |
| 49 | + readonly codexHome: string; |
| 50 | + readonly workspaceDir: string; |
| 51 | + readonly appServerLogsDir: string; |
| 52 | +} |
| 53 | + |
| 54 | +describe.skipIf(!RUN_E2E_TESTS)('E2E tests', {timeout: TEST_TIMEOUT_MS}, () => { |
| 55 | + let fixture: SpawnedAgentFixture | null = null; |
| 56 | + |
| 57 | + afterEach(async () => { |
| 58 | + if (fixture) { |
| 59 | + await fixture.dispose(); |
| 60 | + fixture = null; |
| 61 | + } |
| 62 | + }); |
| 63 | + |
| 64 | + it('returns model response', async () => { |
| 65 | + fixture = await createAuthenticatedFixture(); |
| 66 | + await fixture.expectPromptText("Reply with exactly integration-ok and nothing else.", (text) => { |
| 67 | + expect(text.toLowerCase()).toContain("integration-ok"); |
| 68 | + }); |
| 69 | + }); |
| 70 | + |
| 71 | + it('lists a user skill from the wrapped CODEX_HOME', async () => { |
| 72 | + fixture = await createFixtureWithSkill({ |
| 73 | + name: "integration-skill", |
| 74 | + description: "Integration skill", |
| 75 | + body: "This skill exists only for integration testing.", |
| 76 | + }); |
| 77 | + await fixture.expectPromptText("/skills", (text) => { |
| 78 | + expect(text).toContain("Available skills:"); |
| 79 | + expect(text).toContain("- integration-skill: Integration skill"); |
| 80 | + }); |
| 81 | + }); |
| 82 | +}); |
| 83 | + |
| 84 | +async function createFixtureWithSkill(skill: TestSkill): Promise<SpawnedAgentFixture> { |
| 85 | + const runtimePaths = createTemporaryRuntimePaths(); |
| 86 | + writeSkill(runtimePaths.codexHome, skill); |
| 87 | + return await createAuthenticatedFixture(runtimePaths); |
| 88 | +} |
| 89 | + |
| 90 | +async function createAuthenticatedFixture(runtimePaths = createTemporaryRuntimePaths()): Promise<SpawnedAgentFixture> { |
| 91 | + const apiKey = requireLiveApiKey(); |
| 92 | + const agentProcess = spawn("npm", ["run", "--silent", "start"], { |
| 93 | + cwd: process.cwd(), |
| 94 | + env: { |
| 95 | + ...process.env, |
| 96 | + CODEX_HOME: runtimePaths.codexHome, |
| 97 | + APP_SERVER_LOGS: runtimePaths.appServerLogsDir, |
| 98 | + }, |
| 99 | + stdio: ["pipe", "pipe", "pipe"], |
| 100 | + }); |
| 101 | + |
| 102 | + const client = new RecordingClient(); |
| 103 | + const output = Readable.toWeb(agentProcess.stdout) as ReadableStream<Uint8Array>; |
| 104 | + const connection = new acp.ClientSideConnection( |
| 105 | + () => client, |
| 106 | + acp.ndJsonStream(Writable.toWeb(agentProcess.stdin), output) |
| 107 | + ); |
| 108 | + |
| 109 | + const initializeResponse = await connection.initialize({ |
| 110 | + protocolVersion: acp.PROTOCOL_VERSION, |
| 111 | + clientCapabilities: {}, |
| 112 | + clientInfo: { |
| 113 | + name: "vitest", |
| 114 | + version: "1.0.0", |
| 115 | + }, |
| 116 | + }); |
| 117 | + expect(initializeResponse.protocolVersion).toBe(acp.PROTOCOL_VERSION); |
| 118 | + expect(initializeResponse.authMethods?.map((method) => method.id)).toContain("api-key"); |
| 119 | + |
| 120 | + await connection.authenticate({ |
| 121 | + methodId: "api-key", |
| 122 | + _meta: { |
| 123 | + "api-key": { |
| 124 | + apiKey, |
| 125 | + }, |
| 126 | + }, |
| 127 | + }); |
| 128 | + |
| 129 | + expect(await getAuthenticationStatus(connection)).toEqual({type: "api-key"}); |
| 130 | + |
| 131 | + return { |
| 132 | + async expectPromptText(promptText: string, assertText: (text: string) => void, timeoutMs = 30_000): Promise<void> { |
| 133 | + const newSessionResponse = await connection.newSession({ |
| 134 | + cwd: runtimePaths.workspaceDir, |
| 135 | + mcpServers: [], |
| 136 | + }); |
| 137 | + |
| 138 | + const promptResponse = await connection.prompt({ |
| 139 | + sessionId: newSessionResponse.sessionId, |
| 140 | + prompt: [{ |
| 141 | + type: "text", |
| 142 | + text: promptText, |
| 143 | + }], |
| 144 | + }); |
| 145 | + |
| 146 | + expect(promptResponse.stopReason).toBe("end_turn"); |
| 147 | + |
| 148 | + await vi.waitFor(() => { |
| 149 | + assertText(client.readText(newSessionResponse.sessionId)); |
| 150 | + }, {timeout: timeoutMs}); |
| 151 | + }, |
| 152 | + async dispose() { |
| 153 | + if (!agentProcess.stdin.destroyed && !agentProcess.stdin.writableEnded) { |
| 154 | + agentProcess.stdin.end(); |
| 155 | + } |
| 156 | + |
| 157 | + const exitedAfterStdinClose = await waitForProcessExit(agentProcess, 4_000); |
| 158 | + if (!exitedAfterStdinClose && !agentProcess.killed) { |
| 159 | + agentProcess.kill(); |
| 160 | + await waitForProcessExit(agentProcess, 4_000); |
| 161 | + } |
| 162 | + |
| 163 | + printLogDirectory(runtimePaths.appServerLogsDir); |
| 164 | + await removeDirectoryWithRetry(runtimePaths.rootDir); |
| 165 | + }, |
| 166 | + }; |
| 167 | +} |
| 168 | + |
| 169 | +function requireLiveApiKey(): string { |
| 170 | + const apiKey = process.env["CODEX_API_KEY"] ?? process.env["OPENAI_API_KEY"]; |
| 171 | + if (!apiKey) { |
| 172 | + throw new Error("Live integration test requires CODEX_API_KEY or OPENAI_API_KEY."); |
| 173 | + } |
| 174 | + return apiKey; |
| 175 | +} |
| 176 | + |
| 177 | +function createTemporaryRuntimePaths(): RuntimePaths { |
| 178 | + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "codex-acp-integration-")); |
| 179 | + const codexHome = path.join(rootDir, "codex-home"); |
| 180 | + const workspaceDir = path.join(rootDir, "workspace"); |
| 181 | + const appServerLogsDir = path.join(rootDir, "logs"); |
| 182 | + fs.mkdirSync(codexHome, {recursive: true}); |
| 183 | + fs.mkdirSync(workspaceDir, {recursive: true}); |
| 184 | + fs.mkdirSync(appServerLogsDir, {recursive: true}); |
| 185 | + fs.writeFileSync(path.join(codexHome, "config.toml"), 'cli_auth_credentials_store = "file"\n', "utf8"); |
| 186 | + return { |
| 187 | + rootDir, |
| 188 | + codexHome, |
| 189 | + workspaceDir, |
| 190 | + appServerLogsDir, |
| 191 | + }; |
| 192 | +} |
| 193 | + |
| 194 | +function writeSkill(codexHome: string, skill: TestSkill): void { |
| 195 | + const skillDirectory = path.join(codexHome, "skills", skill.name); |
| 196 | + fs.mkdirSync(skillDirectory, {recursive: true}); |
| 197 | + fs.writeFileSync( |
| 198 | + path.join(skillDirectory, "SKILL.md"), |
| 199 | + [ |
| 200 | + "---", |
| 201 | + `name: ${skill.name}`, |
| 202 | + `description: ${skill.description}`, |
| 203 | + "metadata:", |
| 204 | + ` short-description: ${skill.description}`, |
| 205 | + "---", |
| 206 | + "", |
| 207 | + skill.body, |
| 208 | + "", |
| 209 | + ].join("\n"), |
| 210 | + "utf8", |
| 211 | + ); |
| 212 | +} |
| 213 | + |
| 214 | +async function getAuthenticationStatus(connection: acp.ClientSideConnection): Promise<Record<string, unknown>> { |
| 215 | + return await connection.extMethod("authentication/status", {}); |
| 216 | +} |
| 217 | + |
| 218 | +function printLogDirectory(logDirectory: string): void { |
| 219 | + fs.readdirSync(logDirectory, {withFileTypes: true}) |
| 220 | + .filter((entry) => entry.isFile()) |
| 221 | + .forEach((entry) => { |
| 222 | + const logFilePath = path.join(logDirectory, entry.name); |
| 223 | + const content = fs.readFileSync(logFilePath, "utf8").trim(); |
| 224 | + console.log(`[APP_SERVER_LOGS] Logs from ${logFilePath}:`); |
| 225 | + console.log(content.length > 0 ? content : "[APP_SERVER_LOGS] Log file is empty"); |
| 226 | + console.log('------'); |
| 227 | + }); |
| 228 | +} |
| 229 | + |
| 230 | +async function waitForProcessExit(proc: ChildProcessWithoutNullStreams, timeoutMs: number): Promise<boolean> { |
| 231 | + if (proc.exitCode !== null || proc.signalCode !== null) { |
| 232 | + return true; |
| 233 | + } |
| 234 | + |
| 235 | + return await new Promise<boolean>((resolve) => { |
| 236 | + const timeout = setTimeout(() => { |
| 237 | + cleanup(); |
| 238 | + resolve(false); |
| 239 | + }, timeoutMs); |
| 240 | + |
| 241 | + const cleanup = () => { |
| 242 | + clearTimeout(timeout); |
| 243 | + proc.off("exit", handleExit); |
| 244 | + }; |
| 245 | + |
| 246 | + const handleExit = () => { |
| 247 | + cleanup(); |
| 248 | + resolve(true); |
| 249 | + }; |
| 250 | + |
| 251 | + proc.once("exit", handleExit); |
| 252 | + }); |
| 253 | +} |
0 commit comments