From bc6efd98a5504dfe224d783a6ed3f6ba51a04d2b Mon Sep 17 00:00:00 2001 From: "Ilia.Shulgin" Date: Fri, 17 Apr 2026 16:09:37 +0200 Subject: [PATCH] LLM-25904 Add e2e test checking actual codex interaction --- .github/workflows/ci.yml | 5 + package.json | 1 + .../CodexACPAgent/e2e/acp-e2e-test-utils.ts | 242 ++++++++++++++++++ .../CodexACPAgent/e2e/acp-e2e.test.ts | 37 +++ src/__tests__/acp-test-utils.ts | 2 +- 5 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 src/__tests__/CodexACPAgent/e2e/acp-e2e-test-utils.ts create mode 100644 src/__tests__/CodexACPAgent/e2e/acp-e2e.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8e38080..09277bed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,11 @@ jobs: node-version: '24' - run: npm ci - run: npm test + - name: Run e2e tests + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: + npm run test:e2e build: runs-on: ubuntu-latest diff --git a/package.json b/package.json index 5972985e..7bd55144 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "start": "node --import tsx src/index.ts", "generate-types": "./node_modules/.bin/codex app-server generate-ts --out src/app-server", "test": "vitest run", + "test:e2e": "npm run build && RUN_E2E_TESTS=true vitest run src/__tests__/CodexACPAgent/e2e", "test:watch": "vitest", "typecheck": "tsc --noEmit", "codex-test": "tsx .claude/skills/run-codex/scripts/run-codex-test.ts" diff --git a/src/__tests__/CodexACPAgent/e2e/acp-e2e-test-utils.ts b/src/__tests__/CodexACPAgent/e2e/acp-e2e-test-utils.ts new file mode 100644 index 00000000..b7f6ad42 --- /dev/null +++ b/src/__tests__/CodexACPAgent/e2e/acp-e2e-test-utils.ts @@ -0,0 +1,242 @@ +import * as acp from "@agentclientprotocol/sdk"; +import {type ChildProcessWithoutNullStreams, spawn} from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import {Readable, Writable} from "node:stream"; +import {describe, vi} from "vitest"; +import {removeDirectoryWithRetry} from "../../acp-test-utils"; + +export const RUN_E2E_TESTS = process.env["RUN_E2E_TESTS"] === "true"; +const DEFAULT_E2E_SUITE_TIMEOUT_MS = 60_000; + +export interface SpawnedAgentFixture { + expectPromptText(promptText: string, assertText: (text: string) => void, timeoutMs?: number): Promise; + dispose(): Promise; +} + +export function describeE2E(name: string, factory: () => void, timeoutMs = DEFAULT_E2E_SUITE_TIMEOUT_MS): void { + describe.skipIf(!RUN_E2E_TESTS)(name, {timeout: timeoutMs}, factory); +} + +interface TestSkill { + readonly name: string; + readonly description: string; + readonly body: string; +} + +interface RuntimePaths { + readonly rootDir: string; + readonly codexHome: string; + readonly workspaceDir: string; + readonly appServerLogsDir: string; +} + +class RecordingClient implements acp.Client { + private readonly textBySessionId = new Map(); + + async requestPermission(_params: acp.RequestPermissionRequest): Promise { + return { + outcome: {outcome: "cancelled"}, + }; + } + + async sessionUpdate(params: acp.SessionNotification): Promise { + if (params.update.sessionUpdate !== "agent_message_chunk" || params.update.content.type !== "text") { + return; + } + + const nextText = `${this.textBySessionId.get(params.sessionId) ?? ""}${params.update.content.text}`; + this.textBySessionId.set(params.sessionId, nextText); + } + + readText(sessionId: string): string { + return this.textBySessionId.get(sessionId) ?? ""; + } +} + +export async function createFixtureWithSkill(skill: TestSkill): Promise { + const runtimePaths = createTemporaryRuntimePaths(); + writeSkill(runtimePaths.codexHome, skill); + return await createAuthenticatedFixture(runtimePaths); +} + +export async function createAuthenticatedFixture( + runtimePaths = createTemporaryRuntimePaths() +): Promise { + const apiKey = requireLiveApiKey(); + const agentProcess = spawn("npm", ["run", "--silent", "start"], { + cwd: process.cwd(), + env: { + ...process.env, + CODEX_HOME: runtimePaths.codexHome, + APP_SERVER_LOGS: runtimePaths.appServerLogsDir, + }, + stdio: ["pipe", "pipe", "pipe"], + }); + + const client = new RecordingClient(); + const output = Readable.toWeb(agentProcess.stdout) as ReadableStream; + const connection = new acp.ClientSideConnection( + () => client, + acp.ndJsonStream(Writable.toWeb(agentProcess.stdin), output) + ); + + const initializeResponse = await connection.initialize({ + protocolVersion: acp.PROTOCOL_VERSION, + clientCapabilities: {}, + clientInfo: { + name: "vitest", + version: "1.0.0", + }, + }); + + if (initializeResponse.protocolVersion !== acp.PROTOCOL_VERSION) { + throw new Error(`Unexpected protocol version: ${initializeResponse.protocolVersion}`); + } + + if (!initializeResponse.authMethods?.some((method) => method.id === "api-key")) { + throw new Error("API key authentication is not available."); + } + + await connection.authenticate({ + methodId: "api-key", + _meta: { + "api-key": { + apiKey, + }, + }, + }); + + const authenticationStatus = await getAuthenticationStatus(connection); + if (authenticationStatus["type"] !== "api-key") { + throw new Error(`Unexpected authentication status: ${JSON.stringify(authenticationStatus)}`); + } + + return { + async expectPromptText(promptText: string, assertText: (text: string) => void, timeoutMs = 30_000): Promise { + const newSessionResponse = await connection.newSession({ + cwd: runtimePaths.workspaceDir, + mcpServers: [], + }); + + const promptResponse = await connection.prompt({ + sessionId: newSessionResponse.sessionId, + prompt: [{ + type: "text", + text: promptText, + }], + }); + + if (promptResponse.stopReason !== "end_turn") { + throw new Error(`Unexpected stop reason: ${promptResponse.stopReason}`); + } + + await vi.waitFor(() => { + assertText(client.readText(newSessionResponse.sessionId)); + }, {timeout: timeoutMs}); + }, + async dispose(): Promise { + if (!agentProcess.stdin.destroyed && !agentProcess.stdin.writableEnded) { + agentProcess.stdin.end(); + } + + const exitedAfterStdinClose = await waitForProcessExit(agentProcess, 4_000); + if (!exitedAfterStdinClose && !agentProcess.killed) { + agentProcess.kill(); + await waitForProcessExit(agentProcess, 4_000); + } + + printLogDirectory(runtimePaths.appServerLogsDir); + removeDirectoryWithRetry(runtimePaths.rootDir); + }, + }; +} + +function requireLiveApiKey(): string { + const apiKey = process.env["CODEX_API_KEY"] ?? process.env["OPENAI_API_KEY"]; + if (!apiKey) { + throw new Error("Live integration test requires CODEX_API_KEY or OPENAI_API_KEY."); + } + return apiKey; +} + +function createTemporaryRuntimePaths(): RuntimePaths { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "codex-acp-integration-")); + const codexHome = path.join(rootDir, "codex-home"); + const workspaceDir = path.join(rootDir, "workspace"); + const appServerLogsDir = path.join(rootDir, "logs"); + + fs.mkdirSync(codexHome, {recursive: true}); + fs.mkdirSync(workspaceDir, {recursive: true}); + fs.mkdirSync(appServerLogsDir, {recursive: true}); + fs.writeFileSync(path.join(codexHome, "config.toml"), 'cli_auth_credentials_store = "file"\n', "utf8"); + + return { + rootDir, + codexHome, + workspaceDir, + appServerLogsDir, + }; +} + +function writeSkill(codexHome: string, skill: TestSkill): void { + const skillDirectory = path.join(codexHome, "skills", skill.name); + fs.mkdirSync(skillDirectory, {recursive: true}); + fs.writeFileSync( + path.join(skillDirectory, "SKILL.md"), + [ + "---", + `name: ${skill.name}`, + `description: ${skill.description}`, + "metadata:", + ` short-description: ${skill.description}`, + "---", + "", + skill.body, + "", + ].join("\n"), + "utf8", + ); +} + +async function getAuthenticationStatus(connection: acp.ClientSideConnection): Promise> { + return await connection.extMethod("authentication/status", {}); +} + +function printLogDirectory(logDirectory: string): void { + fs.readdirSync(logDirectory, {withFileTypes: true}) + .filter((entry) => entry.isFile()) + .forEach((entry) => { + const logFilePath = path.join(logDirectory, entry.name); + const content = fs.readFileSync(logFilePath, "utf8").trim(); + console.log(`[APP_SERVER_LOGS] Logs from ${logFilePath}:`); + console.log(content.length > 0 ? content : "[APP_SERVER_LOGS] Log file is empty"); + console.log("------"); + }); +} + +async function waitForProcessExit(proc: ChildProcessWithoutNullStreams, timeoutMs: number): Promise { + if (proc.exitCode !== null || proc.signalCode !== null) { + return true; + } + + return await new Promise((resolve) => { + const timeout = setTimeout(() => { + cleanup(); + resolve(false); + }, timeoutMs); + + const cleanup = () => { + clearTimeout(timeout); + proc.off("exit", handleExit); + }; + + const handleExit = () => { + cleanup(); + resolve(true); + }; + + proc.once("exit", handleExit); + }); +} diff --git a/src/__tests__/CodexACPAgent/e2e/acp-e2e.test.ts b/src/__tests__/CodexACPAgent/e2e/acp-e2e.test.ts new file mode 100644 index 00000000..1fff8f4a --- /dev/null +++ b/src/__tests__/CodexACPAgent/e2e/acp-e2e.test.ts @@ -0,0 +1,37 @@ +import {afterEach, expect, it} from "vitest"; +import { + createAuthenticatedFixture, + createFixtureWithSkill, + describeE2E, + type SpawnedAgentFixture, +} from "./acp-e2e-test-utils"; + +describeE2E("E2E tests", () => { + let fixture: SpawnedAgentFixture | null = null; + + afterEach(async () => { + if (fixture) { + await fixture.dispose(); + fixture = null; + } + }); + + it('returns model response', async () => { + fixture = await createAuthenticatedFixture(); + await fixture.expectPromptText("Reply with exactly integration-ok and nothing else.", (text) => { + expect(text.toLowerCase()).toContain("integration-ok"); + }); + }); + + it('lists a user skill from the wrapped CODEX_HOME', async () => { + fixture = await createFixtureWithSkill({ + name: "integration-skill", + description: "Integration skill", + body: "This skill exists only for integration testing.", + }); + await fixture.expectPromptText("/skills", (text) => { + expect(text).toContain("Available skills:"); + expect(text).toContain("- integration-skill: Integration skill"); + }); + }); +}); diff --git a/src/__tests__/acp-test-utils.ts b/src/__tests__/acp-test-utils.ts index 54053c77..d702f0d2 100644 --- a/src/__tests__/acp-test-utils.ts +++ b/src/__tests__/acp-test-utils.ts @@ -175,7 +175,7 @@ function createTestCodexHome(): string { return codexHome; } -function removeDirectoryWithRetry(directory: string): void { +export function removeDirectoryWithRetry(directory: string): void { for (let attempt = 0; attempt < 5; attempt += 1) { try { fs.rmSync(directory, { recursive: true, force: true });