fixup! LLM-25904 Add e2e test checking actual codex interaction

Author: ishulgin

package.json

@@ -32,7 +32,7 @@
     "start": "bun run src/index.ts",
     "generate-types": "./node_modules/.bin/codex app-server generate-ts --out src/app-server",
     "test": "vitest run",
-    "test:e2e": "npm run build && RUN_E2E_TESTS=true vitest run src/__tests__/CodexACPAgent/acp-e2e.test.ts",
+    "test:e2e": "npm run build && RUN_E2E_TESTS=true vitest run src/__tests__/CodexACPAgent/e2e",
     "test:watch": "vitest",
     "typecheck": "tsc --noEmit",
     "codex-test": "tsx .claude/skills/run-codex/scripts/run-codex-test.ts"

…/__tests__/CodexACPAgent/acp-e2e.test.ts …/CodexACPAgent/e2e/acp-e2e-test-utils.tssrc/__tests__/CodexACPAgent/acp-e2e.test.ts renamed to src/__tests__/CodexACPAgent/e2e/acp-e2e-test-utils.ts src/__tests__/CodexACPAgent/acp-e2e.test.ts renamed to src/__tests__/CodexACPAgent/e2e/acp-e2e-test-utils.ts

@@ -1,14 +1,36 @@
-import {afterEach, describe, expect, it, vi} from 'vitest';
-import * as acp from '@agentclientprotocol/sdk';
-import {type ChildProcessWithoutNullStreams, spawn} from 'node:child_process';
-import {Readable, Writable} from 'node:stream';
+import * as acp from "@agentclientprotocol/sdk";
+import {type ChildProcessWithoutNullStreams, spawn} from "node:child_process";
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
-import {removeDirectoryWithRetry} from "../acp-test-utils";
+import {Readable, Writable} from "node:stream";
+import {describe, vi} from "vitest";
+import {removeDirectoryWithRetry} from "../../acp-test-utils";
 
-const RUN_E2E_TESTS = process.env["RUN_E2E_TESTS"] === "true";
-const TEST_TIMEOUT_MS = 90_000;
+export const RUN_E2E_TESTS = process.env["RUN_E2E_TESTS"] === "true";
+const DEFAULT_E2E_SUITE_TIMEOUT_MS = 60_000;
+
+export interface SpawnedAgentFixture {
+    expectPromptText(promptText: string, assertText: (text: string) => void, timeoutMs?: number): Promise<void>;
+    dispose(): Promise<void>;
+}
+
+export function describeE2E(name: string, factory: () => void, timeoutMs = DEFAULT_E2E_SUITE_TIMEOUT_MS): void {
+    describe.skipIf(!RUN_E2E_TESTS)(name, {timeout: timeoutMs}, factory);
+}
+
+interface TestSkill {
+    readonly name: string;
+    readonly description: string;
+    readonly body: string;
+}
+
+interface RuntimePaths {
+    readonly rootDir: string;
+    readonly codexHome: string;
+    readonly workspaceDir: string;
+    readonly appServerLogsDir: string;
+}
 
 class RecordingClient implements acp.Client {
     private readonly textBySessionId = new Map<string, string>();
@@ -33,61 +55,15 @@ class RecordingClient implements acp.Client {
     }
 }
 
-interface SpawnedAgentFixture {
-    expectPromptText(promptText: string, assertText: (text: string) => void, timeoutMs?: number): Promise<void>;
-    dispose(): Promise<void>;
-}
-
-interface TestSkill {
-    readonly name: string;
-    readonly description: string;
-    readonly body: string;
-}
-
-interface RuntimePaths {
-    readonly rootDir: string;
-    readonly codexHome: string;
-    readonly workspaceDir: string;
-    readonly appServerLogsDir: string;
-}
-
-describe.skipIf(!RUN_E2E_TESTS)('E2E tests', {timeout: TEST_TIMEOUT_MS}, () => {
-    let fixture: SpawnedAgentFixture | null = null;
-
-    afterEach(async () => {
-        if (fixture) {
-            await fixture.dispose();
-            fixture = null;
-        }
-    });
-
-    it('returns model response', async () => {
-        fixture = await createAuthenticatedFixture();
-        await fixture.expectPromptText("Reply with exactly integration-ok and nothing else.", (text) => {
-            expect(text.toLowerCase()).toContain("integration-ok");
-        });
-    });
-
-    it('lists a user skill from the wrapped CODEX_HOME', async () => {
-        fixture = await createFixtureWithSkill({
-            name: "integration-skill",
-            description: "Integration skill",
-            body: "This skill exists only for integration testing.",
-        });
-        await fixture.expectPromptText("/skills", (text) => {
-            expect(text).toContain("Available skills:");
-            expect(text).toContain("- integration-skill: Integration skill");
-        });
-    });
-});
-
-async function createFixtureWithSkill(skill: TestSkill): Promise<SpawnedAgentFixture> {
+export async function createFixtureWithSkill(skill: TestSkill): Promise<SpawnedAgentFixture> {
     const runtimePaths = createTemporaryRuntimePaths();
     writeSkill(runtimePaths.codexHome, skill);
     return await createAuthenticatedFixture(runtimePaths);
 }
 
-async function createAuthenticatedFixture(runtimePaths = createTemporaryRuntimePaths()): Promise<SpawnedAgentFixture> {
+export async function createAuthenticatedFixture(
+    runtimePaths = createTemporaryRuntimePaths()
+): Promise<SpawnedAgentFixture> {
     const apiKey = requireLiveApiKey();
     const agentProcess = spawn("npm", ["run", "--silent", "start"], {
         cwd: process.cwd(),
@@ -114,8 +90,14 @@ async function createAuthenticatedFixture(runtimePaths = createTemporaryRuntimeP
             version: "1.0.0",
         },
     });
-    expect(initializeResponse.protocolVersion).toBe(acp.PROTOCOL_VERSION);
-    expect(initializeResponse.authMethods?.map((method) => method.id)).toContain("api-key");
+
+    if (initializeResponse.protocolVersion !== acp.PROTOCOL_VERSION) {
+        throw new Error(`Unexpected protocol version: ${initializeResponse.protocolVersion}`);
+    }
+
+    if (!initializeResponse.authMethods?.some((method) => method.id === "api-key")) {
+        throw new Error("API key authentication is not available.");
+    }
 
     await connection.authenticate({
         methodId: "api-key",
@@ -126,7 +108,10 @@ async function createAuthenticatedFixture(runtimePaths = createTemporaryRuntimeP
         },
     });
 
-    expect(await getAuthenticationStatus(connection)).toEqual({type: "api-key"});
+    const authenticationStatus = await getAuthenticationStatus(connection);
+    if (authenticationStatus["type"] !== "api-key") {
+        throw new Error(`Unexpected authentication status: ${JSON.stringify(authenticationStatus)}`);
+    }
 
     return {
         async expectPromptText(promptText: string, assertText: (text: string) => void, timeoutMs = 30_000): Promise<void> {
@@ -143,13 +128,15 @@ async function createAuthenticatedFixture(runtimePaths = createTemporaryRuntimeP
                 }],
             });
 
-            expect(promptResponse.stopReason).toBe("end_turn");
+            if (promptResponse.stopReason !== "end_turn") {
+                throw new Error(`Unexpected stop reason: ${promptResponse.stopReason}`);
+            }
 
             await vi.waitFor(() => {
                 assertText(client.readText(newSessionResponse.sessionId));
             }, {timeout: timeoutMs});
         },
-        async dispose() {
+        async dispose(): Promise<void> {
             if (!agentProcess.stdin.destroyed && !agentProcess.stdin.writableEnded) {
                 agentProcess.stdin.end();
             }
@@ -179,10 +166,12 @@ function createTemporaryRuntimePaths(): RuntimePaths {
     const codexHome = path.join(rootDir, "codex-home");
     const workspaceDir = path.join(rootDir, "workspace");
     const appServerLogsDir = path.join(rootDir, "logs");
+
     fs.mkdirSync(codexHome, {recursive: true});
     fs.mkdirSync(workspaceDir, {recursive: true});
     fs.mkdirSync(appServerLogsDir, {recursive: true});
     fs.writeFileSync(path.join(codexHome, "config.toml"), 'cli_auth_credentials_store = "file"\n', "utf8");
+
     return {
         rootDir,
         codexHome,
@@ -223,7 +212,7 @@ function printLogDirectory(logDirectory: string): void {
             const content = fs.readFileSync(logFilePath, "utf8").trim();
             console.log(`[APP_SERVER_LOGS] Logs from ${logFilePath}:`);
             console.log(content.length > 0 ? content : "[APP_SERVER_LOGS] Log file is empty");
-            console.log('------');
+            console.log("------");
         });
 }
 

src/__tests__/CodexACPAgent/e2e/acp-e2e.test.ts

@@ -0,0 +1,37 @@
+import {afterEach, expect, it} from "vitest";
+import {
+    createAuthenticatedFixture,
+    createFixtureWithSkill,
+    describeE2E,
+    type SpawnedAgentFixture,
+} from "./acp-e2e-test-utils";
+
+describeE2E("E2E tests", () => {
+    let fixture: SpawnedAgentFixture | null = null;
+
+    afterEach(async () => {
+        if (fixture) {
+            await fixture.dispose();
+            fixture = null;
+        }
+    });
+
+    it('returns model response', async () => {
+        fixture = await createAuthenticatedFixture();
+        await fixture.expectPromptText("Reply with exactly integration-ok and nothing else.", (text) => {
+            expect(text.toLowerCase()).toContain("integration-ok");
+        });
+    });
+
+    it('lists a user skill from the wrapped CODEX_HOME', async () => {
+        fixture = await createFixtureWithSkill({
+            name: "integration-skill",
+            description: "Integration skill",
+            body: "This skill exists only for integration testing.",
+        });
+        await fixture.expectPromptText("/skills", (text) => {
+            expect(text).toContain("Available skills:");
+            expect(text).toContain("- integration-skill: Integration skill");
+        });
+    });
+});