Skip to content

Commit bc6efd9

Browse files
ishulginishulgin
committed
LLM-25904 Add e2e test checking actual codex interaction
1 parent a870bdb commit bc6efd9

5 files changed

Lines changed: 286 additions & 1 deletion

File tree

‎.github/workflows/ci.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,11 @@ jobs:
2626
node-version: '24'
2727
- run: npm ci
2828
- run: npm test
29+
- name: Run e2e tests
30+
env:
31+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
32+
run:
33+
npm run test:e2e
2934

3035
build:
3136
runs-on: ubuntu-latest

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
"start": "node --import tsx src/index.ts",
3333
"generate-types": "./node_modules/.bin/codex app-server generate-ts --out src/app-server",
3434
"test": "vitest run",
35+
"test:e2e": "npm run build && RUN_E2E_TESTS=true vitest run src/__tests__/CodexACPAgent/e2e",
3536
"test:watch": "vitest",
3637
"typecheck": "tsc --noEmit",
3738
"codex-test": "tsx .claude/skills/run-codex/scripts/run-codex-test.ts"

‎src/__tests__/CodexACPAgent/e2e/acp-e2e-test-utils.ts‎

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
import * as acp from "@agentclientprotocol/sdk";
2+
import {type ChildProcessWithoutNullStreams, spawn} from "node:child_process";
3+
import fs from "node:fs";
4+
import os from "node:os";
5+
import path from "node:path";
6+
import {Readable, Writable} from "node:stream";
7+
import {describe, vi} from "vitest";
8+
import {removeDirectoryWithRetry} from "../../acp-test-utils";
9+
10+
export const RUN_E2E_TESTS = process.env["RUN_E2E_TESTS"] === "true";
11+
const DEFAULT_E2E_SUITE_TIMEOUT_MS = 60_000;
12+
13+
export interface SpawnedAgentFixture {
14+
expectPromptText(promptText: string, assertText: (text: string) => void, timeoutMs?: number): Promise<void>;
15+
dispose(): Promise<void>;
16+
}
17+
18+
export function describeE2E(name: string, factory: () => void, timeoutMs = DEFAULT_E2E_SUITE_TIMEOUT_MS): void {
19+
describe.skipIf(!RUN_E2E_TESTS)(name, {timeout: timeoutMs}, factory);
20+
}
21+
22+
interface TestSkill {
23+
readonly name: string;
24+
readonly description: string;
25+
readonly body: string;
26+
}
27+
28+
interface RuntimePaths {
29+
readonly rootDir: string;
30+
readonly codexHome: string;
31+
readonly workspaceDir: string;
32+
readonly appServerLogsDir: string;
33+
}
34+
35+
class RecordingClient implements acp.Client {
36+
private readonly textBySessionId = new Map<string, string>();
37+
38+
async requestPermission(_params: acp.RequestPermissionRequest): Promise<acp.RequestPermissionResponse> {
39+
return {
40+
outcome: {outcome: "cancelled"},
41+
};
42+
}
43+
44+
async sessionUpdate(params: acp.SessionNotification): Promise<void> {
45+
if (params.update.sessionUpdate !== "agent_message_chunk" || params.update.content.type !== "text") {
46+
return;
47+
}
48+
49+
const nextText = `${this.textBySessionId.get(params.sessionId) ?? ""}${params.update.content.text}`;
50+
this.textBySessionId.set(params.sessionId, nextText);
51+
}
52+
53+
readText(sessionId: string): string {
54+
return this.textBySessionId.get(sessionId) ?? "";
55+
}
56+
}
57+
58+
export async function createFixtureWithSkill(skill: TestSkill): Promise<SpawnedAgentFixture> {
59+
const runtimePaths = createTemporaryRuntimePaths();
60+
writeSkill(runtimePaths.codexHome, skill);
61+
return await createAuthenticatedFixture(runtimePaths);
62+
}
63+
64+
export async function createAuthenticatedFixture(
65+
runtimePaths = createTemporaryRuntimePaths()
66+
): Promise<SpawnedAgentFixture> {
67+
const apiKey = requireLiveApiKey();
68+
const agentProcess = spawn("npm", ["run", "--silent", "start"], {
69+
cwd: process.cwd(),
70+
env: {
71+
...process.env,
72+
CODEX_HOME: runtimePaths.codexHome,
73+
APP_SERVER_LOGS: runtimePaths.appServerLogsDir,
74+
},
75+
stdio: ["pipe", "pipe", "pipe"],
76+
});
77+
78+
const client = new RecordingClient();
79+
const output = Readable.toWeb(agentProcess.stdout) as ReadableStream<Uint8Array>;
80+
const connection = new acp.ClientSideConnection(
81+
() => client,
82+
acp.ndJsonStream(Writable.toWeb(agentProcess.stdin), output)
83+
);
84+
85+
const initializeResponse = await connection.initialize({
86+
protocolVersion: acp.PROTOCOL_VERSION,
87+
clientCapabilities: {},
88+
clientInfo: {
89+
name: "vitest",
90+
version: "1.0.0",
91+
},
92+
});
93+
94+
if (initializeResponse.protocolVersion !== acp.PROTOCOL_VERSION) {
95+
throw new Error(`Unexpected protocol version: ${initializeResponse.protocolVersion}`);
96+
}
97+
98+
if (!initializeResponse.authMethods?.some((method) => method.id === "api-key")) {
99+
throw new Error("API key authentication is not available.");
100+
}
101+
102+
await connection.authenticate({
103+
methodId: "api-key",
104+
_meta: {
105+
"api-key": {
106+
apiKey,
107+
},
108+
},
109+
});
110+
111+
const authenticationStatus = await getAuthenticationStatus(connection);
112+
if (authenticationStatus["type"] !== "api-key") {
113+
throw new Error(`Unexpected authentication status: ${JSON.stringify(authenticationStatus)}`);
114+
}
115+
116+
return {
117+
async expectPromptText(promptText: string, assertText: (text: string) => void, timeoutMs = 30_000): Promise<void> {
118+
const newSessionResponse = await connection.newSession({
119+
cwd: runtimePaths.workspaceDir,
120+
mcpServers: [],
121+
});
122+
123+
const promptResponse = await connection.prompt({
124+
sessionId: newSessionResponse.sessionId,
125+
prompt: [{
126+
type: "text",
127+
text: promptText,
128+
}],
129+
});
130+
131+
if (promptResponse.stopReason !== "end_turn") {
132+
throw new Error(`Unexpected stop reason: ${promptResponse.stopReason}`);
133+
}
134+
135+
await vi.waitFor(() => {
136+
assertText(client.readText(newSessionResponse.sessionId));
137+
}, {timeout: timeoutMs});
138+
},
139+
async dispose(): Promise<void> {
140+
if (!agentProcess.stdin.destroyed && !agentProcess.stdin.writableEnded) {
141+
agentProcess.stdin.end();
142+
}
143+
144+
const exitedAfterStdinClose = await waitForProcessExit(agentProcess, 4_000);
145+
if (!exitedAfterStdinClose && !agentProcess.killed) {
146+
agentProcess.kill();
147+
await waitForProcessExit(agentProcess, 4_000);
148+
}
149+
150+
printLogDirectory(runtimePaths.appServerLogsDir);
151+
removeDirectoryWithRetry(runtimePaths.rootDir);
152+
},
153+
};
154+
}
155+
156+
function requireLiveApiKey(): string {
157+
const apiKey = process.env["CODEX_API_KEY"] ?? process.env["OPENAI_API_KEY"];
158+
if (!apiKey) {
159+
throw new Error("Live integration test requires CODEX_API_KEY or OPENAI_API_KEY.");
160+
}
161+
return apiKey;
162+
}
163+
164+
function createTemporaryRuntimePaths(): RuntimePaths {
165+
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "codex-acp-integration-"));
166+
const codexHome = path.join(rootDir, "codex-home");
167+
const workspaceDir = path.join(rootDir, "workspace");
168+
const appServerLogsDir = path.join(rootDir, "logs");
169+
170+
fs.mkdirSync(codexHome, {recursive: true});
171+
fs.mkdirSync(workspaceDir, {recursive: true});
172+
fs.mkdirSync(appServerLogsDir, {recursive: true});
173+
fs.writeFileSync(path.join(codexHome, "config.toml"), 'cli_auth_credentials_store = "file"\n', "utf8");
174+
175+
return {
176+
rootDir,
177+
codexHome,
178+
workspaceDir,
179+
appServerLogsDir,
180+
};
181+
}
182+
183+
function writeSkill(codexHome: string, skill: TestSkill): void {
184+
const skillDirectory = path.join(codexHome, "skills", skill.name);
185+
fs.mkdirSync(skillDirectory, {recursive: true});
186+
fs.writeFileSync(
187+
path.join(skillDirectory, "SKILL.md"),
188+
[
189+
"---",
190+
`name: ${skill.name}`,
191+
`description: ${skill.description}`,
192+
"metadata:",
193+
` short-description: ${skill.description}`,
194+
"---",
195+
"",
196+
skill.body,
197+
"",
198+
].join("\n"),
199+
"utf8",
200+
);
201+
}
202+
203+
async function getAuthenticationStatus(connection: acp.ClientSideConnection): Promise<Record<string, unknown>> {
204+
return await connection.extMethod("authentication/status", {});
205+
}
206+
207+
function printLogDirectory(logDirectory: string): void {
208+
fs.readdirSync(logDirectory, {withFileTypes: true})
209+
.filter((entry) => entry.isFile())
210+
.forEach((entry) => {
211+
const logFilePath = path.join(logDirectory, entry.name);
212+
const content = fs.readFileSync(logFilePath, "utf8").trim();
213+
console.log(`[APP_SERVER_LOGS] Logs from ${logFilePath}:`);
214+
console.log(content.length > 0 ? content : "[APP_SERVER_LOGS] Log file is empty");
215+
console.log("------");
216+
});
217+
}
218+
219+
async function waitForProcessExit(proc: ChildProcessWithoutNullStreams, timeoutMs: number): Promise<boolean> {
220+
if (proc.exitCode !== null || proc.signalCode !== null) {
221+
return true;
222+
}
223+
224+
return await new Promise<boolean>((resolve) => {
225+
const timeout = setTimeout(() => {
226+
cleanup();
227+
resolve(false);
228+
}, timeoutMs);
229+
230+
const cleanup = () => {
231+
clearTimeout(timeout);
232+
proc.off("exit", handleExit);
233+
};
234+
235+
const handleExit = () => {
236+
cleanup();
237+
resolve(true);
238+
};
239+
240+
proc.once("exit", handleExit);
241+
});
242+
}

‎src/__tests__/CodexACPAgent/e2e/acp-e2e.test.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import {afterEach, expect, it} from "vitest";
2+
import {
3+
createAuthenticatedFixture,
4+
createFixtureWithSkill,
5+
describeE2E,
6+
type SpawnedAgentFixture,
7+
} from "./acp-e2e-test-utils";
8+
9+
describeE2E("E2E tests", () => {
10+
let fixture: SpawnedAgentFixture | null = null;
11+
12+
afterEach(async () => {
13+
if (fixture) {
14+
await fixture.dispose();
15+
fixture = null;
16+
}
17+
});
18+
19+
it('returns model response', async () => {
20+
fixture = await createAuthenticatedFixture();
21+
await fixture.expectPromptText("Reply with exactly integration-ok and nothing else.", (text) => {
22+
expect(text.toLowerCase()).toContain("integration-ok");
23+
});
24+
});
25+
26+
it('lists a user skill from the wrapped CODEX_HOME', async () => {
27+
fixture = await createFixtureWithSkill({
28+
name: "integration-skill",
29+
description: "Integration skill",
30+
body: "This skill exists only for integration testing.",
31+
});
32+
await fixture.expectPromptText("/skills", (text) => {
33+
expect(text).toContain("Available skills:");
34+
expect(text).toContain("- integration-skill: Integration skill");
35+
});
36+
});
37+
});

‎src/__tests__/acp-test-utils.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ function createTestCodexHome(): string {
175175
return codexHome;
176176
}
177177

178-
function removeDirectoryWithRetry(directory: string): void {
178+
export function removeDirectoryWithRetry(directory: string): void {
179179
for (let attempt = 0; attempt < 5; attempt += 1) {
180180
try {
181181
fs.rmSync(directory, { recursive: true, force: true });

0 commit comments

Comments
 (0)