Skip to content

Commit 0aa879b

Browse files
ishulginishulgin
committed
Add e2e permissions tests for different agents modes
Also, create workspace dir inside repository to avoid special treatment of tmp dir
1 parent 8d8b80b commit 0aa879b

6 files changed

Lines changed: 175 additions & 36 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ dist/
33
*.log
44
.DS_Store
55
.idea/
6+
tmp/

‎src/__tests__/CodexACPAgent/e2e/acp-e2e-file-approval.test.ts‎

Lines changed: 78 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,108 @@
11
import fs from "node:fs";
22
import path from "node:path";
3-
import {afterEach, beforeEach, expect, it} from "vitest";
3+
import {afterEach, beforeEach, expect, it, onTestFinished} from "vitest";
44
import {AgentMode} from "../../../AgentMode";
55
import {ApprovalOptionId} from "../../../ApprovalOptionId";
66
import {
77
createAuthenticatedFixture,
88
createPermissionResponder,
99
describeE2E,
10+
expectNoPermissionRequests,
11+
expectPermissionRequests,
12+
generateFileNameForTest,
1013
type SpawnedAgentFixture,
1114
} from "./acp-e2e-test-utils";
1215

13-
const FILE_NAME = "approval-file.txt";
1416
const FILE_CONTENT = "file approval e2e";
1517

1618
describeE2E("E2E file approval tests", () => {
1719
let fixture: SpawnedAgentFixture;
18-
let sessionId: string;
1920

2021
beforeEach(async () => {
2122
fixture = await createAuthenticatedFixture(AgentMode.ReadOnly);
22-
sessionId = (await fixture.createSession()).sessionId;
2323
});
2424

2525
afterEach(async () => {
2626
await fixture.dispose();
2727
});
2828

29-
async function expectFileApproval(
30-
optionId: ApprovalOptionId,
31-
expectedStopReason: "end_turn" | "cancelled",
32-
): Promise<void> {
33-
fixture.setPermissionResponder(createPermissionResponder("edit", optionId));
34-
const response = await fixture.connection.prompt({
35-
sessionId,
36-
prompt: [{
37-
type: "text",
38-
text: `Create ${FILE_NAME} by editing files directly. Content must be exactly: ${FILE_CONTENT}. Do not use shell commands, and stop if the edit is rejected.`,
39-
}],
40-
});
41-
expect(response.stopReason).toBe(expectedStopReason);
42-
expect(fixture.readPermissionRequests(sessionId, "edit").length).toBe(1);
43-
expect(fixture.readPermissionRequests(sessionId, "execute").length).toBe(0);
44-
}
45-
4629
it("applies approved file edits", async () => {
47-
await expectFileApproval(ApprovalOptionId.AllowOnce, "end_turn");
48-
const filePath = path.join(fixture.workspaceDir, FILE_NAME);
49-
expect(fs.existsSync(filePath)).toBe(true);
50-
expect(fs.readFileSync(filePath, "utf8").trim()).toBe(FILE_CONTENT);
30+
fixture.setPermissionResponder(createPermissionResponder("edit", ApprovalOptionId.AllowOnce));
31+
const sessionId = await editFileDirectly(fixture, path.join(fixture.workspaceDir, generateFileNameForTest()), true);
32+
expectPermissionRequests(fixture, sessionId, {edit: 1, execute: 0});
5133
});
5234

5335
it("does not apply rejected file edits", async () => {
54-
await expectFileApproval(ApprovalOptionId.RejectOnce, "cancelled");
55-
expect(fs.existsSync(path.join(fixture.workspaceDir, FILE_NAME))).toBe(false);
36+
fixture.setPermissionResponder(createPermissionResponder("edit", ApprovalOptionId.RejectOnce));
37+
const sessionId = await editFileDirectly(fixture, path.join(fixture.workspaceDir, generateFileNameForTest()), false);
38+
expectPermissionRequests(fixture, sessionId, {edit: 1, execute: 0});
39+
});
40+
});
41+
42+
describeE2E("E2E Agent mode file permission tests", () => {
43+
let fixture: SpawnedAgentFixture;
44+
45+
beforeEach(async () => {
46+
fixture = await createAuthenticatedFixture(AgentMode.Agent);
47+
});
48+
49+
afterEach(async () => {
50+
await fixture.dispose();
51+
});
52+
53+
it("edits a workspace file without prompting for permission", async () => {
54+
const sessionId = await editFileDirectly(fixture, path.join(fixture.workspaceDir, generateFileNameForTest()), true);
55+
expectNoPermissionRequests(fixture, sessionId);
56+
});
57+
58+
it("can't edit file outside workspace", async () => {
59+
const dir = createDirOutsideWorkspace(fixture);
60+
await editFileDirectly(fixture, path.join(dir, generateFileNameForTest()), false);
5661
});
5762
});
63+
64+
describeE2E("E2E Agent with full access file permission tests", () => {
65+
let fixture: SpawnedAgentFixture;
66+
67+
beforeEach(async () => {
68+
fixture = await createAuthenticatedFixture(AgentMode.AgentFullAccess);
69+
});
70+
71+
afterEach(async () => {
72+
await fixture.dispose();
73+
});
74+
75+
it("edits a file outside workspace without prompting for permission", async () => {
76+
const dir = createDirOutsideWorkspace(fixture);
77+
const sessionId = await editFileDirectly(fixture, path.join(dir, generateFileNameForTest()), true);
78+
expectNoPermissionRequests(fixture, sessionId);
79+
});
80+
});
81+
82+
async function editFileDirectly(
83+
fixture: SpawnedAgentFixture,
84+
filePath: string,
85+
expectSuccess: boolean,
86+
): Promise<string> {
87+
const sessionId = (await fixture.createSession()).sessionId;
88+
await fixture.connection.prompt({
89+
sessionId,
90+
prompt: [{
91+
type: "text",
92+
text: `Create ${filePath} by editing files directly. Content must be exactly: ${FILE_CONTENT}. Do not use shell commands.`,
93+
}],
94+
});
95+
if (expectSuccess) {
96+
expect(fs.readFileSync(filePath, "utf8").trim()).toBe(FILE_CONTENT);
97+
} else {
98+
expect(fs.existsSync(filePath)).toBe(false);
99+
}
100+
return sessionId;
101+
}
102+
103+
function createDirOutsideWorkspace(fixture: SpawnedAgentFixture): string {
104+
const outsideWorkspaceDir = path.join(path.dirname(fixture.workspaceDir), "outside-workspace");
105+
onTestFinished(() => fs.rmSync(outsideWorkspaceDir, {recursive: true, force: true}));
106+
fs.mkdirSync(outsideWorkspaceDir);
107+
return outsideWorkspaceDir;
108+
}

‎src/__tests__/CodexACPAgent/e2e/acp-e2e-session-persistence.test.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import {afterEach, expect, it} from "vitest";
2-
import {AgentMode} from "../../../AgentMode";
32
import {
43
createAuthenticatedFixture,
54
describeE2E,

‎src/__tests__/CodexACPAgent/e2e/acp-e2e-shell-approval.test.ts‎

Lines changed: 74 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
import fs from "node:fs";
22
import path from "node:path";
3-
import {afterEach, beforeEach, expect, it, vi} from "vitest";
3+
import {afterEach, beforeEach, expect, it, onTestFinished, vi} from "vitest";
44
import {AgentMode} from "../../../AgentMode";
55
import {ApprovalOptionId} from "../../../ApprovalOptionId";
66
import {
77
createAuthenticatedFixture,
8-
createPermissionResponse,
98
createPermissionResponder,
9+
createPermissionResponse,
1010
describeE2E,
1111
expectEndTurn,
12+
expectNoPermissionRequests,
13+
expectPermissionRequests,
14+
generateFileNameForTest,
1215
type SpawnedAgentFixture,
1316
} from "./acp-e2e-test-utils";
1417

@@ -51,23 +54,66 @@ describeE2E("E2E shell approval tests", () => {
5154
await promptShellCommandTwice();
5255
expect(fs.existsSync(path.join(fixture.workspaceDir, FIRST_FILE_NAME))).toBe(true);
5356
expect(fs.existsSync(path.join(fixture.workspaceDir, SECOND_FILE_NAME))).toBe(false);
54-
expect(fixture.readPermissionRequests(sessionId, "execute").length).toBe(2);
57+
expectPermissionRequests(fixture, sessionId, {execute: 2, edit: 0});
5558
});
5659

5760
it("skips subsequent approvals when allow_always is selected", async () => {
5861
fixture.setPermissionResponder(createPermissionResponder("execute", ApprovalOptionId.AllowAlways));
5962
await promptShellCommandTwice();
6063
expect(fs.existsSync(path.join(fixture.workspaceDir, FIRST_FILE_NAME))).toBe(true);
6164
expect(fs.existsSync(path.join(fixture.workspaceDir, SECOND_FILE_NAME))).toBe(true);
62-
expect(fixture.readPermissionRequests(sessionId, "execute").length).toBe(1);
65+
expectPermissionRequests(fixture, sessionId, {execute: 1, edit: 0});
6366
});
6467

6568
it("prompts for every command when reject_once is selected", async () => {
6669
fixture.setPermissionResponder(createPermissionResponder("execute", ApprovalOptionId.RejectOnce));
6770
await promptShellCommandTwice();
6871
expect(fs.existsSync(path.join(fixture.workspaceDir, FIRST_FILE_NAME))).toBe(false);
6972
expect(fs.existsSync(path.join(fixture.workspaceDir, SECOND_FILE_NAME))).toBe(false);
70-
expect(fixture.readPermissionRequests(sessionId, "execute").length).toBe(2);
73+
expectPermissionRequests(fixture, sessionId, {execute: 2, edit: 0});
74+
});
75+
});
76+
77+
describeE2E("E2E Agent mode shell permission tests", () => {
78+
let fixture: SpawnedAgentFixture;
79+
80+
beforeEach(async () => {
81+
fixture = await createAuthenticatedFixture(AgentMode.Agent);
82+
});
83+
84+
afterEach(async () => {
85+
await fixture.dispose();
86+
});
87+
88+
it("runs a workspace command without prompting for permission", async () => {
89+
const sessionId = await writeToFile(fixture, path.join(fixture.workspaceDir, generateFileNameForTest()));
90+
91+
expectNoPermissionRequests(fixture, sessionId);
92+
});
93+
94+
it("requests permission for a command that writes outside the workspace", async () => {
95+
const dir = createDirOutsideWorkspace(fixture);
96+
fixture.setPermissionResponder(createPermissionResponder("execute", ApprovalOptionId.AllowOnce));
97+
const sessionId = await writeToFile(fixture, path.join(dir, generateFileNameForTest()));
98+
expectPermissionRequests(fixture, sessionId, {execute: 1, edit: 0});
99+
});
100+
});
101+
102+
describeE2E("E2E Agent with full access shell permission tests", () => {
103+
let fixture: SpawnedAgentFixture;
104+
105+
beforeEach(async () => {
106+
fixture = await createAuthenticatedFixture(AgentMode.AgentFullAccess);
107+
});
108+
109+
afterEach(async () => {
110+
await fixture.dispose();
111+
});
112+
113+
it("runs a command outside workspace without prompting for permission", async () => {
114+
const dir = createDirOutsideWorkspace(fixture);
115+
const sessionId = await writeToFile(fixture, path.join(dir, generateFileNameForTest()));
116+
expectNoPermissionRequests(fixture, sessionId);
71117
});
72118
});
73119

@@ -116,3 +162,26 @@ describeE2E("E2E shell cancellation tests", () => {
116162
}, {timeout: 5_000});
117163
});
118164
});
165+
166+
async function writeToFile(fixture: SpawnedAgentFixture, filePath: string): Promise<string> {
167+
const content = "hello from e2e";
168+
const command = `printf '${content}' > '${filePath}'`;
169+
const sessionId = (await fixture.createSession()).sessionId;
170+
const response = await fixture.connection.prompt({
171+
sessionId,
172+
prompt: [{
173+
type: "text",
174+
text: `Use your shell tool to run exactly \`${command}\`. Do not modify files any other way.`,
175+
}],
176+
});
177+
expectEndTurn(response);
178+
expect(fs.readFileSync(filePath, "utf8").trim()).toBe(content);
179+
return sessionId;
180+
}
181+
182+
function createDirOutsideWorkspace(fixture: SpawnedAgentFixture): string {
183+
const outsideWorkspaceDir = path.join(path.dirname(fixture.workspaceDir), "outside-workspace");
184+
onTestFinished(() => fs.rmSync(outsideWorkspaceDir, {recursive: true, force: true}));
185+
fs.mkdirSync(outsideWorkspaceDir);
186+
return outsideWorkspaceDir;
187+
}

‎src/__tests__/CodexACPAgent/e2e/acp-e2e-test-utils.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,26 @@ export function expectEndTurn(response: acp.PromptResponse): void {
2626
expect(response.stopReason).toBe("end_turn");
2727
}
2828

29+
export function expectCancelled(response: acp.PromptResponse): void {
30+
expect(response.stopReason).toBe("cancelled");
31+
}
32+
33+
export function generateFileNameForTest(): string {
34+
return `test-file-${crypto.randomUUID()}.txt`;
35+
}
36+
37+
export function expectPermissionRequests(fixture: SpawnedAgentFixture, sessionId: string, requests: {
38+
edit: number,
39+
execute: number,
40+
}): void {
41+
expect(fixture.readPermissionRequests(sessionId, "edit").length).toBe(requests.edit);
42+
expect(fixture.readPermissionRequests(sessionId, "execute").length).toBe(requests.execute);
43+
}
44+
45+
export function expectNoPermissionRequests(fixture: SpawnedAgentFixture, sessionId: string): void {
46+
expectPermissionRequests(fixture, sessionId, { edit: 0, execute: 0 });
47+
}
48+
2949
export async function createAuthenticatedFixture(initialMode?: AgentMode): Promise<SpawnedAgentFixture> {
3050
const apiKey = requireLiveApiKey();
3151
const extraEnv = initialMode ? {INITIAL_AGENT_MODE: initialMode.id} : undefined;

‎src/__tests__/CodexACPAgent/e2e/spawned-agent-fixture.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import * as acp from "@agentclientprotocol/sdk";
22
import {type ChildProcessWithoutNullStreams, spawn} from "node:child_process";
33
import fs from "node:fs";
4-
import os from "node:os";
54
import path from "node:path";
65
import {Readable, Writable} from "node:stream";
76
import {expect, vi} from "vitest";
@@ -80,9 +79,9 @@ class RuntimePaths {
8079
}
8180

8281
static createTemporary(): RuntimePaths {
83-
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "codex-acp-integration-"));
82+
const rootDir = path.join(process.cwd(), "tmp", crypto.randomUUID());
8483
const paths = new RuntimePaths(rootDir);
85-
for (const dir of [paths.codexHome, paths.workspaceDir, paths.appServerLogsDir]) {
84+
for (const dir of [paths.rootDir, paths.codexHome, paths.workspaceDir, paths.appServerLogsDir]) {
8685
fs.mkdirSync(dir, {recursive: true});
8786
}
8887
writeCodexHomeConfig(paths.codexHome, {

0 commit comments

Comments
 (0)