Skip to content

Commit f73ae42

Browse files
roomoteroomote
committed
test: unskip mutating vscode e2e tool suites
1 parent 4baac4e commit f73ae42

10 files changed

Lines changed: 696 additions & 1164 deletions

File tree

‎apps/vscode-e2e/fixtures/apply-diff.json‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "APPLY_DIFF_SIMPLE_SMOKE"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "apply_diff",
11+
"arguments": "{\"path\":\"apply-diff-tool-fixture/simple-modify.txt\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nHello World\\n=======\\nHello Universe\\n>>>>>>> REPLACE\"}",
12+
"id": "call_apply_diff_simple_001"
13+
}
14+
]
15+
}
16+
},
17+
{
18+
"match": {
19+
"userMessage": "APPLY_DIFF_MULTI_REPLACE_SMOKE"
20+
},
21+
"response": {
22+
"toolCalls": [
23+
{
24+
"name": "apply_diff",
25+
"arguments": "{\"path\":\"apply-diff-tool-fixture/multiple-replace.js\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nfunction calculate(x, y) {\\n\\tconst sum = x + y\\n\\tconst product = x * y\\n\\treturn { sum: sum, product: product }\\n}\\n=======\\nfunction compute(a, b) {\\n\\tconst total = a + b\\n\\tconst result = a * b\\n\\treturn { total: total, result: result }\\n}\\n>>>>>>> REPLACE\"}",
26+
"id": "call_apply_diff_multi_replace_001"
27+
}
28+
]
29+
}
30+
},
31+
{
32+
"match": {
33+
"userMessage": "APPLY_DIFF_LINE_HINTS_SMOKE"
34+
},
35+
"response": {
36+
"toolCalls": [
37+
{
38+
"name": "apply_diff",
39+
"arguments": "{\"path\":\"apply-diff-tool-fixture/line-hints.js\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:2\\n-------\\nfunction oldFunction() {\\n\\tconsole.log(\\\"Old implementation\\\")\\n}\\n=======\\nfunction newFunction() {\\n\\tconsole.log(\\\"New implementation\\\")\\n}\\n>>>>>>> REPLACE\"}",
40+
"id": "call_apply_diff_line_hints_001"
41+
}
42+
]
43+
}
44+
},
45+
{
46+
"match": {
47+
"userMessage": "APPLY_DIFF_ERROR_SMOKE"
48+
},
49+
"response": {
50+
"toolCalls": [
51+
{
52+
"name": "apply_diff",
53+
"arguments": "{\"path\":\"apply-diff-tool-fixture/error-handling.txt\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nThis content does not exist\\n=======\\nNew content\\n>>>>>>> REPLACE\"}",
54+
"id": "call_apply_diff_error_001"
55+
}
56+
]
57+
}
58+
},
59+
{
60+
"match": {
61+
"userMessage": "APPLY_DIFF_MULTI_BLOCK_SMOKE"
62+
},
63+
"response": {
64+
"toolCalls": [
65+
{
66+
"name": "apply_diff",
67+
"arguments": "{\"path\":\"apply-diff-tool-fixture/multi-search-replace.js\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nfunction processData(data) {\\n\\tconsole.log(\\\"Processing data\\\")\\n=======\\nfunction transformData(data) {\\n\\tconsole.log(\\\"Transforming data\\\")\\n>>>>>>> REPLACE\\n\\n<<<<<<< SEARCH\\n:start_line:12\\n-------\\nfunction validateInput(input) {\\n\\tconsole.log(\\\"Validating input\\\")\\n=======\\nfunction checkInput(input) {\\n\\tconsole.log(\\\"Checking input\\\")\\n>>>>>>> REPLACE\"}",
68+
"id": "call_apply_diff_multi_block_001"
69+
}
70+
]
71+
}
72+
}
73+
]
74+
}

‎apps/vscode-e2e/fixtures/execute-command.json‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "EXECUTE_COMMAND_SIMPLE_SMOKE"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "execute_command",
11+
"arguments": "{\"command\":\"printf 'Hello from test\\\\n' > execute-command-tool-fixture/simple-echo.txt\"}",
12+
"id": "call_execute_command_simple_001"
13+
}
14+
]
15+
}
16+
},
17+
{
18+
"match": {
19+
"userMessage": "EXECUTE_COMMAND_CWD_SMOKE"
20+
},
21+
"response": {
22+
"toolCalls": [
23+
{
24+
"name": "execute_command",
25+
"arguments": "{\"command\":\"printf 'Test in subdirectory\\\\n' > output.txt\",\"cwd\":\"execute-command-tool-fixture/custom-cwd\"}",
26+
"id": "call_execute_command_cwd_001"
27+
}
28+
]
29+
}
30+
},
31+
{
32+
"match": {
33+
"userMessage": "EXECUTE_COMMAND_MULTI_SMOKE"
34+
},
35+
"response": {
36+
"toolCalls": [
37+
{
38+
"name": "execute_command",
39+
"arguments": "{\"command\":\"printf 'Line 1\\\\n' > execute-command-tool-fixture/multi-command.txt\"}",
40+
"id": "call_execute_command_multi_001"
41+
}
42+
]
43+
}
44+
},
45+
{
46+
"match": {
47+
"userMessage": "EXECUTE_COMMAND_LONG_RUNNING_SMOKE"
48+
},
49+
"response": {
50+
"toolCalls": [
51+
{
52+
"name": "execute_command",
53+
"arguments": "{\"command\":\"node -e \\\"setTimeout(() => console.log('Command completed after delay'), 1000)\\\"\"}",
54+
"id": "call_execute_command_long_running_001"
55+
}
56+
]
57+
}
58+
}
59+
]
60+
}

‎apps/vscode-e2e/fixtures/write-to-file.json‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "WRITE_TO_FILE_CREATE_SMOKE"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "write_to_file",
11+
"arguments": "{\"path\":\"write-to-file-tool-fixture/write-to-file-smoke.txt\",\"content\":\"Hello, this is a test file!\"}",
12+
"id": "call_write_to_file_create_001"
13+
}
14+
]
15+
}
16+
},
17+
{
18+
"match": {
19+
"userMessage": "WRITE_TO_FILE_NESTED_SMOKE"
20+
},
21+
"response": {
22+
"toolCalls": [
23+
{
24+
"name": "write_to_file",
25+
"arguments": "{\"path\":\"write-to-file-tool-fixture/nested/deep/directory/write-to-file-nested-smoke.txt\",\"content\":\"File in nested directory\"}",
26+
"id": "call_write_to_file_nested_001"
27+
}
28+
]
29+
}
30+
}
31+
]
32+
}

‎apps/vscode-e2e/src/fixtures/apply-diff.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import { LLMock } from "@copilotkit/aimock"
2+
3+
type ApplyDiffFixture = {
4+
toolCallId: string
5+
result: string
6+
id: string
7+
}
8+
9+
export function addApplyDiffResultFixtures(mock: InstanceType<typeof LLMock>) {
10+
const fixtures: ApplyDiffFixture[] = [
11+
{
12+
toolCallId: "call_apply_diff_simple_001",
13+
result: "Updated `apply-diff-tool-fixture/simple-modify.txt` to say `Hello Universe`.",
14+
id: "call_apply_diff_simple_002",
15+
},
16+
{
17+
toolCallId: "call_apply_diff_multi_replace_001",
18+
result: "Updated `apply-diff-tool-fixture/multiple-replace.js` with the renamed function, parameters, and return fields.",
19+
id: "call_apply_diff_multi_replace_002",
20+
},
21+
{
22+
toolCallId: "call_apply_diff_line_hints_001",
23+
result: "Updated `apply-diff-tool-fixture/line-hints.js` so `oldFunction` became `newFunction` with the new log message.",
24+
id: "call_apply_diff_line_hints_002",
25+
},
26+
{
27+
toolCallId: "call_apply_diff_error_001",
28+
result: "Attempted `apply_diff` on `apply-diff-tool-fixture/error-handling.txt`, but the search text was not found so the file was left unchanged.",
29+
id: "call_apply_diff_error_002",
30+
},
31+
{
32+
toolCallId: "call_apply_diff_multi_block_001",
33+
result: "Applied both search/replace blocks in `apply-diff-tool-fixture/multi-search-replace.js` to rename the two target functions.",
34+
id: "call_apply_diff_multi_block_002",
35+
},
36+
]
37+
38+
for (const fixture of fixtures) {
39+
mock.addFixture({
40+
match: {
41+
toolCallId: fixture.toolCallId,
42+
},
43+
response: {
44+
toolCalls: [
45+
{
46+
name: "attempt_completion",
47+
arguments: JSON.stringify({ result: fixture.result }),
48+
id: fixture.id,
49+
},
50+
],
51+
},
52+
})
53+
}
54+
}

‎apps/vscode-e2e/src/fixtures/execute-command.ts‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import { LLMock } from "@copilotkit/aimock"
2+
3+
type ExecuteCommandToolCall = {
4+
name: "execute_command" | "attempt_completion"
5+
params: Record<string, unknown>
6+
id: string
7+
}
8+
9+
type ExecuteCommandFixture = {
10+
toolCallId: string
11+
toolCalls: ExecuteCommandToolCall[]
12+
}
13+
14+
export function addExecuteCommandResultFixtures(mock: InstanceType<typeof LLMock>) {
15+
const fixtures: ExecuteCommandFixture[] = [
16+
{
17+
toolCallId: "call_execute_command_simple_001",
18+
toolCalls: [
19+
{
20+
name: "attempt_completion",
21+
params: {
22+
result: "Ran the echo command and created `execute-command-tool-fixture/simple-echo.txt`.",
23+
},
24+
id: "call_execute_command_simple_002",
25+
},
26+
],
27+
},
28+
{
29+
toolCallId: "call_execute_command_cwd_001",
30+
toolCalls: [
31+
{
32+
name: "attempt_completion",
33+
params: {
34+
result: "Ran the command inside `execute-command-tool-fixture/custom-cwd` and created `output.txt`.",
35+
},
36+
id: "call_execute_command_cwd_002",
37+
},
38+
],
39+
},
40+
{
41+
toolCallId: "call_execute_command_multi_001",
42+
toolCalls: [
43+
{
44+
name: "execute_command",
45+
params: {
46+
command: "printf 'Line 2\\n' >> execute-command-tool-fixture/multi-command.txt",
47+
},
48+
id: "call_execute_command_multi_002",
49+
},
50+
],
51+
},
52+
{
53+
toolCallId: "call_execute_command_multi_002",
54+
toolCalls: [
55+
{
56+
name: "attempt_completion",
57+
params: {
58+
result: "Ran both commands and populated `execute-command-tool-fixture/multi-command.txt` with two lines.",
59+
},
60+
id: "call_execute_command_multi_003",
61+
},
62+
],
63+
},
64+
{
65+
toolCallId: "call_execute_command_long_running_001",
66+
toolCalls: [
67+
{
68+
name: "attempt_completion",
69+
params: {
70+
result: "The delayed command completed and printed `Command completed after delay`.",
71+
},
72+
id: "call_execute_command_long_running_002",
73+
},
74+
],
75+
},
76+
]
77+
78+
for (const fixture of fixtures) {
79+
mock.addFixture({
80+
match: {
81+
toolCallId: fixture.toolCallId,
82+
},
83+
response: {
84+
toolCalls: fixture.toolCalls.map((toolCall) => ({
85+
name: toolCall.name,
86+
arguments: JSON.stringify(toolCall.params),
87+
id: toolCall.id,
88+
})),
89+
},
90+
})
91+
}
92+
}

‎apps/vscode-e2e/src/fixtures/write-to-file.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import { LLMock } from "@copilotkit/aimock"
2+
3+
type WriteToFileFixture = {
4+
toolCallId: string
5+
result: string
6+
id: string
7+
}
8+
9+
export function addWriteToFileResultFixtures(mock: InstanceType<typeof LLMock>) {
10+
const fixtures: WriteToFileFixture[] = [
11+
{
12+
toolCallId: "call_write_to_file_create_001",
13+
result: "Created `write-to-file-tool-fixture/write-to-file-smoke.txt` with the requested content.",
14+
id: "call_write_to_file_create_002",
15+
},
16+
{
17+
toolCallId: "call_write_to_file_nested_001",
18+
result: "Created `write-to-file-tool-fixture/nested/deep/directory/write-to-file-nested-smoke.txt` with the requested content.",
19+
id: "call_write_to_file_nested_002",
20+
},
21+
]
22+
23+
for (const fixture of fixtures) {
24+
mock.addFixture({
25+
match: {
26+
toolCallId: fixture.toolCallId,
27+
},
28+
response: {
29+
toolCalls: [
30+
{
31+
name: "attempt_completion",
32+
arguments: JSON.stringify({ result: fixture.result }),
33+
id: fixture.id,
34+
},
35+
],
36+
},
37+
})
38+
}
39+
}

‎apps/vscode-e2e/src/runTest.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,12 @@ import * as fs from "fs/promises"
55
import { runTests } from "@vscode/test-electron"
66
import { LLMock } from "@copilotkit/aimock"
77

8+
import { addApplyDiffResultFixtures } from "./fixtures/apply-diff"
9+
import { addExecuteCommandResultFixtures } from "./fixtures/execute-command"
810
import { addListFilesResultFixtures } from "./fixtures/list-files"
911
import { addReadFileResultFixtures } from "./fixtures/read-file"
1012
import { addSearchFilesResultFixtures } from "./fixtures/search-files"
13+
import { addWriteToFileResultFixtures } from "./fixtures/write-to-file"
1114

1215
async function main() {
1316
const isRecord = process.env.AIMOCK_RECORD === "true"
@@ -59,9 +62,12 @@ async function main() {
5962
mock.loadFixtureDir(fixturesDir)
6063

6164
if (!isRecord) {
65+
addApplyDiffResultFixtures(mock)
66+
addExecuteCommandResultFixtures(mock)
6267
addListFilesResultFixtures(mock)
6368
addReadFileResultFixtures(mock)
6469
addSearchFilesResultFixtures(mock)
70+
addWriteToFileResultFixtures(mock)
6571

6672
// The modes test (switch_mode → ask) triggers a second API call whose last
6773
// user message starts with <environment_details> directly — no <user_message>

0 commit comments

Comments
 (0)