Skip to content

Commit fd6da8a

Browse files
roomote[bot]edelaunaroomote
authored
[Chore] Unskip VS Code e2e replay for mutating tools (#92)
* test(read-file): unskipping read-file tests * test(list-files,search-files): unskip read-only e2e tools * test: unskip mutating vscode e2e tool suites * test(e2e): replace opaque smoke codes with explicit instructions * test(e2e): address review feedback on readonly tool fixtures * test(list-files,search-files): unskip read-only e2e tools * test(e2e): replace opaque smoke codes with explicit instructions * test(e2e): address review feedback on readonly tool fixtures * test(e2e): address search-files review feedback * test(e2e): address mutating tool review feedback * test: validate mutating replay payload fixtures * test: drop read-file fixture whitespace churn --------- Co-authored-by: Elliott de Launay <edelauna@gmail.com> Co-authored-by: Roomote <roomote@roocode.com>
1 parent 4a8e5f2 commit fd6da8a

14 files changed

Lines changed: 904 additions & 1386 deletions

‎apps/vscode-e2e/fixtures/apply-diff.json‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "APPLY_DIFF_SIMPLE_SMOKE"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "apply_diff",
11+
"arguments": "{\"path\":\"apply-diff-tool-fixture/simple-modify.txt\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nHello World\\n=======\\nHello Universe\\n>>>>>>> REPLACE\"}",
12+
"id": "call_apply_diff_simple_001"
13+
}
14+
]
15+
}
16+
},
17+
{
18+
"match": {
19+
"userMessage": "APPLY_DIFF_MULTI_REPLACE_SMOKE"
20+
},
21+
"response": {
22+
"toolCalls": [
23+
{
24+
"name": "apply_diff",
25+
"arguments": "{\"path\":\"apply-diff-tool-fixture/multiple-replace.js\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nfunction calculate(x, y) {\\n\\tconst sum = x + y\\n\\tconst product = x * y\\n\\treturn { sum: sum, product: product }\\n}\\n=======\\nfunction compute(a, b) {\\n\\tconst total = a + b\\n\\tconst result = a * b\\n\\treturn { total: total, result: result }\\n}\\n>>>>>>> REPLACE\"}",
26+
"id": "call_apply_diff_multi_replace_001"
27+
}
28+
]
29+
}
30+
},
31+
{
32+
"match": {
33+
"userMessage": "APPLY_DIFF_LINE_HINTS_SMOKE"
34+
},
35+
"response": {
36+
"toolCalls": [
37+
{
38+
"name": "apply_diff",
39+
"arguments": "{\"path\":\"apply-diff-tool-fixture/line-hints.js\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:2\\n-------\\nfunction oldFunction() {\\n\\tconsole.log(\\\"Old implementation\\\")\\n}\\n=======\\nfunction newFunction() {\\n\\tconsole.log(\\\"New implementation\\\")\\n}\\n>>>>>>> REPLACE\"}",
40+
"id": "call_apply_diff_line_hints_001"
41+
}
42+
]
43+
}
44+
},
45+
{
46+
"match": {
47+
"userMessage": "APPLY_DIFF_ERROR_SMOKE"
48+
},
49+
"response": {
50+
"toolCalls": [
51+
{
52+
"name": "apply_diff",
53+
"arguments": "{\"path\":\"apply-diff-tool-fixture/error-handling.txt\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nThis content does not exist\\n=======\\nNew content\\n>>>>>>> REPLACE\"}",
54+
"id": "call_apply_diff_error_001"
55+
}
56+
]
57+
}
58+
},
59+
{
60+
"match": {
61+
"userMessage": "APPLY_DIFF_MULTI_BLOCK_SMOKE"
62+
},
63+
"response": {
64+
"toolCalls": [
65+
{
66+
"name": "apply_diff",
67+
"arguments": "{\"path\":\"apply-diff-tool-fixture/multi-search-replace.js\",\"diff\":\"<<<<<<< SEARCH\\n:start_line:1\\n-------\\nfunction processData(data) {\\n\\tconsole.log(\\\"Processing data\\\")\\n=======\\nfunction transformData(data) {\\n\\tconsole.log(\\\"Transforming data\\\")\\n>>>>>>> REPLACE\\n\\n<<<<<<< SEARCH\\n:start_line:12\\n-------\\nfunction validateInput(input) {\\n\\tconsole.log(\\\"Validating input\\\")\\n=======\\nfunction checkInput(input) {\\n\\tconsole.log(\\\"Checking input\\\")\\n>>>>>>> REPLACE\"}",
68+
"id": "call_apply_diff_multi_block_001"
69+
}
70+
]
71+
}
72+
}
73+
]
74+
}

‎apps/vscode-e2e/fixtures/execute-command.json‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "EXECUTE_COMMAND_SIMPLE_SMOKE"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "execute_command",
11+
"arguments": "{\"command\":\"printf 'Hello from test\\\\n' > execute-command-tool-fixture/simple-echo.txt\"}",
12+
"id": "call_execute_command_simple_001"
13+
}
14+
]
15+
}
16+
},
17+
{
18+
"match": {
19+
"userMessage": "EXECUTE_COMMAND_CWD_SMOKE"
20+
},
21+
"response": {
22+
"toolCalls": [
23+
{
24+
"name": "execute_command",
25+
"arguments": "{\"command\":\"printf 'Test in subdirectory\\\\n' > output.txt\",\"cwd\":\"execute-command-tool-fixture/custom-cwd\"}",
26+
"id": "call_execute_command_cwd_001"
27+
}
28+
]
29+
}
30+
},
31+
{
32+
"match": {
33+
"userMessage": "EXECUTE_COMMAND_MULTI_SMOKE"
34+
},
35+
"response": {
36+
"toolCalls": [
37+
{
38+
"name": "execute_command",
39+
"arguments": "{\"command\":\"printf 'Line 1\\\\n' > execute-command-tool-fixture/multi-command.txt\"}",
40+
"id": "call_execute_command_multi_001"
41+
}
42+
]
43+
}
44+
},
45+
{
46+
"match": {
47+
"userMessage": "EXECUTE_COMMAND_LONG_RUNNING_SMOKE"
48+
},
49+
"response": {
50+
"toolCalls": [
51+
{
52+
"name": "execute_command",
53+
"arguments": "{\"command\":\"node -e \\\"setTimeout(() => console.log('Command completed after delay'), 1000)\\\"\"}",
54+
"id": "call_execute_command_long_running_001"
55+
}
56+
]
57+
}
58+
}
59+
]
60+
}

‎apps/vscode-e2e/fixtures/write-to-file.json‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "WRITE_TO_FILE_CREATE_SMOKE"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "write_to_file",
11+
"arguments": "{\"path\":\"write-to-file-tool-fixture/write-to-file-smoke.txt\",\"content\":\"Hello, this is a test file!\"}",
12+
"id": "call_write_to_file_create_001"
13+
}
14+
]
15+
}
16+
},
17+
{
18+
"match": {
19+
"userMessage": "WRITE_TO_FILE_NESTED_SMOKE"
20+
},
21+
"response": {
22+
"toolCalls": [
23+
{
24+
"name": "write_to_file",
25+
"arguments": "{\"path\":\"write-to-file-tool-fixture/nested/deep/directory/write-to-file-nested-smoke.txt\",\"content\":\"File in nested directory\"}",
26+
"id": "call_write_to_file_nested_001"
27+
}
28+
]
29+
}
30+
}
31+
]
32+
}

‎apps/vscode-e2e/src/fixtures/apply-diff.ts‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import { LLMock } from "@copilotkit/aimock"
2+
3+
import { toolResultContains } from "./tool-result"
4+
5+
type ApplyDiffFixture = {
6+
toolCallId: string
7+
expected: string[]
8+
result: string
9+
id: string
10+
}
11+
12+
export function addApplyDiffResultFixtures(mock: InstanceType<typeof LLMock>) {
13+
const fixtures: ApplyDiffFixture[] = [
14+
{
15+
toolCallId: "call_apply_diff_simple_001",
16+
expected: ['"path":"apply-diff-tool-fixture/simple-modify.txt"', '"operation":"modified"'],
17+
result: "Updated `apply-diff-tool-fixture/simple-modify.txt` to say `Hello Universe`.",
18+
id: "call_apply_diff_simple_002",
19+
},
20+
{
21+
toolCallId: "call_apply_diff_multi_replace_001",
22+
expected: ['"path":"apply-diff-tool-fixture/multiple-replace.js"', '"operation":"modified"'],
23+
result: "Updated `apply-diff-tool-fixture/multiple-replace.js` with the renamed function, parameters, and return fields.",
24+
id: "call_apply_diff_multi_replace_002",
25+
},
26+
{
27+
toolCallId: "call_apply_diff_line_hints_001",
28+
expected: ['"path":"apply-diff-tool-fixture/line-hints.js"', '"operation":"modified"'],
29+
result: "Updated `apply-diff-tool-fixture/line-hints.js` so `oldFunction` became `newFunction` with the new log message.",
30+
id: "call_apply_diff_line_hints_002",
31+
},
32+
{
33+
toolCallId: "call_apply_diff_error_001",
34+
expected: ["No sufficiently similar match found at line: 1", "This content does not exist"],
35+
result: "The apply_diff operation on `apply-diff-tool-fixture/error-handling.txt` was rejected - the search content did not match any content in the file, so it was not modified.",
36+
id: "call_apply_diff_error_002",
37+
},
38+
{
39+
toolCallId: "call_apply_diff_multi_block_001",
40+
expected: ['"path":"apply-diff-tool-fixture/multi-search-replace.js"', '"operation":"modified"'],
41+
result: "Applied both search/replace blocks in `apply-diff-tool-fixture/multi-search-replace.js` to rename the two target functions.",
42+
id: "call_apply_diff_multi_block_002",
43+
},
44+
]
45+
46+
for (const fixture of fixtures) {
47+
mock.addFixture({
48+
match: {
49+
toolCallId: fixture.toolCallId,
50+
predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected),
51+
},
52+
response: {
53+
toolCalls: [
54+
{
55+
name: "attempt_completion",
56+
arguments: JSON.stringify({ result: fixture.result }),
57+
id: fixture.id,
58+
},
59+
],
60+
},
61+
})
62+
}
63+
}

‎apps/vscode-e2e/src/fixtures/execute-command.ts‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
import { LLMock } from "@copilotkit/aimock"
2+
3+
import { toolResultContains } from "./tool-result"
4+
5+
type ExecuteCommandToolCall = {
6+
name: "execute_command" | "attempt_completion"
7+
params: Record<string, unknown>
8+
id: string
9+
}
10+
11+
type ExecuteCommandFixture = {
12+
toolCallId: string
13+
expected: string[]
14+
toolCalls: ExecuteCommandToolCall[]
15+
}
16+
17+
export function addExecuteCommandResultFixtures(mock: InstanceType<typeof LLMock>) {
18+
const fixtures: ExecuteCommandFixture[] = [
19+
{
20+
toolCallId: "call_execute_command_simple_001",
21+
expected: ["Command executed in terminal within working directory '", "Exit code: 0\nOutput:\n"],
22+
toolCalls: [
23+
{
24+
name: "attempt_completion",
25+
params: {
26+
result: "Ran the echo command and created `execute-command-tool-fixture/simple-echo.txt`.",
27+
},
28+
id: "call_execute_command_simple_002",
29+
},
30+
],
31+
},
32+
{
33+
toolCallId: "call_execute_command_cwd_001",
34+
expected: ["execute-command-tool-fixture/custom-cwd'. Exit code: 0", "Output:\n"],
35+
toolCalls: [
36+
{
37+
name: "attempt_completion",
38+
params: {
39+
result: "Ran the command inside `execute-command-tool-fixture/custom-cwd` and created `output.txt`.",
40+
},
41+
id: "call_execute_command_cwd_002",
42+
},
43+
],
44+
},
45+
{
46+
toolCallId: "call_execute_command_multi_001",
47+
expected: ["Command executed in terminal within working directory '", "Exit code: 0\nOutput:\n"],
48+
toolCalls: [
49+
{
50+
name: "execute_command",
51+
params: {
52+
command: "printf 'Line 2\\n' >> execute-command-tool-fixture/multi-command.txt",
53+
},
54+
id: "call_execute_command_multi_002",
55+
},
56+
],
57+
},
58+
{
59+
toolCallId: "call_execute_command_multi_002",
60+
expected: ["Command executed in terminal within working directory '", "Exit code: 0\nOutput:\n"],
61+
toolCalls: [
62+
{
63+
name: "attempt_completion",
64+
params: {
65+
result: "Ran both commands and populated `execute-command-tool-fixture/multi-command.txt` with two lines.",
66+
},
67+
id: "call_execute_command_multi_003",
68+
},
69+
],
70+
},
71+
{
72+
toolCallId: "call_execute_command_long_running_001",
73+
expected: ["Exit code: 0", "Command completed after delay"],
74+
toolCalls: [
75+
{
76+
name: "attempt_completion",
77+
params: {
78+
result: "The delayed command completed and printed `Command completed after delay`.",
79+
},
80+
id: "call_execute_command_long_running_002",
81+
},
82+
],
83+
},
84+
]
85+
86+
for (const fixture of fixtures) {
87+
mock.addFixture({
88+
match: {
89+
toolCallId: fixture.toolCallId,
90+
predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected),
91+
},
92+
response: {
93+
toolCalls: fixture.toolCalls.map((toolCall) => ({
94+
name: toolCall.name,
95+
arguments: JSON.stringify(toolCall.params),
96+
id: toolCall.id,
97+
})),
98+
},
99+
})
100+
}
101+
}

‎apps/vscode-e2e/src/fixtures/read-file.ts‎

Lines changed: 6 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
import { LLMock } from "@copilotkit/aimock"
2-
import type { ChatCompletionRequest, ChatMessage } from "@copilotkit/aimock"
32

4-
type ToolResultExpectation = { toolCallId: string; expected: string[] }
3+
import {
4+
isToolResultExpectation,
5+
toolResultContains,
6+
toolResultsContain,
7+
type ToolResultExpectation,
8+
} from "./tool-result"
59

610
type ReadFileResultFixture = {
711
toolCallId: string
@@ -10,28 +14,6 @@ type ReadFileResultFixture = {
1014
id: string
1115
}
1216

13-
function isToolResultExpectation(value: unknown): value is ToolResultExpectation {
14-
return typeof value === "object" && value !== null && "toolCallId" in value && "expected" in value
15-
}
16-
17-
function toolResultContains(req: ChatCompletionRequest, toolCallId: string, expected: string[]) {
18-
const messages = Array.isArray(req?.messages) ? req.messages : []
19-
const toolMessage = messages.find(
20-
(message: ChatMessage) => message?.role === "tool" && message.tool_call_id === toolCallId,
21-
)
22-
23-
const content = toolMessage?.content
24-
if (typeof content !== "string") {
25-
return false
26-
}
27-
28-
return expected.every((text) => content.includes(text))
29-
}
30-
31-
function toolResultsContain(req: ChatCompletionRequest, expectations: ToolResultExpectation[]) {
32-
return expectations.every(({ toolCallId, expected }) => toolResultContains(req, toolCallId, expected))
33-
}
34-
3517
export function addReadFileResultFixtures(mock: InstanceType<typeof LLMock>) {
3618
const fixtures: ReadFileResultFixture[] = [
3719
{

0 commit comments

Comments
 (0)