Skip to content

Commit 4a8e5f2

Browse files
roomote[bot]roomoteedelauna
authored
test(list-files,search-files): unskip read-only e2e tools
* test(list-files,search-files): unskip read-only e2e tools * chore: tighten list_files replay predicates * test(e2e): replace opaque smoke codes with explicit instructions * test(e2e): log say messages when running against real endpoints * test(e2e): restore replay result validation * test(e2e): address review feedback on readonly tool fixtures * fix(e2e): address readonly tool review feedback * test(e2e): address search-files review feedback * test: fix Windows list-files cwd assertions * test: fix Windows list-files expected file path --------- Co-authored-by: Roomote <roomote@roocode.com> Co-authored-by: Elliott de Launay <edelauna@gmail.com>
1 parent 69ef6a3 commit 4a8e5f2

11 files changed

Lines changed: 364 additions & 427 deletions

File tree

‎apps/vscode-e2e/fixtures/task-hello-world.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
"toolCalls": [
99
{
1010
"name": "attempt_completion",
11-
"arguments": "{\"result\":\"My name is Roo! I'm your AI coding assistant, here to help you with development tasks.\"}",
11+
"arguments": "{\"result\":\"My name is Zoo! I'm your AI coding assistant, here to help you with development tasks.\"}",
1212
"id": "call_task_hello_world_001"
1313
}
1414
]

‎apps/vscode-e2e/src/fixtures/fixture-utils.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import type { ChatCompletionRequest, ChatMessage } from "@copilotkit/aimock"
2+
3+
export function toolResultContains(req: ChatCompletionRequest, toolCallId: string, expected: string[]) {
4+
const messages = Array.isArray(req?.messages) ? req.messages : []
5+
const toolMessage = messages.find(
6+
(message: ChatMessage) => message?.role === "tool" && message.tool_call_id === toolCallId,
7+
)
8+
9+
const content = toolMessage?.content
10+
if (typeof content !== "string") {
11+
return false
12+
}
13+
14+
return expected.every((text) => content.includes(text))
15+
}

‎apps/vscode-e2e/src/fixtures/list-files.ts‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import { LLMock } from "@copilotkit/aimock"
2+
3+
import { toolResultContains } from "./fixture-utils"
4+
5+
type ListFilesFixture = {
6+
userMessagePattern: string
7+
toolName: string
8+
arguments: string
9+
toolCallId: string
10+
expected: string[]
11+
result: string
12+
id: string
13+
}
14+
15+
export function addListFilesResultFixtures(mock: InstanceType<typeof LLMock>) {
16+
const fixtures: ListFilesFixture[] = [
17+
{
18+
userMessagePattern: "without recursing into subdirectories",
19+
toolName: "list_files",
20+
arguments: '{"path":"list-files-tool-fixture","recursive":false}',
21+
toolCallId: "call_list_files_non_recursive_001",
22+
expected: ["root-file-1.txt", ".hidden-file", "nested/"],
23+
result: "The non-recursive listing for `list-files-tool-fixture` includes `root-file-1.txt`, `root-file-2.js`, `config.yaml`, `README.md`, `.hidden-file`, and the `nested/` directory.",
24+
id: "call_list_files_non_recursive_002",
25+
},
26+
{
27+
userMessagePattern: "deep-nested-file.ts is included",
28+
toolName: "list_files",
29+
arguments: '{"path":"list-files-tool-fixture","recursive":true}',
30+
toolCallId: "call_list_files_recursive_001",
31+
expected: ["nested/", "nested/deep/", "deep-nested-file.ts"],
32+
result: "The recursive listing for `list-files-tool-fixture` reached the nested structure and includes `nested/`, `nested/deep/`, and `deep-nested-file.ts`.",
33+
id: "call_list_files_recursive_002",
34+
},
35+
{
36+
userMessagePattern: "path='list-files-symlink-fixture'",
37+
toolName: "list_files",
38+
arguments: '{"path":"list-files-symlink-fixture","recursive":false}',
39+
toolCallId: "call_list_files_symlink_001",
40+
expected: ["link-to-file.txt", "source/"],
41+
result: "The symlink fixture listing shows the original `source/` directory and its `source-file.txt`, plus the symlink entry `link-to-file.txt` in `list-files-symlink-fixture`.",
42+
id: "call_list_files_symlink_002",
43+
},
44+
{
45+
userMessagePattern: "confirm whether list-files-tool-fixture or list-files-symlink-fixture is present",
46+
toolName: "list_files",
47+
arguments: '{"path":".","recursive":false}',
48+
toolCallId: "call_list_files_workspace_root_001",
49+
expected: ["list-files-tool-fixture/"],
50+
result: "The workspace root currently contains the `list-files-tool-fixture/` and `list-files-symlink-fixture/` test directories.",
51+
id: "call_list_files_workspace_root_002",
52+
},
53+
]
54+
55+
for (const fixture of fixtures) {
56+
mock.addFixture({
57+
match: {
58+
userMessage: new RegExp(fixture.userMessagePattern),
59+
},
60+
response: {
61+
toolCalls: [
62+
{
63+
name: fixture.toolName,
64+
arguments: fixture.arguments,
65+
id: fixture.toolCallId,
66+
},
67+
],
68+
},
69+
})
70+
71+
mock.addFixture({
72+
match: {
73+
toolCallId: fixture.toolCallId,
74+
predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected),
75+
},
76+
response: {
77+
toolCalls: [
78+
{
79+
name: "attempt_completion",
80+
arguments: JSON.stringify({ result: fixture.result }),
81+
id: fixture.id,
82+
},
83+
],
84+
},
85+
})
86+
}
87+
}

‎apps/vscode-e2e/src/fixtures/search-files.ts‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
import { LLMock } from "@copilotkit/aimock"
2+
3+
import { toolResultContains } from "./fixture-utils"
4+
5+
type SearchFilesFixture = {
6+
userMessagePattern: string
7+
toolName: string
8+
arguments: string
9+
toolCallId: string
10+
expected: string[]
11+
result: string
12+
id: string
13+
}
14+
15+
export function addSearchFilesResultFixtures(mock: InstanceType<typeof LLMock>) {
16+
const fixtures: SearchFilesFixture[] = [
17+
{
18+
userMessagePattern: "JavaScript function declarations",
19+
toolName: "search_files",
20+
arguments: '{"path":"search-files-tool-fixture","regex":"function\\\\s+\\\\w+"}',
21+
toolCallId: "call_search_files_functions_001",
22+
expected: [
23+
"# search-files-tool-fixture/search-fixture.js",
24+
"function calculateTotal(items) {",
25+
"function validateUser(user) {",
26+
],
27+
result: "The function search found declarations including `calculateTotal`, `validateUser`, and `formatCurrency`.",
28+
id: "call_search_files_functions_002",
29+
},
30+
{
31+
userMessagePattern: "TODO comments using the regex TODO",
32+
toolName: "search_files",
33+
arguments: '{"path":"search-files-tool-fixture","regex":"TODO.*"}',
34+
toolCallId: "call_search_files_todo_001",
35+
expected: [
36+
"# search-files-tool-fixture/search-fixture.js",
37+
"// TODO: Add more validation functions",
38+
"// TODO: Implement user fetching",
39+
],
40+
result: "The TODO search found matching TODO entries in the fixture files, including the validation and user-fetching notes.",
41+
id: "call_search_files_todo_002",
42+
},
43+
{
44+
userMessagePattern: "TypeScript interfaces you find",
45+
toolName: "search_files",
46+
arguments: '{"path":"search-files-tool-fixture","regex":"interface\\\\s+\\\\w+","file_pattern":"*.ts"}',
47+
toolCallId: "call_search_files_typescript_001",
48+
expected: ["# search-files-tool-fixture/search-fixture.ts", "interface User {", "interface Product {"],
49+
result: "The TypeScript-only search found the `User` and `Product` interface definitions.",
50+
id: "call_search_files_typescript_002",
51+
},
52+
{
53+
userMessagePattern: "JSON configuration keys",
54+
toolName: "search_files",
55+
arguments: '{"path":"search-files-tool-fixture","regex":"\\"\\\\w+\\":\\\\s*","file_pattern":"*.json"}',
56+
toolCallId: "call_search_files_json_001",
57+
expected: ["# search-files-tool-fixture/search-config.json", '"name": "test-app",', '"dependencies": {'],
58+
result: "The JSON search found configuration keys such as `name`, `version`, and `dependencies` in `search-config.json`.",
59+
id: "call_search_files_json_002",
60+
},
61+
{
62+
userMessagePattern: "formatCurrency and debounce",
63+
toolName: "search_files",
64+
arguments: '{"path":"search-files-tool-fixture","regex":"function\\\\s+(format|debounce)"}',
65+
toolCallId: "call_search_files_nested_001",
66+
expected: [
67+
"# search-files-tool-fixture/nested/nested-search.js",
68+
"function formatCurrency(amount) {",
69+
"function debounce(func, wait) {",
70+
],
71+
result: "The nested-directory search found the utility functions `formatCurrency` and `debounce`.",
72+
id: "call_search_files_nested_002",
73+
},
74+
{
75+
userMessagePattern: "import and export statements",
76+
toolName: "search_files",
77+
arguments: '{"path":"search-files-tool-fixture","regex":"(import|export).*","file_pattern":"*.{js,ts}"}',
78+
toolCallId: "call_search_files_complex_regex_001",
79+
expected: [
80+
"# search-files-tool-fixture/search-fixture.js",
81+
"export { calculateTotal, validateUser }",
82+
"module.exports = { formatCurrency, debounce }",
83+
],
84+
result: "The import/export search found the `export` statement in the JavaScript fixture module.",
85+
id: "call_search_files_complex_regex_002",
86+
},
87+
{
88+
userMessagePattern: "nonExistentPattern12345 and report that there are no matches",
89+
toolName: "search_files",
90+
arguments: '{"path":"search-files-tool-fixture","regex":"nonExistentPattern12345"}',
91+
toolCallId: "call_search_files_no_match_001",
92+
expected: ["No results found"],
93+
result: "No matches were found for `nonExistentPattern12345` in the search fixture directory.",
94+
id: "call_search_files_no_match_002",
95+
},
96+
{
97+
userMessagePattern: "TypeScript class definitions and async methods",
98+
toolName: "search_files",
99+
arguments:
100+
'{"path":"search-files-tool-fixture","regex":"(class\\\\s+\\\\w+|async\\\\s+\\\\w+)","file_pattern":"*.ts"}',
101+
toolCallId: "call_search_files_class_method_001",
102+
expected: [
103+
"# search-files-tool-fixture/search-fixture.ts",
104+
"class UserService {",
105+
"async getUser(id: number): Promise<User> {",
106+
],
107+
result: "The class-and-method search found `UserService` and its async `getUser` method in the TypeScript fixture.",
108+
id: "call_search_files_class_method_002",
109+
},
110+
]
111+
112+
for (const fixture of fixtures) {
113+
mock.addFixture({
114+
match: {
115+
userMessage: new RegExp(fixture.userMessagePattern),
116+
},
117+
response: {
118+
toolCalls: [
119+
{
120+
name: fixture.toolName,
121+
arguments: fixture.arguments,
122+
id: fixture.toolCallId,
123+
},
124+
],
125+
},
126+
})
127+
128+
mock.addFixture({
129+
match: {
130+
toolCallId: fixture.toolCallId,
131+
predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected),
132+
},
133+
response: {
134+
toolCalls: [
135+
{
136+
name: "attempt_completion",
137+
arguments: JSON.stringify({ result: fixture.result }),
138+
id: fixture.id,
139+
},
140+
],
141+
},
142+
})
143+
}
144+
}

‎apps/vscode-e2e/src/runTest.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ import * as fs from "fs/promises"
55
import { runTests } from "@vscode/test-electron"
66
import { LLMock } from "@copilotkit/aimock"
77

8+
import { addListFilesResultFixtures } from "./fixtures/list-files"
89
import { addReadFileResultFixtures } from "./fixtures/read-file"
10+
import { addSearchFilesResultFixtures } from "./fixtures/search-files"
911

1012
function getCliFlagValue(flag: string) {
1113
return process.argv.find((arg, index) => process.argv[index - 1] === flag)
@@ -77,7 +79,9 @@ async function main() {
7779
mock.loadFixtureDir(fixturesDir)
7880

7981
if (!isRecord) {
82+
addListFilesResultFixtures(mock)
8083
addReadFileResultFixtures(mock)
84+
addSearchFilesResultFixtures(mock)
8185

8286
// The modes test (switch_mode → ask) triggers a second API call whose last
8387
// user message starts with <environment_details> directly — no <user_message>

‎apps/vscode-e2e/src/suite/index.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import Mocha from "mocha"
33
import { glob } from "glob"
44
import * as vscode from "vscode"
55

6-
import type { RooCodeAPI, RooCodeEventName } from "@roo-code/types"
6+
import { RooCodeEventName, type RooCodeAPI } from "@roo-code/types"
77

88
import { waitFor } from "./utils"
99

@@ -33,12 +33,20 @@ export async function run() {
3333

3434
// Automatically approve completion_result asks so tests don't stall waiting
3535
// for a button that the webview routes to "start new task" rather than "yes".
36-
api.on("message" as RooCodeEventName.Message, ({ message }) => {
36+
api.on(RooCodeEventName.Message, ({ message }) => {
3737
if (message.type === "ask" && message.ask === "completion_result") {
3838
api.approveCurrentAsk()
3939
}
4040
})
4141

42+
if (!aimockUrl) {
43+
api.on(RooCodeEventName.Message, ({ message }) => {
44+
if (message.type === "say" && !message.partial) {
45+
console.log(`[say:${message.say}]`, message.text?.slice(0, 300))
46+
}
47+
})
48+
}
49+
4250
globalThis.api = api
4351

4452
const mochaOptions: Mocha.MochaOptions = {

‎apps/vscode-e2e/src/suite/task.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,9 @@ suite("Roo Code Task", function () {
2828

2929
assert.ok(
3030
!!messages.find(
31-
({ say, text }) => (say === "completion_result" || say === "text") && text?.includes("My name is Roo"),
31+
({ say, text }) => (say === "completion_result" || say === "text") && text?.includes("My name is Zoo"),
3232
),
33-
`Completion should include "My name is Roo"`,
33+
`Completion should include "My name is Zoo"`,
3434
)
3535
})
3636
})

0 commit comments

Comments
 (0)