Skip to content

Commit 7de61e6

Browse files
roomote[bot]roomoteedelauna
authored
[Chore] Unskip VS Code e2e replay for use_mcp_tool (#93)
* test(e2e): unskip use_mcp_tool replay coverage * test(e2e): relax mcp completion wording checks * test(e2e): use real MCP prompts in use_mcp_tool suite * test(e2e): unskip use_mcp_tool replay coverage with local MCP server --------- Co-authored-by: Roomote <roomote@roocode.com> Co-authored-by: Elliott de Launay <edelauna@gmail.com>
1 parent 2e8e72e commit 7de61e6

6 files changed

Lines changed: 779 additions & 845 deletions

File tree

‎apps/vscode-e2e/fixtures/list-files.json‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "LIST_FILES_NON_RECURSIVE_SMOKE"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "list_files",
11+
"arguments": "{\"path\":\"list-files-tool-fixture\",\"recursive\":false}",
12+
"id": "call_list_files_non_recursive_001"
13+
}
14+
]
15+
}
16+
},
17+
{
18+
"match": {
19+
"userMessage": "LIST_FILES_RECURSIVE_SMOKE"
20+
},
21+
"response": {
22+
"toolCalls": [
23+
{
24+
"name": "list_files",
25+
"arguments": "{\"path\":\"list-files-tool-fixture\",\"recursive\":true}",
26+
"id": "call_list_files_recursive_001"
27+
}
28+
]
29+
}
30+
},
31+
{
32+
"match": {
33+
"userMessage": "LIST_FILES_SYMLINK_SMOKE"
34+
},
35+
"response": {
36+
"toolCalls": [
37+
{
38+
"name": "list_files",
39+
"arguments": "{\"path\":\"list-files-symlink-fixture\",\"recursive\":false}",
40+
"id": "call_list_files_symlink_001"
41+
}
42+
]
43+
}
44+
},
45+
{
46+
"match": {
47+
"userMessage": "LIST_FILES_WORKSPACE_ROOT_SMOKE"
48+
},
49+
"response": {
50+
"toolCalls": [
51+
{
52+
"name": "list_files",
53+
"arguments": "{\"path\":\".\",\"recursive\":false}",
54+
"id": "call_list_files_workspace_root_001"
55+
}
56+
]
57+
}
58+
}
59+
]
60+
}

‎apps/vscode-e2e/fixtures/search-files.json‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "SEARCH_FILES_FUNCTIONS_SMOKE"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "search_files",
11+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"function\\\\s+\\\\w+\"}",
12+
"id": "call_search_files_functions_001"
13+
}
14+
]
15+
}
16+
},
17+
{
18+
"match": {
19+
"userMessage": "SEARCH_FILES_TODO_SMOKE"
20+
},
21+
"response": {
22+
"toolCalls": [
23+
{
24+
"name": "search_files",
25+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"TODO.*\"}",
26+
"id": "call_search_files_todo_001"
27+
}
28+
]
29+
}
30+
},
31+
{
32+
"match": {
33+
"userMessage": "SEARCH_FILES_TYPESCRIPT_SMOKE"
34+
},
35+
"response": {
36+
"toolCalls": [
37+
{
38+
"name": "search_files",
39+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"interface\\\\s+\\\\w+\",\"file_pattern\":\"*.ts\"}",
40+
"id": "call_search_files_typescript_001"
41+
}
42+
]
43+
}
44+
},
45+
{
46+
"match": {
47+
"userMessage": "SEARCH_FILES_JSON_SMOKE"
48+
},
49+
"response": {
50+
"toolCalls": [
51+
{
52+
"name": "search_files",
53+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"\\\"\\\\w+\\\":\\\\s*\",\"file_pattern\":\"*.json\"}",
54+
"id": "call_search_files_json_001"
55+
}
56+
]
57+
}
58+
},
59+
{
60+
"match": {
61+
"userMessage": "SEARCH_FILES_NESTED_SMOKE"
62+
},
63+
"response": {
64+
"toolCalls": [
65+
{
66+
"name": "search_files",
67+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"function\\\\s+(format|debounce)\"}",
68+
"id": "call_search_files_nested_001"
69+
}
70+
]
71+
}
72+
},
73+
{
74+
"match": {
75+
"userMessage": "SEARCH_FILES_COMPLEX_REGEX_SMOKE"
76+
},
77+
"response": {
78+
"toolCalls": [
79+
{
80+
"name": "search_files",
81+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"(import|export).*\",\"file_pattern\":\"*.{js,ts}\"}",
82+
"id": "call_search_files_complex_regex_001"
83+
}
84+
]
85+
}
86+
},
87+
{
88+
"match": {
89+
"userMessage": "SEARCH_FILES_NO_MATCH_SMOKE"
90+
},
91+
"response": {
92+
"toolCalls": [
93+
{
94+
"name": "search_files",
95+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"nonExistentPattern12345\"}",
96+
"id": "call_search_files_no_match_001"
97+
}
98+
]
99+
}
100+
},
101+
{
102+
"match": {
103+
"userMessage": "SEARCH_FILES_CLASS_METHOD_SMOKE"
104+
},
105+
"response": {
106+
"toolCalls": [
107+
{
108+
"name": "search_files",
109+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"(class\\\\s+\\\\w+|async\\\\s+\\\\w+)\",\"file_pattern\":\"*.ts\"}",
110+
"id": "call_search_files_class_method_001"
111+
}
112+
]
113+
}
114+
}
115+
]
116+
}

‎apps/vscode-e2e/src/fixtures/use-mcp-tool.ts‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
import { LLMock } from "@copilotkit/aimock"
2+
3+
const TEST_DIR_NAME = "use-mcp-tool-fixture"
4+
const FILESYSTEM_SERVER_NAME = "filesystem"
5+
const READ_FILE_RELATIVE_PATH = `${TEST_DIR_NAME}/mcp-read-target.txt`
6+
const WRITE_FILE_RELATIVE_PATH = `${TEST_DIR_NAME}/mcp-write-target.txt`
7+
8+
type UseMcpToolFixture = {
9+
userMessagePattern: string
10+
toolCallId: string
11+
toolName: string
12+
toolArguments?: Record<string, unknown>
13+
serverName?: string
14+
result: string
15+
id: string
16+
}
17+
18+
export function addUseMcpToolResultFixtures(mock: InstanceType<typeof LLMock>) {
19+
const fixtures: UseMcpToolFixture[] = [
20+
{
21+
userMessagePattern: "USE_MCP_TOOL_READ_FILE_SMOKE",
22+
toolCallId: "call_use_mcp_tool_read_file_001",
23+
toolName: "read_file",
24+
toolArguments: { path: READ_FILE_RELATIVE_PATH },
25+
result: "Read the requested file through the MCP filesystem server.",
26+
id: "call_use_mcp_tool_read_file_002",
27+
},
28+
{
29+
userMessagePattern: "USE_MCP_TOOL_WRITE_FILE_SMOKE",
30+
toolCallId: "call_use_mcp_tool_write_file_001",
31+
toolName: "write_file",
32+
toolArguments: { path: WRITE_FILE_RELATIVE_PATH, content: "Hello from MCP!" },
33+
result: "Created the requested file through the MCP filesystem server.",
34+
id: "call_use_mcp_tool_write_file_002",
35+
},
36+
{
37+
userMessagePattern: "USE_MCP_TOOL_LIST_DIRECTORY_SMOKE",
38+
toolCallId: "call_use_mcp_tool_list_directory_001",
39+
toolName: "list_directory",
40+
toolArguments: { path: TEST_DIR_NAME },
41+
result: "Listed the requested directory through the MCP filesystem server.",
42+
id: "call_use_mcp_tool_list_directory_002",
43+
},
44+
{
45+
userMessagePattern: "USE_MCP_TOOL_DIRECTORY_TREE_SMOKE",
46+
toolCallId: "call_use_mcp_tool_directory_tree_001",
47+
toolName: "directory_tree",
48+
toolArguments: { path: TEST_DIR_NAME },
49+
result: "Returned the directory tree through the MCP filesystem server.",
50+
id: "call_use_mcp_tool_directory_tree_002",
51+
},
52+
{
53+
userMessagePattern: "USE_MCP_TOOL_GET_FILE_INFO_SMOKE",
54+
toolCallId: "call_use_mcp_tool_get_file_info_001",
55+
toolName: "get_file_info",
56+
toolArguments: { path: READ_FILE_RELATIVE_PATH },
57+
result: "Returned the requested file metadata through the MCP filesystem server.",
58+
id: "call_use_mcp_tool_get_file_info_002",
59+
},
60+
{
61+
userMessagePattern: "USE_MCP_TOOL_UNKNOWN_SERVER_SMOKE",
62+
toolCallId: "call_use_mcp_tool_unknown_server_001",
63+
serverName: "nonexistent-server",
64+
toolName: "read_file",
65+
toolArguments: { path: READ_FILE_RELATIVE_PATH },
66+
result: "MCP server 'nonexistent-server' is not configured. Available servers: filesystem",
67+
id: "call_use_mcp_tool_unknown_server_002",
68+
},
69+
]
70+
71+
for (const fixture of fixtures) {
72+
const serverName = fixture.serverName ?? FILESYSTEM_SERVER_NAME
73+
const isConfiguredFilesystemTool = serverName === FILESYSTEM_SERVER_NAME
74+
75+
mock.addFixture({
76+
match: {
77+
userMessage: new RegExp(fixture.userMessagePattern),
78+
},
79+
response: {
80+
toolCalls: [
81+
{
82+
name: isConfiguredFilesystemTool
83+
? `mcp--${FILESYSTEM_SERVER_NAME}--${fixture.toolName}`
84+
: "use_mcp_tool",
85+
arguments: JSON.stringify(
86+
isConfiguredFilesystemTool
87+
? fixture.toolArguments
88+
: {
89+
server_name: serverName,
90+
tool_name: fixture.toolName,
91+
arguments: fixture.toolArguments,
92+
},
93+
),
94+
id: fixture.toolCallId,
95+
},
96+
],
97+
},
98+
})
99+
100+
mock.addFixture({
101+
match: {
102+
toolCallId: fixture.toolCallId,
103+
},
104+
response: {
105+
toolCalls: [
106+
{
107+
name: "attempt_completion",
108+
arguments: JSON.stringify({ result: fixture.result }),
109+
id: fixture.id,
110+
},
111+
],
112+
},
113+
})
114+
}
115+
}

‎apps/vscode-e2e/src/runTest.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { addExecuteCommandResultFixtures } from "./fixtures/execute-command"
1010
import { addListFilesResultFixtures } from "./fixtures/list-files"
1111
import { addReadFileResultFixtures } from "./fixtures/read-file"
1212
import { addSearchFilesResultFixtures } from "./fixtures/search-files"
13+
import { addUseMcpToolResultFixtures } from "./fixtures/use-mcp-tool"
1314
import { addWriteToFileResultFixtures } from "./fixtures/write-to-file"
1415

1516
function getCliFlagValue(flag: string) {
@@ -59,6 +60,10 @@ async function main() {
5960
let testWorkspace: string | undefined
6061

6162
try {
63+
// Create a temporary workspace folder for tests before installing fixtures that
64+
// need workspace-specific paths.
65+
testWorkspace = await fs.mkdtemp(path.join(os.tmpdir(), "roo-test-workspace-"))
66+
6267
if (useMock) {
6368
const fixturesDir = path.resolve(__dirname, "../fixtures")
6469

@@ -87,6 +92,7 @@ async function main() {
8792
addListFilesResultFixtures(mock)
8893
addReadFileResultFixtures(mock)
8994
addSearchFilesResultFixtures(mock)
95+
addUseMcpToolResultFixtures(mock)
9096
addWriteToFileResultFixtures(mock)
9197

9298
// The modes test (switch_mode → ask) triggers a second API call whose last
@@ -110,9 +116,6 @@ async function main() {
110116

111117
await mock.start()
112118
}
113-
114-
// Create a temporary workspace folder for tests
115-
testWorkspace = await fs.mkdtemp(path.join(os.tmpdir(), "roo-test-workspace-"))
116119
// Get test filter from command line arguments or environment variable
117120
// Usage examples:
118121
// - npm run test:e2e -- --grep "write-to-file"

0 commit comments

Comments
 (0)