Skip to content

Commit c31a59a

Browse files
roomoteroomote
committed
test(e2e): unskip use_mcp_tool replay coverage
1 parent fd6da8a commit c31a59a

5 files changed

Lines changed: 539 additions & 849 deletions

File tree

‎apps/vscode-e2e/fixtures/list-files.json‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "LIST_FILES_NON_RECURSIVE_SMOKE"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "list_files",
11+
"arguments": "{\"path\":\"list-files-tool-fixture\",\"recursive\":false}",
12+
"id": "call_list_files_non_recursive_001"
13+
}
14+
]
15+
}
16+
},
17+
{
18+
"match": {
19+
"userMessage": "LIST_FILES_RECURSIVE_SMOKE"
20+
},
21+
"response": {
22+
"toolCalls": [
23+
{
24+
"name": "list_files",
25+
"arguments": "{\"path\":\"list-files-tool-fixture\",\"recursive\":true}",
26+
"id": "call_list_files_recursive_001"
27+
}
28+
]
29+
}
30+
},
31+
{
32+
"match": {
33+
"userMessage": "LIST_FILES_SYMLINK_SMOKE"
34+
},
35+
"response": {
36+
"toolCalls": [
37+
{
38+
"name": "list_files",
39+
"arguments": "{\"path\":\"list-files-symlink-fixture\",\"recursive\":false}",
40+
"id": "call_list_files_symlink_001"
41+
}
42+
]
43+
}
44+
},
45+
{
46+
"match": {
47+
"userMessage": "LIST_FILES_WORKSPACE_ROOT_SMOKE"
48+
},
49+
"response": {
50+
"toolCalls": [
51+
{
52+
"name": "list_files",
53+
"arguments": "{\"path\":\".\",\"recursive\":false}",
54+
"id": "call_list_files_workspace_root_001"
55+
}
56+
]
57+
}
58+
}
59+
]
60+
}

‎apps/vscode-e2e/fixtures/search-files.json‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "SEARCH_FILES_FUNCTIONS_SMOKE"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "search_files",
11+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"function\\\\s+\\\\w+\"}",
12+
"id": "call_search_files_functions_001"
13+
}
14+
]
15+
}
16+
},
17+
{
18+
"match": {
19+
"userMessage": "SEARCH_FILES_TODO_SMOKE"
20+
},
21+
"response": {
22+
"toolCalls": [
23+
{
24+
"name": "search_files",
25+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"TODO.*\"}",
26+
"id": "call_search_files_todo_001"
27+
}
28+
]
29+
}
30+
},
31+
{
32+
"match": {
33+
"userMessage": "SEARCH_FILES_TYPESCRIPT_SMOKE"
34+
},
35+
"response": {
36+
"toolCalls": [
37+
{
38+
"name": "search_files",
39+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"interface\\\\s+\\\\w+\",\"file_pattern\":\"*.ts\"}",
40+
"id": "call_search_files_typescript_001"
41+
}
42+
]
43+
}
44+
},
45+
{
46+
"match": {
47+
"userMessage": "SEARCH_FILES_JSON_SMOKE"
48+
},
49+
"response": {
50+
"toolCalls": [
51+
{
52+
"name": "search_files",
53+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"\\\"\\\\w+\\\":\\\\s*\",\"file_pattern\":\"*.json\"}",
54+
"id": "call_search_files_json_001"
55+
}
56+
]
57+
}
58+
},
59+
{
60+
"match": {
61+
"userMessage": "SEARCH_FILES_NESTED_SMOKE"
62+
},
63+
"response": {
64+
"toolCalls": [
65+
{
66+
"name": "search_files",
67+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"function\\\\s+(format|debounce)\"}",
68+
"id": "call_search_files_nested_001"
69+
}
70+
]
71+
}
72+
},
73+
{
74+
"match": {
75+
"userMessage": "SEARCH_FILES_COMPLEX_REGEX_SMOKE"
76+
},
77+
"response": {
78+
"toolCalls": [
79+
{
80+
"name": "search_files",
81+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"(import|export).*\",\"file_pattern\":\"*.{js,ts}\"}",
82+
"id": "call_search_files_complex_regex_001"
83+
}
84+
]
85+
}
86+
},
87+
{
88+
"match": {
89+
"userMessage": "SEARCH_FILES_NO_MATCH_SMOKE"
90+
},
91+
"response": {
92+
"toolCalls": [
93+
{
94+
"name": "search_files",
95+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"nonExistentPattern12345\"}",
96+
"id": "call_search_files_no_match_001"
97+
}
98+
]
99+
}
100+
},
101+
{
102+
"match": {
103+
"userMessage": "SEARCH_FILES_CLASS_METHOD_SMOKE"
104+
},
105+
"response": {
106+
"toolCalls": [
107+
{
108+
"name": "search_files",
109+
"arguments": "{\"path\":\"search-files-tool-fixture\",\"regex\":\"(class\\\\s+\\\\w+|async\\\\s+\\\\w+)\",\"file_pattern\":\"*.ts\"}",
110+
"id": "call_search_files_class_method_001"
111+
}
112+
]
113+
}
114+
}
115+
]
116+
}

‎apps/vscode-e2e/src/fixtures/use-mcp-tool.ts‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
import * as path from "path"
2+
3+
import { LLMock } from "@copilotkit/aimock"
4+
5+
const TEST_DIR_NAME = "use-mcp-tool-fixture"
6+
const FILESYSTEM_SERVER_NAME = "filesystem"
7+
8+
type UseMcpToolFixture = {
9+
userMessagePattern: string
10+
toolCallId: string
11+
toolName: string
12+
toolArguments?: Record<string, unknown>
13+
serverName?: string
14+
result: string
15+
id: string
16+
}
17+
18+
export function addUseMcpToolResultFixtures(mock: InstanceType<typeof LLMock>, workspaceDir: string) {
19+
const readFilePath = path.join(workspaceDir, TEST_DIR_NAME, "mcp-read-target.txt")
20+
const writeFilePath = path.join(workspaceDir, TEST_DIR_NAME, "mcp-write-target.txt")
21+
22+
const fixtures: UseMcpToolFixture[] = [
23+
{
24+
userMessagePattern: "USE_MCP_TOOL_READ_FILE_SMOKE",
25+
toolCallId: "call_use_mcp_tool_read_file_001",
26+
toolName: "read_file",
27+
toolArguments: { path: readFilePath },
28+
result: "Read the requested file through the MCP filesystem server.",
29+
id: "call_use_mcp_tool_read_file_002",
30+
},
31+
{
32+
userMessagePattern: "USE_MCP_TOOL_WRITE_FILE_SMOKE",
33+
toolCallId: "call_use_mcp_tool_write_file_001",
34+
toolName: "write_file",
35+
toolArguments: { path: writeFilePath, content: "Hello from MCP!" },
36+
result: "Created the requested file through the MCP filesystem server.",
37+
id: "call_use_mcp_tool_write_file_002",
38+
},
39+
{
40+
userMessagePattern: "USE_MCP_TOOL_LIST_DIRECTORY_SMOKE",
41+
toolCallId: "call_use_mcp_tool_list_directory_001",
42+
toolName: "list_directory",
43+
toolArguments: { path: path.join(workspaceDir, TEST_DIR_NAME) },
44+
result: "Listed the requested directory through the MCP filesystem server.",
45+
id: "call_use_mcp_tool_list_directory_002",
46+
},
47+
{
48+
userMessagePattern: "USE_MCP_TOOL_DIRECTORY_TREE_SMOKE",
49+
toolCallId: "call_use_mcp_tool_directory_tree_001",
50+
toolName: "directory_tree",
51+
toolArguments: { path: path.join(workspaceDir, TEST_DIR_NAME) },
52+
result: "Returned the directory tree through the MCP filesystem server.",
53+
id: "call_use_mcp_tool_directory_tree_002",
54+
},
55+
{
56+
userMessagePattern: "USE_MCP_TOOL_GET_FILE_INFO_SMOKE",
57+
toolCallId: "call_use_mcp_tool_get_file_info_001",
58+
toolName: "get_file_info",
59+
toolArguments: { path: readFilePath },
60+
result: "Returned the requested file metadata through the MCP filesystem server.",
61+
id: "call_use_mcp_tool_get_file_info_002",
62+
},
63+
{
64+
userMessagePattern: "USE_MCP_TOOL_UNKNOWN_SERVER_SMOKE",
65+
toolCallId: "call_use_mcp_tool_unknown_server_001",
66+
serverName: "nonexistent-server",
67+
toolName: "read_file",
68+
toolArguments: { path: readFilePath },
69+
result: "Handled the missing MCP server gracefully.",
70+
id: "call_use_mcp_tool_unknown_server_002",
71+
},
72+
]
73+
74+
for (const fixture of fixtures) {
75+
mock.addFixture({
76+
match: {
77+
userMessage: new RegExp(fixture.userMessagePattern),
78+
},
79+
response: {
80+
toolCalls: [
81+
{
82+
name: "use_mcp_tool",
83+
arguments: JSON.stringify({
84+
server_name: fixture.serverName ?? FILESYSTEM_SERVER_NAME,
85+
tool_name: fixture.toolName,
86+
arguments: fixture.toolArguments,
87+
}),
88+
id: fixture.toolCallId,
89+
},
90+
],
91+
},
92+
})
93+
94+
mock.addFixture({
95+
match: {
96+
toolCallId: fixture.toolCallId,
97+
},
98+
response: {
99+
toolCalls: [
100+
{
101+
name: "attempt_completion",
102+
arguments: JSON.stringify({ result: fixture.result }),
103+
id: fixture.id,
104+
},
105+
],
106+
},
107+
})
108+
}
109+
}

‎apps/vscode-e2e/src/runTest.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { addExecuteCommandResultFixtures } from "./fixtures/execute-command"
1010
import { addListFilesResultFixtures } from "./fixtures/list-files"
1111
import { addReadFileResultFixtures } from "./fixtures/read-file"
1212
import { addSearchFilesResultFixtures } from "./fixtures/search-files"
13+
import { addUseMcpToolResultFixtures } from "./fixtures/use-mcp-tool"
1314
import { addWriteToFileResultFixtures } from "./fixtures/write-to-file"
1415

1516
function getCliFlagValue(flag: string) {
@@ -59,6 +60,10 @@ async function main() {
5960
let testWorkspace: string | undefined
6061

6162
try {
63+
// Create a temporary workspace folder for tests before installing fixtures that
64+
// need workspace-specific paths.
65+
testWorkspace = await fs.mkdtemp(path.join(os.tmpdir(), "roo-test-workspace-"))
66+
6267
if (useMock) {
6368
const fixturesDir = path.resolve(__dirname, "../fixtures")
6469

@@ -87,6 +92,7 @@ async function main() {
8792
addListFilesResultFixtures(mock)
8893
addReadFileResultFixtures(mock)
8994
addSearchFilesResultFixtures(mock)
95+
addUseMcpToolResultFixtures(mock, testWorkspace)
9096
addWriteToFileResultFixtures(mock)
9197

9298
// The modes test (switch_mode → ask) triggers a second API call whose last
@@ -110,9 +116,6 @@ async function main() {
110116

111117
await mock.start()
112118
}
113-
114-
// Create a temporary workspace folder for tests
115-
testWorkspace = await fs.mkdtemp(path.join(os.tmpdir(), "roo-test-workspace-"))
116119
// Get test filter from command line arguments or environment variable
117120
// Usage examples:
118121
// - npm run test:e2e -- --grep "write-to-file"

0 commit comments

Comments
 (0)