Skip to content

Commit fa293b2

Browse files
committed
test(e2e): unskip use_mcp_tool replay coverage with local MCP server
1 parent 1076501 commit fa293b2

4 files changed

Lines changed: 306 additions & 60 deletions

File tree

apps/vscode-e2e/src/fixtures/use-mcp-tool.ts

Lines changed: 25 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
import * as path from "path"
2-
31
import { LLMock } from "@copilotkit/aimock"
42

53
const TEST_DIR_NAME = "use-mcp-tool-fixture"
64
const FILESYSTEM_SERVER_NAME = "filesystem"
5+
const READ_FILE_RELATIVE_PATH = `${TEST_DIR_NAME}/mcp-read-target.txt`
6+
const WRITE_FILE_RELATIVE_PATH = `${TEST_DIR_NAME}/mcp-write-target.txt`
77

88
type UseMcpToolFixture = {
99
userMessagePattern: string
@@ -15,48 +15,45 @@ type UseMcpToolFixture = {
1515
id: string
1616
}
1717

18-
export function addUseMcpToolResultFixtures(mock: InstanceType<typeof LLMock>, workspaceDir: string) {
19-
const readFilePath = path.join(workspaceDir, TEST_DIR_NAME, "mcp-read-target.txt")
20-
const writeFilePath = path.join(workspaceDir, TEST_DIR_NAME, "mcp-write-target.txt")
21-
18+
export function addUseMcpToolResultFixtures(mock: InstanceType<typeof LLMock>) {
2219
const fixtures: UseMcpToolFixture[] = [
2320
{
2421
userMessagePattern: "USE_MCP_TOOL_READ_FILE_SMOKE",
2522
toolCallId: "call_use_mcp_tool_read_file_001",
2623
toolName: "read_file",
27-
toolArguments: { path: readFilePath },
24+
toolArguments: { path: READ_FILE_RELATIVE_PATH },
2825
result: "Read the requested file through the MCP filesystem server.",
2926
id: "call_use_mcp_tool_read_file_002",
3027
},
3128
{
3229
userMessagePattern: "USE_MCP_TOOL_WRITE_FILE_SMOKE",
3330
toolCallId: "call_use_mcp_tool_write_file_001",
3431
toolName: "write_file",
35-
toolArguments: { path: writeFilePath, content: "Hello from MCP!" },
32+
toolArguments: { path: WRITE_FILE_RELATIVE_PATH, content: "Hello from MCP!" },
3633
result: "Created the requested file through the MCP filesystem server.",
3734
id: "call_use_mcp_tool_write_file_002",
3835
},
3936
{
4037
userMessagePattern: "USE_MCP_TOOL_LIST_DIRECTORY_SMOKE",
4138
toolCallId: "call_use_mcp_tool_list_directory_001",
4239
toolName: "list_directory",
43-
toolArguments: { path: path.join(workspaceDir, TEST_DIR_NAME) },
40+
toolArguments: { path: TEST_DIR_NAME },
4441
result: "Listed the requested directory through the MCP filesystem server.",
4542
id: "call_use_mcp_tool_list_directory_002",
4643
},
4744
{
4845
userMessagePattern: "USE_MCP_TOOL_DIRECTORY_TREE_SMOKE",
4946
toolCallId: "call_use_mcp_tool_directory_tree_001",
5047
toolName: "directory_tree",
51-
toolArguments: { path: path.join(workspaceDir, TEST_DIR_NAME) },
48+
toolArguments: { path: TEST_DIR_NAME },
5249
result: "Returned the directory tree through the MCP filesystem server.",
5350
id: "call_use_mcp_tool_directory_tree_002",
5451
},
5552
{
5653
userMessagePattern: "USE_MCP_TOOL_GET_FILE_INFO_SMOKE",
5754
toolCallId: "call_use_mcp_tool_get_file_info_001",
5855
toolName: "get_file_info",
59-
toolArguments: { path: readFilePath },
56+
toolArguments: { path: READ_FILE_RELATIVE_PATH },
6057
result: "Returned the requested file metadata through the MCP filesystem server.",
6158
id: "call_use_mcp_tool_get_file_info_002",
6259
},
@@ -65,26 +62,35 @@ export function addUseMcpToolResultFixtures(mock: InstanceType<typeof LLMock>, w
6562
toolCallId: "call_use_mcp_tool_unknown_server_001",
6663
serverName: "nonexistent-server",
6764
toolName: "read_file",
68-
toolArguments: { path: readFilePath },
69-
result: "Handled the missing MCP server gracefully.",
65+
toolArguments: { path: READ_FILE_RELATIVE_PATH },
66+
result: "MCP server 'nonexistent-server' is not configured. Available servers: filesystem",
7067
id: "call_use_mcp_tool_unknown_server_002",
7168
},
7269
]
7370

7471
for (const fixture of fixtures) {
72+
const serverName = fixture.serverName ?? FILESYSTEM_SERVER_NAME
73+
const isConfiguredFilesystemTool = serverName === FILESYSTEM_SERVER_NAME
74+
7575
mock.addFixture({
7676
match: {
7777
userMessage: new RegExp(fixture.userMessagePattern),
7878
},
7979
response: {
8080
toolCalls: [
8181
{
82-
name: "use_mcp_tool",
83-
arguments: JSON.stringify({
84-
server_name: fixture.serverName ?? FILESYSTEM_SERVER_NAME,
85-
tool_name: fixture.toolName,
86-
arguments: fixture.toolArguments,
87-
}),
82+
name: isConfiguredFilesystemTool
83+
? `mcp--${FILESYSTEM_SERVER_NAME}--${fixture.toolName}`
84+
: "use_mcp_tool",
85+
arguments: JSON.stringify(
86+
isConfiguredFilesystemTool
87+
? fixture.toolArguments
88+
: {
89+
server_name: serverName,
90+
tool_name: fixture.toolName,
91+
arguments: fixture.toolArguments,
92+
},
93+
),
8894
id: fixture.toolCallId,
8995
},
9096
],

apps/vscode-e2e/src/runTest.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ async function main() {
9292
addListFilesResultFixtures(mock)
9393
addReadFileResultFixtures(mock)
9494
addSearchFilesResultFixtures(mock)
95-
addUseMcpToolResultFixtures(mock, testWorkspace)
95+
addUseMcpToolResultFixtures(mock)
9696
addWriteToFileResultFixtures(mock)
9797

9898
// The modes test (switch_mode → ask) triggers a second API call whose last
Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
import * as fs from "fs/promises"
2+
import * as path from "path"
3+
import { stdin, stdout, stderr } from "process"
4+
5+
type JsonRpcMessage = {
6+
jsonrpc: "2.0"
7+
id?: string | number
8+
method?: string
9+
params?: {
10+
name?: string
11+
arguments?: Record<string, unknown>
12+
}
13+
}
14+
15+
const workspaceDir = process.argv[2]
16+
const readyFile = process.env.MCP_TEST_READY_FILE
17+
18+
if (!workspaceDir) {
19+
stderr.write("Missing workspace directory argument\n")
20+
process.exit(1)
21+
}
22+
23+
let buffer = ""
24+
25+
stdin.setEncoding("utf8")
26+
stdin.on("data", (chunk) => {
27+
buffer += chunk
28+
processBuffer().catch((error) => {
29+
stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`)
30+
})
31+
})
32+
33+
async function processBuffer() {
34+
let newlineIndex = buffer.indexOf("\n")
35+
while (newlineIndex !== -1) {
36+
const line = buffer.slice(0, newlineIndex).trim()
37+
buffer = buffer.slice(newlineIndex + 1)
38+
39+
if (line) {
40+
await handleMessage(JSON.parse(line) as JsonRpcMessage)
41+
}
42+
43+
newlineIndex = buffer.indexOf("\n")
44+
}
45+
}
46+
47+
async function handleMessage(message: JsonRpcMessage) {
48+
if (message.id === undefined) {
49+
return
50+
}
51+
52+
try {
53+
switch (message.method) {
54+
case "initialize":
55+
sendResult(message.id, {
56+
protocolVersion: "2024-11-05",
57+
capabilities: { tools: {} },
58+
serverInfo: { name: "test-filesystem-server", version: "1.0.0" },
59+
})
60+
break
61+
case "tools/list":
62+
await markReady()
63+
sendResult(message.id, { tools: getTools() })
64+
break
65+
case "resources/list":
66+
sendResult(message.id, { resources: [] })
67+
break
68+
case "resources/templates/list":
69+
sendResult(message.id, { resourceTemplates: [] })
70+
break
71+
case "tools/call":
72+
sendResult(message.id, await callTool(message.params?.name, message.params?.arguments ?? {}))
73+
break
74+
default:
75+
sendResult(message.id, {})
76+
}
77+
} catch (error) {
78+
sendError(message.id, error instanceof Error ? error.message : String(error))
79+
}
80+
}
81+
82+
function sendResult(id: string | number, result: unknown) {
83+
stdout.write(`${JSON.stringify({ jsonrpc: "2.0", id, result })}\n`)
84+
}
85+
86+
function sendError(id: string | number, message: string) {
87+
stdout.write(`${JSON.stringify({ jsonrpc: "2.0", id, error: { code: -32603, message } })}\n`)
88+
}
89+
90+
async function markReady() {
91+
if (!readyFile) {
92+
return
93+
}
94+
95+
await fs.mkdir(path.dirname(readyFile), { recursive: true })
96+
await fs.writeFile(readyFile, "ready")
97+
}
98+
99+
function getTools() {
100+
const pathInputSchema = {
101+
type: "object",
102+
properties: {
103+
path: { type: "string" },
104+
},
105+
required: ["path"],
106+
}
107+
108+
return [
109+
{
110+
name: "read_file",
111+
description: "Read a file from the test workspace.",
112+
inputSchema: pathInputSchema,
113+
},
114+
{
115+
name: "write_file",
116+
description: "Write a file in the test workspace.",
117+
inputSchema: {
118+
type: "object",
119+
properties: {
120+
path: { type: "string" },
121+
content: { type: "string" },
122+
},
123+
required: ["path", "content"],
124+
},
125+
},
126+
{
127+
name: "list_directory",
128+
description: "List a directory in the test workspace.",
129+
inputSchema: pathInputSchema,
130+
},
131+
{
132+
name: "directory_tree",
133+
description: "Return a JSON directory tree for a test workspace path.",
134+
inputSchema: pathInputSchema,
135+
},
136+
{
137+
name: "get_file_info",
138+
description: "Return basic metadata for a file in the test workspace.",
139+
inputSchema: pathInputSchema,
140+
},
141+
]
142+
}
143+
144+
async function callTool(name: string | undefined, args: Record<string, unknown>) {
145+
const requestedPath = typeof args.path === "string" ? args.path : ""
146+
147+
switch (name) {
148+
case "read_file": {
149+
const filePath = resolveWorkspacePath(requestedPath)
150+
return textResult(await fs.readFile(filePath, "utf8"))
151+
}
152+
case "write_file": {
153+
const filePath = resolveWorkspacePath(requestedPath)
154+
const content = typeof args.content === "string" ? args.content : ""
155+
await fs.mkdir(path.dirname(filePath), { recursive: true })
156+
await fs.writeFile(filePath, content)
157+
return textResult(`Successfully wrote to ${requestedPath}`)
158+
}
159+
case "list_directory": {
160+
const directoryPath = resolveWorkspacePath(requestedPath)
161+
const entries = await fs.readdir(directoryPath, { withFileTypes: true })
162+
const listing = entries
163+
.sort((a, b) => a.name.localeCompare(b.name))
164+
.map((entry) => `${entry.isDirectory() ? "[DIR]" : "[FILE]"} ${entry.name}`)
165+
.join("\n")
166+
return textResult(listing)
167+
}
168+
case "directory_tree": {
169+
const directoryPath = resolveWorkspacePath(requestedPath)
170+
return textResult(JSON.stringify(await buildDirectoryTree(directoryPath), null, 2))
171+
}
172+
case "get_file_info": {
173+
const filePath = resolveWorkspacePath(requestedPath)
174+
const stats = await fs.stat(filePath)
175+
return textResult(
176+
[
177+
`size: ${stats.size}`,
178+
`isFile: ${stats.isFile()}`,
179+
`isDirectory: ${stats.isDirectory()}`,
180+
`permissions: ${stats.mode.toString(8)}`,
181+
].join("\n"),
182+
)
183+
}
184+
default:
185+
throw new Error(`Unknown tool: ${name}`)
186+
}
187+
}
188+
189+
function textResult(text: string) {
190+
return {
191+
content: [{ type: "text", text }],
192+
}
193+
}
194+
195+
function resolveWorkspacePath(requestedPath: string) {
196+
const resolvedPath = path.resolve(workspaceDir!, requestedPath)
197+
const workspaceRoot = path.resolve(workspaceDir!)
198+
199+
if (resolvedPath !== workspaceRoot && !resolvedPath.startsWith(`${workspaceRoot}${path.sep}`)) {
200+
throw new Error(`Path is outside the test workspace: ${requestedPath}`)
201+
}
202+
203+
return resolvedPath
204+
}
205+
206+
async function buildDirectoryTree(
207+
directoryPath: string,
208+
): Promise<{ name: string; type: string; children?: unknown[] }> {
209+
const stats = await fs.stat(directoryPath)
210+
const node: { name: string; type: string; children?: unknown[] } = {
211+
name: path.basename(directoryPath),
212+
type: stats.isDirectory() ? "directory" : "file",
213+
}
214+
215+
if (!stats.isDirectory()) {
216+
return node
217+
}
218+
219+
const entries = await fs.readdir(directoryPath, { withFileTypes: true })
220+
node.children = await Promise.all(
221+
entries
222+
.sort((a, b) => a.name.localeCompare(b.name))
223+
.map((entry) => buildDirectoryTree(path.join(directoryPath, entry.name))),
224+
)
225+
226+
return node
227+
}

0 commit comments

Comments
 (0)