test(e2e): restore replay result validation

Author: roomote

apps/vscode-e2e/src/fixtures/list-files.ts

@@ -6,7 +6,7 @@ type ListFilesFixture = {
 	toolName: string
 	arguments: string
 	toolCallId: string
-	expected?: string[]
+	expected: string[]
 	result: string
 	id: string
 }
@@ -32,6 +32,7 @@ export function addListFilesResultFixtures(mock: InstanceType<typeof LLMock>) {
 			toolName: "list_files",
 			arguments: '{"path":"list-files-tool-fixture","recursive":false}',
 			toolCallId: "call_list_files_non_recursive_001",
+			expected: ["root-file-1.txt", ".hidden-file", "nested/"],
 			result: "The non-recursive listing for `list-files-tool-fixture` includes `root-file-1.txt`, `root-file-2.js`, `config.yaml`, `README.md`, `.hidden-file`, and the `nested/` directory.",
 			id: "call_list_files_non_recursive_002",
 		},
@@ -50,7 +51,7 @@ export function addListFilesResultFixtures(mock: InstanceType<typeof LLMock>) {
 			arguments: '{"path":"list-files-symlink-fixture","recursive":false}',
 			toolCallId: "call_list_files_symlink_001",
 			expected: ["link-to-file.txt", "source/"],
-			result: "The symlink fixture listing shows the original `source/` directory and its `source-file.txt`, alongside the symlinked entries in `list-files-symlink-fixture`.",
+			result: "The symlink fixture listing shows the original `source/` directory and its `source-file.txt`, plus the symlink entry `link-to-file.txt` in `list-files-symlink-fixture`.",
 			id: "call_list_files_symlink_002",
 		},
 		{
@@ -83,9 +84,7 @@ export function addListFilesResultFixtures(mock: InstanceType<typeof LLMock>) {
 		mock.addFixture({
 			match: {
 				toolCallId: fixture.toolCallId,
-				...(fixture.expected && {
-					predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected!),
-				}),
+				predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected),
 			},
 			response: {
 				toolCalls: [

apps/vscode-e2e/src/fixtures/search-files.ts

@@ -1,21 +1,42 @@
 import { LLMock } from "@copilotkit/aimock"
+import type { ChatCompletionRequest, ChatMessage } from "@copilotkit/aimock"
 
 type SearchFilesFixture = {
 	userMessagePattern: string
 	toolName: string
 	arguments: string
 	toolCallId: string
+	expected: string[]
 	result: string
 	id: string
 }
 
+function toolResultContains(req: ChatCompletionRequest, toolCallId: string, expected: string[]) {
+	const messages = Array.isArray(req?.messages) ? req.messages : []
+	const toolMessage = messages.find(
+		(message: ChatMessage) => message?.role === "tool" && message.tool_call_id === toolCallId,
+	)
+
+	const content = toolMessage?.content
+	if (typeof content !== "string") {
+		return false
+	}
+
+	return expected.every((text) => content.includes(text))
+}
+
 export function addSearchFilesResultFixtures(mock: InstanceType<typeof LLMock>) {
 	const fixtures: SearchFilesFixture[] = [
 		{
 			userMessagePattern: "SEARCH_FILES_FUNCTIONS_SMOKE",
 			toolName: "search_files",
 			arguments: '{"path":"search-files-tool-fixture","regex":"function\\\\s+\\\\w+"}',
 			toolCallId: "call_search_files_functions_001",
+			expected: [
+				"# search-files-tool-fixture/search-fixture.js",
+				"function calculateTotal(items) {",
+				"function validateUser(user) {",
+			],
 			result: "The function search found declarations including `calculateTotal`, `validateUser`, and `formatCurrency`.",
 			id: "call_search_files_functions_002",
 		},
@@ -24,6 +45,11 @@ export function addSearchFilesResultFixtures(mock: InstanceType<typeof LLMock>)
 			toolName: "search_files",
 			arguments: '{"path":"search-files-tool-fixture","regex":"TODO.*"}',
 			toolCallId: "call_search_files_todo_001",
+			expected: [
+				"# search-files-tool-fixture/search-fixture.js",
+				"// TODO: Add more validation functions",
+				"// TODO: Implement user fetching",
+			],
 			result: "The TODO search found matching TODO entries in the fixture files, including the validation and user-fetching notes.",
 			id: "call_search_files_todo_002",
 		},
@@ -32,6 +58,7 @@ export function addSearchFilesResultFixtures(mock: InstanceType<typeof LLMock>)
 			toolName: "search_files",
 			arguments: '{"path":"search-files-tool-fixture","regex":"interface\\\\s+\\\\w+","file_pattern":"*.ts"}',
 			toolCallId: "call_search_files_typescript_001",
+			expected: ["# search-files-tool-fixture/search-fixture.ts", "interface User {", "interface Product {"],
 			result: "The TypeScript-only search found the `User` and `Product` interface definitions.",
 			id: "call_search_files_typescript_002",
 		},
@@ -40,6 +67,7 @@ export function addSearchFilesResultFixtures(mock: InstanceType<typeof LLMock>)
 			toolName: "search_files",
 			arguments: '{"path":"search-files-tool-fixture","regex":"\\"\\\\w+\\":\\\\s*","file_pattern":"*.json"}',
 			toolCallId: "call_search_files_json_001",
+			expected: ["# search-files-tool-fixture/search-config.json", '"name": "test-app",', '"dependencies": {'],
 			result: "The JSON search found configuration keys such as `name`, `version`, and `dependencies` in `search-config.json`.",
 			id: "call_search_files_json_002",
 		},
@@ -48,6 +76,11 @@ export function addSearchFilesResultFixtures(mock: InstanceType<typeof LLMock>)
 			toolName: "search_files",
 			arguments: '{"path":"search-files-tool-fixture","regex":"function\\\\s+(format|debounce)"}',
 			toolCallId: "call_search_files_nested_001",
+			expected: [
+				"# search-files-tool-fixture/nested/nested-search.js",
+				"function formatCurrency(amount) {",
+				"function debounce(func, wait) {",
+			],
 			result: "The nested-directory search found the utility functions `formatCurrency` and `debounce`.",
 			id: "call_search_files_nested_002",
 		},
@@ -56,6 +89,11 @@ export function addSearchFilesResultFixtures(mock: InstanceType<typeof LLMock>)
 			toolName: "search_files",
 			arguments: '{"path":"search-files-tool-fixture","regex":"(import|export).*","file_pattern":"*.{js,ts}"}',
 			toolCallId: "call_search_files_complex_regex_001",
+			expected: [
+				"# search-files-tool-fixture/search-fixture.js",
+				"export { calculateTotal, validateUser }",
+				"module.exports = { formatCurrency, debounce }",
+			],
 			result: "The import/export search found the `export` statement in the JavaScript fixture module.",
 			id: "call_search_files_complex_regex_002",
 		},
@@ -64,6 +102,7 @@ export function addSearchFilesResultFixtures(mock: InstanceType<typeof LLMock>)
 			toolName: "search_files",
 			arguments: '{"path":"search-files-tool-fixture","regex":"nonExistentPattern12345"}',
 			toolCallId: "call_search_files_no_match_001",
+			expected: ["Found 0 results."],
 			result: "No matches were found for `nonExistentPattern12345` in the search fixture directory.",
 			id: "call_search_files_no_match_002",
 		},
@@ -73,6 +112,11 @@ export function addSearchFilesResultFixtures(mock: InstanceType<typeof LLMock>)
 			arguments:
 				'{"path":"search-files-tool-fixture","regex":"(class\\\\s+\\\\w+|async\\\\s+\\\\w+)","file_pattern":"*.ts"}',
 			toolCallId: "call_search_files_class_method_001",
+			expected: [
+				"# search-files-tool-fixture/search-fixture.ts",
+				"class UserService {",
+				"async getUser(id: number): Promise<User> {",
+			],
 			result: "The class-and-method search found `UserService` and its async `getUser` method in the TypeScript fixture.",
 			id: "call_search_files_class_method_002",
 		},
@@ -97,6 +141,7 @@ export function addSearchFilesResultFixtures(mock: InstanceType<typeof LLMock>)
 		mock.addFixture({
 			match: {
 				toolCallId: fixture.toolCallId,
+				predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected),
 			},
 			response: {
 				toolCalls: [

apps/vscode-e2e/src/suite/tools/list-files.test.ts

@@ -350,15 +350,18 @@ This directory contains various files and subdirectories for testing the list_fi
 			// Wait for task completion
 			await waitFor(() => taskCompleted, { timeout: 120_000 })
 
-			const completionMessage = messages.find(
-				(m) =>
-					m.type === "say" &&
-					(m.say === "completion_result" || m.say === "text") &&
-					(m.text?.includes("source-file.txt") ||
-						m.text?.includes("link-to-file.txt") ||
-						m.text?.includes("source/")),
-			)
-			assert.ok(completionMessage, "AI should have summarized the symlink directory contents")
+			const completionMessage = messages.find((m) => {
+				if (m.type !== "say" || (m.say !== "completion_result" && m.say !== "text")) {
+					return false
+				}
+
+				const text = m.text ?? ""
+				const mentionsOriginalEntry = text.includes("source-file.txt") || text.includes("source/")
+				const mentionsSymlinkEntry = text.includes("link-to-file.txt") || text.includes("link-to-dir")
+
+				return mentionsOriginalEntry && mentionsSymlinkEntry
+			})
+			assert.ok(completionMessage, "AI should have summarized both the original and symlinked directory contents")
 
 			console.log("Test passed! Symlinked files and directories are now visible")