Skip to content

Commit dafa9c9

Browse files
roomoteroomote
committed
test: validate mutating replay payload fixtures
1 parent 9fa41b7 commit dafa9c9

5 files changed

Lines changed: 59 additions & 25 deletions

File tree

‎apps/vscode-e2e/src/fixtures/apply-diff.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
import { LLMock } from "@copilotkit/aimock"
22

3+
import { toolResultContains } from "./tool-result"
4+
35
type ApplyDiffFixture = {
46
toolCallId: string
7+
expected: string[]
58
result: string
69
id: string
710
}
@@ -10,26 +13,31 @@ export function addApplyDiffResultFixtures(mock: InstanceType<typeof LLMock>) {
1013
const fixtures: ApplyDiffFixture[] = [
1114
{
1215
toolCallId: "call_apply_diff_simple_001",
16+
expected: ['"path":"apply-diff-tool-fixture/simple-modify.txt"', '"operation":"modified"'],
1317
result: "Updated `apply-diff-tool-fixture/simple-modify.txt` to say `Hello Universe`.",
1418
id: "call_apply_diff_simple_002",
1519
},
1620
{
1721
toolCallId: "call_apply_diff_multi_replace_001",
22+
expected: ['"path":"apply-diff-tool-fixture/multiple-replace.js"', '"operation":"modified"'],
1823
result: "Updated `apply-diff-tool-fixture/multiple-replace.js` with the renamed function, parameters, and return fields.",
1924
id: "call_apply_diff_multi_replace_002",
2025
},
2126
{
2227
toolCallId: "call_apply_diff_line_hints_001",
28+
expected: ['"path":"apply-diff-tool-fixture/line-hints.js"', '"operation":"modified"'],
2329
result: "Updated `apply-diff-tool-fixture/line-hints.js` so `oldFunction` became `newFunction` with the new log message.",
2430
id: "call_apply_diff_line_hints_002",
2531
},
2632
{
2733
toolCallId: "call_apply_diff_error_001",
28-
result: "The apply_diff operation on `apply-diff-tool-fixture/error-handling.txt` was rejected — the search content did not match any content in the file, so it was not modified.",
34+
expected: ["No sufficiently similar match found at line: 1", "This content does not exist"],
35+
result: "The apply_diff operation on `apply-diff-tool-fixture/error-handling.txt` was rejected - the search content did not match any content in the file, so it was not modified.",
2936
id: "call_apply_diff_error_002",
3037
},
3138
{
3239
toolCallId: "call_apply_diff_multi_block_001",
40+
expected: ['"path":"apply-diff-tool-fixture/multi-search-replace.js"', '"operation":"modified"'],
3341
result: "Applied both search/replace blocks in `apply-diff-tool-fixture/multi-search-replace.js` to rename the two target functions.",
3442
id: "call_apply_diff_multi_block_002",
3543
},
@@ -39,6 +47,7 @@ export function addApplyDiffResultFixtures(mock: InstanceType<typeof LLMock>) {
3947
mock.addFixture({
4048
match: {
4149
toolCallId: fixture.toolCallId,
50+
predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected),
4251
},
4352
response: {
4453
toolCalls: [

‎apps/vscode-e2e/src/fixtures/execute-command.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import { LLMock } from "@copilotkit/aimock"
22

3+
import { toolResultContains } from "./tool-result"
4+
35
type ExecuteCommandToolCall = {
46
name: "execute_command" | "attempt_completion"
57
params: Record<string, unknown>
@@ -8,13 +10,15 @@ type ExecuteCommandToolCall = {
810

911
type ExecuteCommandFixture = {
1012
toolCallId: string
13+
expected: string[]
1114
toolCalls: ExecuteCommandToolCall[]
1215
}
1316

1417
export function addExecuteCommandResultFixtures(mock: InstanceType<typeof LLMock>) {
1518
const fixtures: ExecuteCommandFixture[] = [
1619
{
1720
toolCallId: "call_execute_command_simple_001",
21+
expected: ["Command executed in terminal within working directory '", "Exit code: 0\nOutput:\n"],
1822
toolCalls: [
1923
{
2024
name: "attempt_completion",
@@ -27,6 +31,7 @@ export function addExecuteCommandResultFixtures(mock: InstanceType<typeof LLMock
2731
},
2832
{
2933
toolCallId: "call_execute_command_cwd_001",
34+
expected: ["execute-command-tool-fixture/custom-cwd'. Exit code: 0", "Output:\n"],
3035
toolCalls: [
3136
{
3237
name: "attempt_completion",
@@ -39,6 +44,7 @@ export function addExecuteCommandResultFixtures(mock: InstanceType<typeof LLMock
3944
},
4045
{
4146
toolCallId: "call_execute_command_multi_001",
47+
expected: ["Command executed in terminal within working directory '", "Exit code: 0\nOutput:\n"],
4248
toolCalls: [
4349
{
4450
name: "execute_command",
@@ -51,6 +57,7 @@ export function addExecuteCommandResultFixtures(mock: InstanceType<typeof LLMock
5157
},
5258
{
5359
toolCallId: "call_execute_command_multi_002",
60+
expected: ["Command executed in terminal within working directory '", "Exit code: 0\nOutput:\n"],
5461
toolCalls: [
5562
{
5663
name: "attempt_completion",
@@ -63,6 +70,7 @@ export function addExecuteCommandResultFixtures(mock: InstanceType<typeof LLMock
6370
},
6471
{
6572
toolCallId: "call_execute_command_long_running_001",
73+
expected: ["Exit code: 0", "Command completed after delay"],
6674
toolCalls: [
6775
{
6876
name: "attempt_completion",
@@ -79,6 +87,7 @@ export function addExecuteCommandResultFixtures(mock: InstanceType<typeof LLMock
7987
mock.addFixture({
8088
match: {
8189
toolCallId: fixture.toolCallId,
90+
predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected),
8291
},
8392
response: {
8493
toolCalls: fixture.toolCalls.map((toolCall) => ({

‎apps/vscode-e2e/src/fixtures/read-file.ts‎

Lines changed: 6 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
import { LLMock } from "@copilotkit/aimock"
2-
import type { ChatCompletionRequest, ChatMessage } from "@copilotkit/aimock"
32

4-
type ToolResultExpectation = { toolCallId: string; expected: string[] }
3+
import {
4+
isToolResultExpectation,
5+
toolResultContains,
6+
toolResultsContain,
7+
type ToolResultExpectation,
8+
} from "./tool-result"
59

610
type ReadFileResultFixture = {
711
toolCallId: string
@@ -10,28 +14,6 @@ type ReadFileResultFixture = {
1014
id: string
1115
}
1216

13-
function isToolResultExpectation(value: unknown): value is ToolResultExpectation {
14-
return typeof value === "object" && value !== null && "toolCallId" in value && "expected" in value
15-
}
16-
17-
function toolResultContains(req: ChatCompletionRequest, toolCallId: string, expected: string[]) {
18-
const messages = Array.isArray(req?.messages) ? req.messages : []
19-
const toolMessage = messages.find(
20-
(message: ChatMessage) => message?.role === "tool" && message.tool_call_id === toolCallId,
21-
)
22-
23-
const content = toolMessage?.content
24-
if (typeof content !== "string") {
25-
return false
26-
}
27-
28-
return expected.every((text) => content.includes(text))
29-
}
30-
31-
function toolResultsContain(req: ChatCompletionRequest, expectations: ToolResultExpectation[]) {
32-
return expectations.every(({ toolCallId, expected }) => toolResultContains(req, toolCallId, expected))
33-
}
34-
3517
export function addReadFileResultFixtures(mock: InstanceType<typeof LLMock>) {
3618
const fixtures: ReadFileResultFixture[] = [
3719
{

‎apps/vscode-e2e/src/fixtures/tool-result.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import type { ChatCompletionRequest, ChatMessage } from "@copilotkit/aimock"
2+
3+
export type ToolResultExpectation = { toolCallId: string; expected: string[] }
4+
5+
export function isToolResultExpectation(value: unknown): value is ToolResultExpectation {
6+
return typeof value === "object" && value !== null && "toolCallId" in value && "expected" in value
7+
}
8+
9+
export function toolResultContains(req: ChatCompletionRequest, toolCallId: string, expected: string[]) {
10+
const messages = Array.isArray(req?.messages) ? req.messages : []
11+
const toolMessage = messages.find(
12+
(message: ChatMessage) => message?.role === "tool" && message.tool_call_id === toolCallId,
13+
)
14+
15+
const content = toolMessage?.content
16+
if (typeof content !== "string") {
17+
return false
18+
}
19+
20+
return expected.every((text) => content.includes(text))
21+
}
22+
23+
export function toolResultsContain(req: ChatCompletionRequest, expectations: ToolResultExpectation[]) {
24+
return expectations.every(({ toolCallId, expected }) => toolResultContains(req, toolCallId, expected))
25+
}

‎apps/vscode-e2e/src/fixtures/write-to-file.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
import { LLMock } from "@copilotkit/aimock"
22

3+
import { toolResultContains } from "./tool-result"
4+
35
type WriteToFileFixture = {
46
toolCallId: string
7+
expected: string[]
58
result: string
69
id: string
710
}
@@ -10,11 +13,16 @@ export function addWriteToFileResultFixtures(mock: InstanceType<typeof LLMock>)
1013
const fixtures: WriteToFileFixture[] = [
1114
{
1215
toolCallId: "call_write_to_file_create_001",
16+
expected: ['"path":"write-to-file-tool-fixture/write-to-file-smoke.txt"', '"operation":"created"'],
1317
result: "Created `write-to-file-tool-fixture/write-to-file-smoke.txt` with the requested content.",
1418
id: "call_write_to_file_create_002",
1519
},
1620
{
1721
toolCallId: "call_write_to_file_nested_001",
22+
expected: [
23+
'"path":"write-to-file-tool-fixture/nested/deep/directory/write-to-file-nested-smoke.txt"',
24+
'"operation":"created"',
25+
],
1826
result: "Created `write-to-file-tool-fixture/nested/deep/directory/write-to-file-nested-smoke.txt` with the requested content.",
1927
id: "call_write_to_file_nested_002",
2028
},
@@ -24,6 +32,7 @@ export function addWriteToFileResultFixtures(mock: InstanceType<typeof LLMock>)
2432
mock.addFixture({
2533
match: {
2634
toolCallId: fixture.toolCallId,
35+
predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected),
2736
},
2837
response: {
2938
toolCalls: [

0 commit comments

Comments
 (0)