Skip to content

Commit f8b0271

Browse files
easonliang28easonliang28
committed
test(vscode-e2e): cover follow-up mode isolation
1 parent 5b1157c commit f8b0271

5 files changed

Lines changed: 308 additions & 8 deletions

File tree

‎apps/vscode-e2e/src/fixtures/view-state.ts‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
import type { ChatCompletionRequest, ChatMessage, LLMock } from "@copilotkit/aimock"
2+
3+
const TASKS = ["A", "B", "C"] as const
4+
const ROUNDS = 10
5+
6+
const MODE_SEQUENCES: Record<(typeof TASKS)[number], string[]> = {
7+
A: ["ask", "debug", "architect", "orchestrator", "code", "ask", "debug", "architect", "orchestrator", "code"],
8+
B: ["debug", "architect", "orchestrator", "code", "ask", "debug", "architect", "orchestrator", "code", "ask"],
9+
C: ["architect", "orchestrator", "code", "ask", "debug", "architect", "orchestrator", "code", "ask", "debug"],
10+
}
11+
12+
const markerFor = (taskName: (typeof TASKS)[number]) => `FOLLOWUP_MODE_ISOLATION_${taskName}`
13+
const answerFor = (taskName: (typeof TASKS)[number], round: number) => `${taskName} follow-up round ${round}`
14+
const callIdFor = (taskName: (typeof TASKS)[number], round: number) =>
15+
`call_followup_mode_${taskName.toLowerCase()}_${String(round).padStart(2, "0")}`
16+
17+
const lastToolResultContains = (req: ChatCompletionRequest, toolCallId: string, expected: string[]) => {
18+
const messages = Array.isArray(req?.messages) ? req.messages : []
19+
const toolMessage = messages.filter((message: ChatMessage) => message?.role === "tool").at(-1)
20+
const content = toolMessage?.content
21+
22+
return (
23+
toolMessage?.tool_call_id === toolCallId &&
24+
typeof content === "string" &&
25+
expected.every((text) => content.includes(text))
26+
)
27+
}
28+
29+
const followupToolCall = (taskName: (typeof TASKS)[number], round: number) => ({
30+
name: "ask_followup_question",
31+
arguments: JSON.stringify({
32+
question: `Task ${taskName}: choose mode for round ${round}`,
33+
follow_up: [
34+
{
35+
text: answerFor(taskName, round),
36+
mode: MODE_SEQUENCES[taskName][round - 1],
37+
},
38+
],
39+
}),
40+
id: callIdFor(taskName, round),
41+
})
42+
43+
export const getFollowupModeIsolationPlan = () =>
44+
TASKS.map((taskName) => ({
45+
taskName,
46+
marker: markerFor(taskName),
47+
rounds: MODE_SEQUENCES[taskName].map((mode, index) => ({
48+
round: index + 1,
49+
answer: answerFor(taskName, index + 1),
50+
mode,
51+
})),
52+
}))
53+
54+
export function addViewStateFixtures(mock: InstanceType<typeof LLMock>) {
55+
for (const taskName of TASKS) {
56+
mock.addFixture({
57+
match: {
58+
userMessage: markerFor(taskName),
59+
},
60+
response: {
61+
toolCalls: [followupToolCall(taskName, 1)],
62+
},
63+
})
64+
65+
for (let round = 1; round < ROUNDS; round++) {
66+
mock.addFixture({
67+
match: {
68+
predicate: (req) =>
69+
lastToolResultContains(req, callIdFor(taskName, round), [answerFor(taskName, round)]),
70+
},
71+
response: {
72+
toolCalls: [followupToolCall(taskName, round + 1)],
73+
},
74+
})
75+
}
76+
77+
mock.addFixture({
78+
match: {
79+
predicate: (req) =>
80+
lastToolResultContains(req, callIdFor(taskName, ROUNDS), [answerFor(taskName, ROUNDS)]),
81+
},
82+
response: {
83+
toolCalls: [
84+
{
85+
name: "attempt_completion",
86+
arguments: JSON.stringify({
87+
result: `Task ${taskName} completed ${ROUNDS} follow-up mode switches.`,
88+
}),
89+
id: `call_followup_mode_${taskName.toLowerCase()}_complete`,
90+
},
91+
],
92+
},
93+
})
94+
}
95+
}

‎apps/vscode-e2e/src/runTest.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import { addSubtaskFixtures } from "./fixtures/subtasks"
2222
import { addUseMcpToolResultFixtures } from "./fixtures/use-mcp-tool"
2323
import { addWriteToFileResultFixtures } from "./fixtures/write-to-file"
2424
import { toolResultContains } from "./fixtures/tool-result"
25+
import { addViewStateFixtures } from "./fixtures/view-state"
2526

2627
function getCliFlagValue(flag: string) {
2728
return process.argv.find((arg, index) => process.argv[index - 1] === flag)
@@ -130,6 +131,7 @@ async function main() {
130131
addUseMcpToolResultFixtures(mock)
131132
addWriteToFileResultFixtures(mock)
132133
addDeepSeekV4Fixtures(mock)
134+
addViewStateFixtures(mock)
133135

134136
mock.addFixture({
135137
match: {

‎apps/vscode-e2e/src/suite/view-state.test.ts‎

Lines changed: 155 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ import * as assert from "assert"
22

33
import { isSecretStateKey, RooCodeEventName, type ClineMessage, type GlobalState } from "@roo-code/types"
44

5-
import { waitUntilCompleted } from "./utils"
5+
import { getFollowupModeIsolationPlan } from "../fixtures/view-state"
6+
import { sleep, waitFor, waitUntilCompleted } from "./utils"
67
import { setDefaultSuiteTimeout } from "./test-utils"
78

89
const findSecretStatePath = (value: unknown, path: string[] = []): string | undefined => {
@@ -112,4 +113,157 @@ suite("Roo Code View State", function () {
112113
globalThis.api.off(RooCodeEventName.Message, completionHandler)
113114
}
114115
})
116+
test("three panels keep follow-up option mode switches isolated across ten staggered rounds", async () => {
117+
const plan = getFollowupModeIsolationPlan()
118+
const modeEvents: Array<{ taskId: string; mode: string }> = []
119+
const taskIds = new Map<string, string>()
120+
const taskNamesById = new Map<string, string>()
121+
const pendingSuggestions = new Map<string, { answer: string; mode?: string }>()
122+
const answeredSuggestions = new Set<string>()
123+
const suggestionKey = (taskId: string, answer: string) => `${taskId}:${answer}`
124+
let releasedRounds = 0
125+
let roundInFlight = false
126+
127+
const taskIdsInPlanOrder = () =>
128+
plan.map((taskPlan) => taskIds.get(taskPlan.taskName)).filter((taskId): taskId is string => !!taskId)
129+
const modeCountForTask = (taskId: string) => modeEvents.filter((event) => event.taskId === taskId).length
130+
131+
const maybeReleaseRound = () => {
132+
if (roundInFlight || taskIds.size !== plan.length) {
133+
return
134+
}
135+
136+
const taskIdsInOrder = taskIdsInPlanOrder()
137+
if (
138+
taskIdsInOrder.length !== plan.length ||
139+
!taskIdsInOrder.every((taskId) => pendingSuggestions.has(taskId))
140+
) {
141+
return
142+
}
143+
144+
roundInFlight = true
145+
releasedRounds++
146+
147+
for (const taskId of taskIdsInOrder) {
148+
const suggestion = pendingSuggestions.get(taskId)
149+
assert.ok(suggestion, `Expected pending suggestion for task ${taskId}`)
150+
pendingSuggestions.delete(taskId)
151+
answeredSuggestions.add(suggestionKey(taskId, suggestion.answer))
152+
void globalThis.api.selectTaskFollowupSuggestion({ taskId, ...suggestion })
153+
}
154+
}
155+
156+
const messageHandler = ({ taskId, message }: { taskId: string; message: ClineMessage }) => {
157+
if (message.type === "ask" && message.ask === "followup" && message.text) {
158+
try {
159+
const parsed = JSON.parse(message.text) as { suggest?: Array<{ answer: string; mode?: string }> }
160+
const suggestion = parsed.suggest?.[0]
161+
162+
if (suggestion && !answeredSuggestions.has(suggestionKey(taskId, suggestion.answer))) {
163+
pendingSuggestions.set(taskId, suggestion)
164+
maybeReleaseRound()
165+
}
166+
} catch {
167+
// Ignore partial or malformed follow-up payloads.
168+
}
169+
}
170+
171+
if (message.type === "ask" && message.ask === "completion_result") {
172+
void globalThis.api.approveTaskAsk(taskId)
173+
}
174+
}
175+
const modeHandler = (taskId: string, mode: string) => {
176+
modeEvents.push({ taskId, mode })
177+
178+
if (roundInFlight && taskIdsInPlanOrder().every((id) => modeCountForTask(id) >= releasedRounds)) {
179+
roundInFlight = false
180+
maybeReleaseRound()
181+
}
182+
}
183+
184+
globalThis.api.on(RooCodeEventName.Message, messageHandler)
185+
globalThis.api.on(RooCodeEventName.TaskModeSwitched, modeHandler)
186+
187+
try {
188+
for (const [index, taskPlan] of plan.entries()) {
189+
if (index > 0) {
190+
await sleep(1_000)
191+
}
192+
193+
const taskId = await globalThis.api.startNewTask({
194+
configuration: {
195+
mode: "code",
196+
alwaysAllowModeSwitch: true,
197+
autoApprovalEnabled: true,
198+
apiKey: `followup-secret-${taskPlan.taskName}-must-not-persist`,
199+
},
200+
text: taskPlan.marker,
201+
newTab: true,
202+
preserveOpenTabs: index > 0,
203+
})
204+
taskIds.set(taskPlan.taskName, taskId)
205+
taskNamesById.set(taskId, taskPlan.taskName)
206+
maybeReleaseRound()
207+
}
208+
209+
await waitFor(
210+
() => {
211+
const expectedSwitches = plan.length * 10
212+
return modeEvents.length >= expectedSwitches
213+
},
214+
{ timeout: 30_000 },
215+
).catch((error) => {
216+
const counts = plan.map((taskPlan) => {
217+
const taskId = taskIds.get(taskPlan.taskName)
218+
return `${taskPlan.taskName}:${taskId ? modeCountForTask(taskId) : 0}`
219+
})
220+
throw new Error(
221+
`Timed out after ${releasedRounds} coordinated rounds; mode event counts: ${counts.join(", ")}; pending suggestions: ${pendingSuggestions.size}. ${error instanceof Error ? error.message : String(error)}`,
222+
)
223+
})
224+
225+
for (let roundIndex = 0; roundIndex < 10; roundIndex++) {
226+
const actualRoundModes = plan.map((taskPlan) => {
227+
const taskId = taskIds.get(taskPlan.taskName)
228+
assert.ok(taskId, `Expected task id for task ${taskPlan.taskName}`)
229+
return modeEvents.filter((event) => event.taskId === taskId).map((event) => event.mode)[roundIndex]
230+
})
231+
const expectedRoundModes = plan.map((taskPlan) => {
232+
const round = taskPlan.rounds[roundIndex]
233+
assert.ok(round, `Expected round ${roundIndex + 1} for task ${taskPlan.taskName}`)
234+
return round.mode
235+
})
236+
237+
assert.deepStrictEqual(
238+
actualRoundModes,
239+
expectedRoundModes,
240+
`Round ${roundIndex + 1} should count only after all three tasks switch once`,
241+
)
242+
}
243+
244+
for (const taskPlan of plan) {
245+
const taskId = taskIds.get(taskPlan.taskName)
246+
assert.ok(taskId, `Expected task id for task ${taskPlan.taskName}`)
247+
assert.deepStrictEqual(
248+
modeEvents.filter((event) => event.taskId === taskId).map((event) => event.mode),
249+
taskPlan.rounds.map((round) => round.mode),
250+
)
251+
}
252+
253+
const viewStates = globalThis.api.getGlobalState("viewStates") as GlobalState["viewStates"]
254+
assert.ok(viewStates, "Expected persisted viewStates to exist")
255+
256+
for (const [viewStateId, entry] of Object.entries(viewStates)) {
257+
const secretStatePath = findSecretStatePath(entry)
258+
assert.strictEqual(
259+
secretStatePath,
260+
undefined,
261+
`Persisted viewStates.${viewStateId} leaked secret state at ${secretStatePath}`,
262+
)
263+
}
264+
} finally {
265+
globalThis.api.off(RooCodeEventName.Message, messageHandler)
266+
globalThis.api.off(RooCodeEventName.TaskModeSwitched, modeHandler)
267+
}
268+
})
115269
})

‎packages/types/src/api.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ export interface RooCodeAPI extends EventEmitter<RooCodeAPIEvents> {
2626
text?: string
2727
images?: string[]
2828
newTab?: boolean
29+
preserveOpenTabs?: boolean
2930
}): Promise<string>
3031
/**
3132
* Resumes a task with the given ID.
@@ -96,6 +97,11 @@ export interface RooCodeAPI extends EventEmitter<RooCodeAPIEvents> {
9697
* Programmatically approves the pending ask for a task by ID. Intended for use in tests only.
9798
*/
9899
approveTaskAsk(taskId: string): Promise<boolean>
100+
/**
101+
* Simulates selecting a follow-up suggestion for a task by ID, including its optional mode switch.
102+
* Intended for use in tests only.
103+
*/
104+
selectTaskFollowupSuggestion(options: { taskId: string; answer: string; mode?: string }): Promise<boolean>
99105
/**
100106
* Returns true if the API is ready to use.
101107
*/

0 commit comments

Comments
 (0)