test(vscode-e2e): cover orchestrator cancellation recovery

easonliang28 · easonliang28 · commit 4c5c41d03150 · 2026-07-31T01:58:43.000+08:00
diff --git a/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts b/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts
@@ -3,6 +3,11 @@ import { describe, expect, it } from "vitest"
 import {
 	ORCHESTRATOR_FAN_OUT_CHILD_STEPS,
 	ORCHESTRATOR_FAN_OUT_FINAL_RESULT,
+	ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP,
+	ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT,
+	ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER,
+	ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER,
+	ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT,
 	ORCHESTRATOR_FAN_OUT_PARENT_PROMPT,
 	ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT,
 	ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP,
@@ -14,10 +19,14 @@ import {
 	ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT,
 	ORCHESTRATOR_REPEATED_DELEGATION_MARKER,
 	ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT,
+	buildOrchestratorCancellationRecoveryResumeExpectations,
 	buildOrchestratorNestedChildResumeExpectations,
 	buildOrchestratorNestedParentResumeExpectations,
 	buildOrchestratorRepeatedResumeExpectations,
 	buildOrchestratorResumeExpectations,
+	shouldMatchOrchestratorCancellationChildCompletionRequest,
+	shouldMatchOrchestratorCancellationChildRequest,
+	shouldMatchOrchestratorCancellationRecoveryResumeRequest,
 	shouldMatchOrchestratorChildRequest,
 	shouldMatchOrchestratorNestedChildResumeRequest,
 	shouldMatchOrchestratorNestedParentResumeRequest,
@@ -276,4 +285,84 @@ describe("orchestrator fan-out delegation plan", () => {
 		}
 		expect(ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT).toContain(ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT)
 	})
+
+	it("defines a cancellation recovery parent/child plan with a pending ask child", () => {
+		expect(ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT).toContain(ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER)
+		expect(ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP).toEqual(
+			expect.objectContaining({
+				mode: "ask",
+				role: "cancellation-child",
+				summary: "Cancellation recovery child summary: resumed after interruption.",
+			}),
+		)
+		expect(ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.prompt).toContain("Ask the user exactly")
+		expect(ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.prompt).toContain(
+			ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER,
+		)
+	})
+
+	it("matches cancellation child requests without colliding with other orchestrator scenarios", () => {
+		expect(
+			shouldMatchOrchestratorCancellationChildRequest(
+				`child ${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker}`,
+			),
+		).toBe(true)
+		expect(shouldMatchOrchestratorCancellationChildRequest(ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT)).toBe(
+			false,
+		)
+		expect(
+			shouldMatchOrchestratorCancellationChildRequest(
+				`${ORCHESTRATOR_NESTED_DELEGATION_MARKER} embeds ${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker}`,
+			),
+		).toBe(false)
+	})
+
+	it("matches cancellation child completion only after the recovery answer", () => {
+		expect(
+			shouldMatchOrchestratorCancellationChildCompletionRequest(
+				`${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker} call_orchestrator_cancellation_child_followup_001 ${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}`,
+			),
+		).toBe(true)
+		expect(
+			shouldMatchOrchestratorCancellationChildCompletionRequest(
+				`${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker} without the answer`,
+			),
+		).toBe(false)
+		expect(
+			shouldMatchOrchestratorCancellationChildCompletionRequest(
+				`${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER} call_cancellation_child_followup_001 ${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}`,
+			),
+		).toBe(false)
+	})
+
+	it("matches cancellation recovery parent resumes only after child result injection", () => {
+		const expectations = buildOrchestratorCancellationRecoveryResumeExpectations()
+		expect(expectations).toEqual([
+			{
+				stepIndex: 1,
+				requiredSummaries: [ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary],
+				nextMode: undefined,
+			},
+		])
+
+		const parentResumeRequest = `${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER} completed.\\n\\nResult:\\n${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary}`
+		expect(
+			shouldMatchOrchestratorCancellationRecoveryResumeRequest(parentResumeRequest, [
+				ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary,
+			]),
+		).toBe(true)
+		expect(
+			shouldMatchOrchestratorCancellationRecoveryResumeRequest(
+				`${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER} completed.\\n\\nResult:\\nmissing`,
+				[ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary],
+			),
+		).toBe(false)
+	})
+
+	it("composes cancellation recovery final result with an explicit cancellation marker", () => {
+		expect(ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT).toContain("cancellation recovery")
+		expect(ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT).toContain(
+			ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary,
+		)
+	})
 })
diff --git a/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts b/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts
@@ -11,6 +11,7 @@ export type OrchestratorFanOutChildStep = {
 
 export type OrchestratorRepeatedDelegationRole = "requirements" | "design" | "implementation"
 export type OrchestratorNestedDelegationRole = "child-orchestrator" | "requirements" | "implementation"
+export type OrchestratorCancellationRecoveryRole = "cancellation-child"
 
 export type OrchestratorRepeatedDelegationChildStep = OrchestratorFanOutChildStep & {
 	readonly round: number
@@ -21,6 +22,11 @@ export type OrchestratorNestedDelegationStep = OrchestratorFanOutChildStep & {
 	readonly role: OrchestratorNestedDelegationRole
 }
 
+export type OrchestratorCancellationRecoveryStep = OrchestratorFanOutChildStep & {
+	readonly role: OrchestratorCancellationRecoveryRole
+	readonly followupAnswer: string
+}
+
 export type OrchestratorResumeExpectation = {
 	readonly stepIndex: number
 	readonly requiredSummaries: string[]
@@ -31,6 +37,7 @@ export const ORCHESTRATOR_FAN_OUT_MARKER = "ORCHESTRATOR_SINGLE_ROUND_FAN_OUT"
 export const ORCHESTRATOR_FAN_OUT_RESULT_INJECTION = "completed.\\n\\nResult:"
 export const ORCHESTRATOR_REPEATED_DELEGATION_MARKER = "ORCHESTRATOR_REPEATED_DELEGATION_STRESS"
 export const ORCHESTRATOR_NESTED_DELEGATION_MARKER = "ORCHESTRATOR_NESTED_DELEGATION"
+export const ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER = "ORCHESTRATOR_CANCELLATION_RECOVERY"
 
 export const ORCHESTRATOR_FAN_OUT_CHILD_STEPS: readonly OrchestratorFanOutChildStep[] = [
 	{
@@ -158,6 +165,23 @@ export const ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT = `${ORCHESTRATOR_NEST
 
 export const ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT = `Nested top-level orchestrator complete:\n- B: ${ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT}`
 
+export const ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER = "resume recovered cancellation child"
+
+export const ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP: OrchestratorCancellationRecoveryStep = {
+	role: "cancellation-child",
+	mode: "ask",
+	marker: "ORCH_CANCEL_RECOVERY_CHILD",
+	prompt: `ORCH_CANCEL_RECOVERY_CHILD: Ask the user exactly this follow-up question: Type "${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}" to recover the cancelled child. After the user answers, complete with the exact result "Cancellation recovery child summary: resumed after interruption."`,
+	summary: "Cancellation recovery child summary: resumed after interruption.",
+	followupAnswer: ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER,
+	newTaskToolCallId: "call_orchestrator_cancellation_parent_new_task_001",
+	completionToolCallId: "call_orchestrator_cancellation_child_completion_002",
+}
+
+export const ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT = `${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER}: Delegate exactly one ask-mode child B with this exact message: "${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.prompt}". Wait for B to return; do not complete before B returns. After B returns, complete with a final cancellation recovery summary containing B's result.`
+
+export const ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT = `Orchestrator cancellation recovery complete:\n- B: ${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary}`
+
 function buildResumeExpectations(steps: readonly OrchestratorFanOutChildStep[]): OrchestratorResumeExpectation[] {
 	return steps.map((_step, index) => ({
 		stepIndex: index + 1,
@@ -182,15 +206,55 @@ export function buildOrchestratorNestedParentResumeExpectations(): OrchestratorR
 	return buildResumeExpectations([ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP])
 }
 
+export function buildOrchestratorCancellationRecoveryResumeExpectations(): OrchestratorResumeExpectation[] {
+	return buildResumeExpectations([ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP])
+}
+
 export function shouldMatchOrchestratorChildRequest(rawRequest: string, childMarker: string): boolean {
 	return (
 		rawRequest.includes(childMarker) &&
 		!rawRequest.includes(ORCHESTRATOR_FAN_OUT_MARKER) &&
 		!rawRequest.includes(ORCHESTRATOR_REPEATED_DELEGATION_MARKER) &&
 		!rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_MARKER) &&
+		!rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER) &&
 		!rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.marker)
 	)
 }
+
+function shouldMatchOrchestratorCancellationChildBase(rawRequest: string): boolean {
+	return (
+		rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker) &&
+		!rawRequest.includes(`${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER}: Delegate exactly one ask-mode child B`) &&
+		!rawRequest.includes(ORCHESTRATOR_FAN_OUT_MARKER) &&
+		!rawRequest.includes(ORCHESTRATOR_REPEATED_DELEGATION_MARKER) &&
+		!rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_MARKER)
+	)
+}
+
+export function shouldMatchOrchestratorCancellationChildRequest(rawRequest: string): boolean {
+	const followupToolCallId = ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.completionToolCallId.replace(
+		"completion_002",
+		"followup_001",
+	)
+
+	return shouldMatchOrchestratorCancellationChildBase(rawRequest) && !rawRequest.includes(followupToolCallId)
+}
+
+export function shouldMatchOrchestratorCancellationChildCompletionRequest(rawRequest: string): boolean {
+	const followupToolCallId = ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.completionToolCallId.replace(
+		"completion_002",
+		"followup_001",
+	)
+
+	return (
+		shouldMatchOrchestratorCancellationChildBase(rawRequest) &&
+		((rawRequest.includes(followupToolCallId) &&
+			rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER)) ||
+			rawRequest.includes(
+				`<user_message>\\n${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}\\n</user_message>`,
+			))
+	)
+}
 function requestContainsResultSummary(rawRequest: string, summary: string): boolean {
 	const jsonEscapedSummary = summary.replaceAll("\n", "\\n")
 
@@ -240,3 +304,14 @@ export function shouldMatchOrchestratorNestedParentResumeRequest(
 		requiredSummaries.every((summary) => requestContainsResultSummary(rawRequest, summary))
 	)
 }
+
+export function shouldMatchOrchestratorCancellationRecoveryResumeRequest(
+	rawRequest: string,
+	requiredSummaries: readonly string[],
+): boolean {
+	return (
+		rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER) &&
+		rawRequest.includes(ORCHESTRATOR_FAN_OUT_RESULT_INJECTION) &&
+		requiredSummaries.every((summary) => requestContainsResultSummary(rawRequest, summary))
+	)
+}
diff --git a/apps/vscode-e2e/src/fixtures/orchestrator.ts b/apps/vscode-e2e/src/fixtures/orchestrator.ts
@@ -2,6 +2,11 @@ import { LLMock } from "@copilotkit/aimock"
 import type { ChatCompletionRequest } from "@copilotkit/aimock"
 
 import {
+	ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP,
+	ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT,
+	ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER,
+	ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER,
+	ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT,
 	ORCHESTRATOR_FAN_OUT_CHILD_STEPS,
 	ORCHESTRATOR_FAN_OUT_FINAL_RESULT,
 	ORCHESTRATOR_FAN_OUT_MARKER,
@@ -17,10 +22,14 @@ import {
 	ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT,
 	ORCHESTRATOR_REPEATED_DELEGATION_MARKER,
 	ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT,
+	buildOrchestratorCancellationRecoveryResumeExpectations,
 	buildOrchestratorNestedChildResumeExpectations,
 	buildOrchestratorNestedParentResumeExpectations,
 	buildOrchestratorRepeatedResumeExpectations,
 	buildOrchestratorResumeExpectations,
+	shouldMatchOrchestratorCancellationChildCompletionRequest,
+	shouldMatchOrchestratorCancellationChildRequest,
+	shouldMatchOrchestratorCancellationRecoveryResumeRequest,
 	shouldMatchOrchestratorChildRequest,
 	shouldMatchOrchestratorNestedChildResumeRequest,
 	shouldMatchOrchestratorNestedParentResumeRequest,
@@ -29,6 +38,11 @@ import {
 } from "./orchestrator-plan"
 
 export {
+	ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP,
+	ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT,
+	ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER,
+	ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER,
+	ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT,
 	ORCHESTRATOR_FAN_OUT_CHILD_STEPS,
 	ORCHESTRATOR_FAN_OUT_FINAL_RESULT,
 	ORCHESTRATOR_FAN_OUT_MARKER,
@@ -44,10 +58,14 @@ export {
 	ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT,
 	ORCHESTRATOR_REPEATED_DELEGATION_MARKER,
 	ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT,
+	buildOrchestratorCancellationRecoveryResumeExpectations,
 	buildOrchestratorNestedChildResumeExpectations,
 	buildOrchestratorNestedParentResumeExpectations,
 	buildOrchestratorRepeatedResumeExpectations,
 	buildOrchestratorResumeExpectations,
+	shouldMatchOrchestratorCancellationChildCompletionRequest,
+	shouldMatchOrchestratorCancellationChildRequest,
+	shouldMatchOrchestratorCancellationRecoveryResumeRequest,
 	shouldMatchOrchestratorChildRequest,
 	shouldMatchOrchestratorNestedChildResumeRequest,
 	shouldMatchOrchestratorNestedParentResumeRequest,
@@ -303,4 +321,79 @@ export function addOrchestratorFixtures(mock: InstanceType<typeof LLMock>) {
 			},
 		})
 	}
+
+	mock.addFixture({
+		match: {
+			userMessage: new RegExp(ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER),
+			sequenceIndex: 0,
+		},
+		response: {
+			toolCalls: [
+				{
+					name: "new_task",
+					arguments: JSON.stringify({
+						mode: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.mode,
+						message: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.prompt,
+					}),
+					id: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.newTaskToolCallId,
+				},
+			],
+		},
+	})
+
+	mock.addFixture({
+		match: {
+			predicate: (req: ChatCompletionRequest) =>
+				shouldMatchOrchestratorCancellationChildRequest(requestText(req)),
+		},
+		response: {
+			toolCalls: [
+				{
+					name: "ask_followup_question",
+					arguments: JSON.stringify({
+						question: `Type "${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.followupAnswer}" to recover the cancelled child.`,
+						follow_up: [{ text: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.followupAnswer }],
+					}),
+					id: "call_orchestrator_cancellation_child_followup_001",
+				},
+			],
+		},
+	})
+
+	mock.addFixture({
+		match: {
+			predicate: (req: ChatCompletionRequest) =>
+				shouldMatchOrchestratorCancellationChildCompletionRequest(requestText(req)),
+		},
+		response: {
+			toolCalls: [
+				{
+					name: "attempt_completion",
+					arguments: JSON.stringify({ result: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary }),
+					id: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.completionToolCallId,
+				},
+			],
+		},
+	})
+
+	for (const expectation of [...buildOrchestratorCancellationRecoveryResumeExpectations()].reverse()) {
+		mock.addFixture({
+			match: {
+				predicate: (req: ChatCompletionRequest) =>
+					shouldMatchOrchestratorCancellationRecoveryResumeRequest(
+						requestText(req),
+						expectation.requiredSummaries,
+					),
+			},
+			response: {
+				toolCalls: [
+					{
+						name: "attempt_completion",
+						arguments: JSON.stringify({ result: ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT }),
+						id: "call_orchestrator_cancellation_parent_completion_002",
+					},
+				],
+			},
+		})
+	}
 }
diff --git a/apps/vscode-e2e/src/suite/orchestrator.test.ts b/apps/vscode-e2e/src/suite/orchestrator.test.ts
