Fix recipe-backed fuzz suite resets (#1878)

Author: chubes4

packages/cli/src/commands/recipe-run-workflow-evidence.ts

@@ -3,6 +3,7 @@ import { readFile } from "node:fs/promises"
 import { resolve } from "node:path"
 import { commandArgValue, parseCommandJson, parseCommandJsonObject, RUNTIME_BACKED_FUZZ_SUITE_RUNNER_CAPABILITIES, runFuzzSuite, runtimeCheckpointUnsupportedDiagnostic, type ArtifactBundle, type ArtifactManifestFile, type ExecutionResult, type FuzzSuiteContract, type Runtime, type RuntimeCheckpointFailureDiagnostic, type RuntimeCheckpointOperation, type RuntimeCheckpointResult, type WorkspaceRecipe, type WorkspaceRecipeDistributionSetupArtifact, type WorkspaceRecipeDistributionStartupProbe, type WorkspaceRecipeProbe } from "@automattic/wp-codebox-core"
 import { stripUndefined } from "@automattic/wp-codebox-core/internals"
+import { createWordPressFuzzSuiteResetExecutor } from "@automattic/wp-codebox-playground/public"
 import { correlateObservedHostsToExternalServiceBoundaries, recipeExternalServiceBoundarySummaries } from "../recipe-external-services.js"
 import { recipeExecutionSpec, sandboxWorkspaceContract } from "../agent-sandbox.js"
 import { executeAgentFanoutFromArgs } from "../agent-fanout.js"
@@ -523,6 +524,7 @@ async function executeRunFuzzSuiteRecipeCommand(runtime: Runtime, args: string[]
   const result = await runFuzzSuite(suite, {
     runnerCapabilities: RUNTIME_BACKED_FUZZ_SUITE_RUNNER_CAPABILITIES,
     executor: async (spec) => executeRecipeWorkflowStep(runtime, { phase: "steps", index: 0, step: workflowStepFromExecutionSpec(spec) }, recipeDirectory, sandboxWorkspace, undefined, undefined, inputMountPathMap),
+    resetExecutor: createWordPressFuzzSuiteResetExecutor(recipeRuntimeFuzzSuiteResetEpisode(runtime, recipeDirectory, sandboxWorkspace, inputMountPathMap)),
     runtimeWorkloadExecutor: async ({ suite, workload, case: fuzzCase }) => {
       const workloadJson = JSON.stringify(workload)
       const execution = await executeWordPressRunWorkloadJsonRecipeCommand(runtime, [`workload-json=${workloadJson}`], recipeDirectory, sandboxWorkspace, suite, fuzzCase, inputMountPathMap)
@@ -552,6 +554,37 @@ async function executeRunFuzzSuiteRecipeCommand(runtime: Runtime, args: string[]
   }
 }
 
+function recipeRuntimeFuzzSuiteResetEpisode(runtime: Runtime, recipeDirectory: string, sandboxWorkspace: ReturnType<typeof sandboxWorkspaceContract> | undefined, inputMountPathMap: readonly InputMountPathMapping[]) {
+  let index = 0
+  return {
+    async step(action: { command: string; args?: string[]; metadata?: Record<string, unknown> }) {
+      index += 1
+      const id = `recipe-fuzz-reset-${index}`
+      const execution = await executeRecipeWorkflowStep(runtime, {
+        phase: "steps",
+        index,
+        step: { command: action.command, args: action.args, metadata: action.metadata },
+      }, recipeDirectory, sandboxWorkspace, undefined, undefined, inputMountPathMap)
+      return {
+        id,
+        index,
+        action: {
+          schema: "wp-codebox/runtime-episode-action/v1" as const,
+          id,
+          kind: "command" as const,
+          command: action.command,
+          args: action.args ?? [],
+          metadata: action.metadata,
+          digest: { algorithm: "sha256" as const, value: createHash("sha256").update(JSON.stringify(action)).digest("hex") },
+        },
+        actionRef: { kind: "action", id },
+        execution,
+        executionRef: { kind: "execution", id: execution.id },
+      }
+    },
+  }
+}
+
 async function fuzzSuiteFromRecipeCommandArgs(args: string[], recipeDirectory: string): Promise<FuzzSuiteContract> {
   const inline = commandArgValue(args, "input-json") ?? commandArgValue(args, "suite-json")
   if (inline) {

packages/runtime-playground/src/public.ts

@@ -213,7 +213,8 @@ export interface WordPressFuzzSuiteExecutionOptions extends Omit<FuzzSuiteRunOpt
   artifactBundles?: Array<Pick<ArtifactBundle, "id" | "directory">>
   artifactStorage?: RuntimeArtifactStorageInput | RuntimeArtifactStorageDescriptor
 }
-type WordPressFuzzSuiteResetEpisode = Pick<RuntimeEpisode, "reset" | "step"> & Partial<Pick<RuntimeEpisode, "restoreSnapshot">>
+type WordPressFuzzSuiteResetEpisode = Pick<RuntimeEpisode, "step"> & Partial<Pick<RuntimeEpisode, "restoreSnapshot">>
+type WordPressFuzzSuiteExecutionEpisode = WordPressFuzzSuiteResetEpisode & Pick<RuntimeEpisode, "reset">
 
 export interface WordPressFuzzSuiteResetExecutorOptions {
   artifactBundles?: Array<Pick<ArtifactBundle, "id" | "directory">>
@@ -1217,7 +1218,7 @@ export function createWordPressFuzzSuiteResetExecutor(episode: WordPressFuzzSuit
 }
 
 export async function executeWordPressFuzzSuite(
-  episode: WordPressFuzzSuiteResetEpisode,
+  episode: WordPressFuzzSuiteExecutionEpisode,
   suite: FuzzSuiteContract,
   options: WordPressFuzzSuiteExecutionOptions = {},
 ): Promise<FuzzSuiteResultEnvelope> {

tests/nested-fuzz-suite-recipe-command.test.ts

@@ -28,7 +28,17 @@ const recipe: WorkspaceRecipe = {
 assertWorkspaceRecipeJsonSchema(recipe, { recipeCommandIds: ["wp-codebox/run-fuzz-suite", "wordpress.run-php"] })
 
 const executed: ExecutionSpec[] = []
+let checkpointCreates = 0
+let checkpointRestores = 0
 const runtime = {
+  async createCheckpoint({ name }: { name: string }) {
+    checkpointCreates += 1
+    return { schema: "wp-codebox/runtime-checkpoint-result/v1" as const, status: "created", operation: "create" as const, checkpoint: { name, snapshotId: `snapshot-${name}`, createdAt: "2026-01-01T00:00:00.000Z" } }
+  },
+  async restoreCheckpoint(name: string) {
+    checkpointRestores += 1
+    return { schema: "wp-codebox/runtime-checkpoint-result/v1" as const, status: "restored", operation: "restore" as const, checkpoint: { name, snapshotId: `snapshot-${name}`, createdAt: "2026-01-01T00:00:00.000Z", restoredAt: "2026-01-01T00:00:01.000Z" } }
+  },
   async execute(spec: ExecutionSpec): Promise<ExecutionResult> {
     executed.push(spec)
     if (spec.command === "wordpress.run-php" && (spec.args ?? []).some((arg) => arg.includes("emit-rest-db-profile"))) {
@@ -104,6 +114,24 @@ assert.equal(result.schema, "wp-codebox/fuzz-suite-result/v1")
 assert.equal(result.status, "passed")
 assert.deepEqual(executed.map((spec) => spec.command), ["wordpress.run-php"])
 
+executed.length = 0
+const checkpointSuite = fuzzSuiteContract({
+  id: "nested-checkpoint-suite",
+  resetPolicy: { mode: "checkpoint-per-case", checkpointName: "nested-baseline" },
+  cases: [
+    { id: "checkpoint-case-one", target: { kind: "command", id: "wordpress.run-php", entrypoint: "wordpress.run-php" }, input: { args: ["code=echo 'one';"] } },
+    { id: "checkpoint-case-two", target: { kind: "command", id: "wordpress.run-php", entrypoint: "wordpress.run-php" }, input: { args: ["code=echo 'two';"] } },
+  ],
+})
+const checkpointExecution = await executeRecipeWorkflowStep(runtime, { phase: "steps", index: 0, step: { command: "wp-codebox/run-fuzz-suite", args: [`input-json=${JSON.stringify(checkpointSuite)}`] } }, process.cwd())
+const checkpointResult = JSON.parse(checkpointExecution.stdout)
+assert.equal(checkpointExecution.exitCode, 0)
+assert.equal(checkpointResult.status, "passed")
+assert.equal(checkpointCreates, 1)
+assert.equal(checkpointRestores, 2)
+assert.deepEqual(checkpointResult.cases.map((fuzzCase: { id: string; reset: { mode: string; status: string } }) => [fuzzCase.id, fuzzCase.reset.mode, fuzzCase.reset.status]), [["checkpoint-case-one", "checkpoint-per-case", "passed"], ["checkpoint-case-two", "checkpoint-per-case", "passed"]])
+assert.deepEqual(executed.map((spec) => spec.command), ["wordpress.run-php", "wordpress.run-php"])
+
 const suiteAliasDir = await mkdtemp(join(tmpdir(), "wp-codebox-nested-fuzz-suite-alias-"))
 await writeFile(join(suiteAliasDir, "suite.json"), JSON.stringify(suite), "utf8")
 const suiteAliasRecipe: WorkspaceRecipe = {