Persist managed service cleanup evidence

Author: chubes4

packages/cli/src/commands/recipe-run-finalizer.ts

@@ -17,6 +17,13 @@ export interface RunResourceCleanupEvidence {
   error?: RunOutput["error"]
 }
 
+export class RunResourceCleanupError extends Error {
+  constructor(readonly evidence: RunResourceCleanupEvidence, options: ErrorOptions) {
+    super("Recipe resource cleanup failed", options)
+    this.name = "RunResourceCleanupError"
+  }
+}
+
 interface RunResourceEvidenceOptions {
   startedAtMs: number
   status: RuntimeRunRecord["status"]
@@ -227,8 +234,10 @@ export async function runRecipeCleanup(runRegistry: RuntimeRunRegistry, runRecor
   } catch (error) {
     const updatedRunRecord = await runRegistry.update(runRecord.runId, { cleanup: { status: "failed", error: serializeError(error) } })
     const cleanupError = serializeRecipeRunError(error)
-    cleanupEvidenceFromRunRecord(updatedRunRecord, Date.now() - startedAtMs, cleanupError)
-    throw error
+    throw new RunResourceCleanupError(
+      cleanupEvidenceFromRunRecord(updatedRunRecord, Date.now() - startedAtMs, cleanupError),
+      { cause: error },
+    )
   }
 }
 

packages/cli/src/commands/recipe-run.ts

@@ -20,7 +20,7 @@ import { previewSpec, releaseRuntime, runtimeMetadata, type RunOutput } from "..
 import { artifactManifestFilesByPath, parseBenchResults, writeBenchmarkArtifactEvidence } from "./recipe-run-benchmark-artifacts.js"
 import { createRecipeRunContext } from "./recipe-run-context.js"
 import { collectRecipeDeclaredArtifacts, materializeTypedRecipeDeclaredArtifacts, recipeDeclaredArtifactFailure, recipeProbeFailure, recipeRuntimeEvidenceFiles } from "./recipe-declared-artifacts.js"
-import { completedRecipeOutputFields, finalizeCompletedRecipeRun, finalizeRecipeValidationFailure, finalizeRecoveredRecipeFailure, runRecipeCleanup, type RunResourceCleanupEvidence } from "./recipe-run-finalizer.js"
+import { completedRecipeOutputFields, finalizeCompletedRecipeRun, finalizeRecipeValidationFailure, finalizeRecoveredRecipeFailure, runRecipeCleanup, RunResourceCleanupError, type RunResourceCleanupEvidence } from "./recipe-run-finalizer.js"
 import { RecipeRunPhaseExecutor } from "./recipe-run-phase-executor.js"
 import { RecipeArtifactsMountConflictError, recipeArtifactsMountConflict } from "./recipe-run-artifacts-mount-guard.js"
 import { createRecipeInterruptionController, interruptedRecipeOutput, markRecipeArtifactsFinalized, recipeInterruptionSerializedError } from "./recipe-run-interruption.js"
@@ -353,7 +353,7 @@ async function runRecipe(options: RecipeRunOptions, interruption?: RecipeInterru
       await markPreviewLeaseAvailable(options.previewLeaseFile, { runId: runRecord.runId, preview: artifacts.preview, holdSeconds: options.previewHoldSeconds })
     }
     const activeRuntime = runtime
-    cleanupEvidence = await runRecipeCleanup(runRegistry, runRecord, async () => {
+    cleanupEvidence = await runManagedServiceCleanup(runRegistry, runRecord, serviceEvidence, false, async () => {
       await awaitRecipe("runtime.release", async () => {
         try {
           await releaseRuntime(activeRuntime, successfulRecipe && options.previewHoldBlocking ? options.previewHoldSeconds : 0, async () => {
@@ -371,7 +371,6 @@ async function runRecipe(options: RecipeRunOptions, interruption?: RecipeInterru
       await cleanupRecipePreparedSources(workspaceMounts, extraPlugins, stagedFiles, overlays, dependencyOverlays)
       await cleanupInputMountBaselines(inputMountBaselinePaths)
     })
-    runRecord = await runRegistry.update(runRecord.runId, { metadata: { managedRuntimeServices: serviceEvidence } })
     runRecord = await runRegistry.read(runRecord.runId)
     interruption?.throwIfInterrupted()
 
@@ -506,12 +505,11 @@ async function runRecipe(options: RecipeRunOptions, interruption?: RecipeInterru
       ])
     }
 
-    cleanupEvidence = await runRecipeCleanup(runRegistry, runRecord, async () => {
+    cleanupEvidence = await runManagedServiceCleanup(runRegistry, runRecord, serviceEvidence, true, async () => {
       await managedServices?.release()
       await cleanupRecipePreparedSources(workspaceMounts, extraPlugins, stagedFiles, overlays, dependencyOverlays)
       await cleanupInputMountBaselines(inputMountBaselinePaths)
     })
-    runRecord = await runRegistry.update(runRecord.runId, { metadata: { managedRuntimeServices: serviceEvidence } })
     runRecord = await runRegistry.read(runRecord.runId)
     const fuzzRunResult = recipeFuzzRunResult(recipe, executions)
     return await finalizeRecoveredRecipeFailure({
@@ -553,6 +551,27 @@ async function runRecipe(options: RecipeRunOptions, interruption?: RecipeInterru
   }
 }
 
+export async function runManagedServiceCleanup(
+  runRegistry: RuntimeRunRegistry,
+  runRecord: Awaited<ReturnType<RuntimeRunRegistry["read"]>>,
+  serviceEvidence: RuntimeServiceEvidence[],
+  preservePrimaryFailure: boolean,
+  cleanup: () => Promise<void>,
+): Promise<RunResourceCleanupEvidence> {
+  try {
+    return await runRecipeCleanup(runRegistry, runRecord, cleanup)
+  } catch (error) {
+    if (preservePrimaryFailure && error instanceof RunResourceCleanupError) {
+      return error.evidence
+    }
+    throw error
+  } finally {
+    await runRegistry.update(runRecord.runId, {
+      metadata: { managedRuntimeServices: serviceEvidence },
+    })
+  }
+}
+
 async function recipeArtifactsMountConflictFailure(options: RecipeRunOptions): Promise<RecipeRunOutput | undefined> {
   const recipePath = resolve(options.recipePath)
   const recipeDirectory = dirname(recipePath)

packages/cli/src/runtime-services.ts

@@ -129,7 +129,7 @@ async function releaseServices(services: ManagedRuntimeService[]): Promise<void>
 }
 
 async function releaseService(container: string, evidence: RuntimeServiceEvidence, dependencies: RuntimeServiceDependencies, signal?: AbortSignal): Promise<void> {
-  if (evidence.teardown) return
+  if (evidence.teardown === "completed") return
   try {
     await dependencies.execute("docker", ["rm", "--force", container], { signal, timeout: 30_000 })
     evidence.lifecycle = "released"

tests/runtime-services-lifecycle.test.ts

@@ -3,32 +3,42 @@ import { mkdtemp, rm } from "node:fs/promises"
 import { tmpdir } from "node:os"
 import { join } from "node:path"
 import { RuntimeRunRegistry } from "../packages/runtime-core/src/run-registry.ts"
-import { runRecipeCleanup } from "../packages/cli/src/commands/recipe-run-finalizer.ts"
+import { runManagedServiceCleanup } from "../packages/cli/src/commands/recipe-run.ts"
+import { RunResourceCleanupError } from "../packages/cli/src/commands/recipe-run-finalizer.ts"
 import type { RuntimeServiceEvidence } from "../packages/cli/src/runtime-services.ts"
 
-async function assertTerminalCleanup(terminal: string): Promise<void> {
-  const directory = await mkdtemp(join(tmpdir(), "wp-codebox-service-lifecycle-"))
-  try {
-    const registry = new RuntimeRunRegistry(directory)
-    const run = await registry.create({ runId: `service-${terminal}`, status: "running", metadata: {} })
-    const evidence: RuntimeServiceEvidence[] = [{ id: "mysql", kind: "mysql", provider: "test", version: "test", readiness: "ready", lifecycle: "provisioned" }]
-    let releases = 0
-    await runRecipeCleanup(registry, run, async () => {
-      releases += 1
-      evidence[0]!.lifecycle = "released"
-      evidence[0]!.teardown = "completed"
-    })
-    await registry.update(run.runId, { metadata: { managedRuntimeServices: evidence, terminal } })
-    const completed = await registry.read(run.runId)
-    assert.equal(releases, 1, `${terminal} releases services once`)
-    assert.deepEqual(completed.metadata.managedRuntimeServices, evidence, `${terminal} persists final service evidence`)
-  } finally {
-    await rm(directory, { recursive: true, force: true })
-  }
-}
+const directory = await mkdtemp(join(tmpdir(), "wp-codebox-service-lifecycle-"))
+try {
+  const registry = new RuntimeRunRegistry(directory)
+
+  const succeeded = await registry.create({ runId: "service-success", status: "running", metadata: {} })
+  const succeededEvidence: RuntimeServiceEvidence[] = [{ id: "mysql", kind: "mysql", provider: "test", version: "test", readiness: "ready", lifecycle: "provisioned" }]
+  const cleanup = await runManagedServiceCleanup(registry, succeeded, succeededEvidence, false, async () => {
+    succeededEvidence[0]!.lifecycle = "released"
+    succeededEvidence[0]!.teardown = "completed"
+  })
+  assert.equal(cleanup.state, "completed")
+  assert.deepEqual((await registry.read(succeeded.runId)).metadata.managedRuntimeServices, succeededEvidence)
+
+  const failed = await registry.create({ runId: "service-failure", status: "running", metadata: {} })
+  const failedEvidence: RuntimeServiceEvidence[] = [{ id: "mysql", kind: "mysql", provider: "test", version: "test", readiness: "ready", lifecycle: "provisioned" }]
+  const preserved = await runManagedServiceCleanup(registry, failed, failedEvidence, true, async () => {
+    failedEvidence[0]!.lifecycle = "failed"
+    failedEvidence[0]!.teardown = "failed"
+    failedEvidence[0]!.diagnostic = { code: "teardown-failed" }
+    throw new Error("fixture teardown failure")
+  })
+  assert.equal(preserved.state, "failed", "a primary recipe failure keeps structured cleanup evidence")
+  assert.deepEqual((await registry.read(failed.runId)).metadata.managedRuntimeServices, failedEvidence)
 
-for (const terminal of ["runtime-creation-failure", "workflow-failure", "timeout", "cancellation", "interruption", "success"]) {
-  await assertTerminalCleanup(terminal)
+  const terminal = await registry.create({ runId: "service-terminal-cleanup-failure", status: "running", metadata: {} })
+  await assert.rejects(
+    runManagedServiceCleanup(registry, terminal, [], false, async () => { throw new Error("fixture teardown failure") }),
+    (error: unknown) => error instanceof RunResourceCleanupError && error.evidence.state === "failed",
+    "cleanup failure becomes the terminal error when there is no earlier recipe failure",
+  )
+} finally {
+  await rm(directory, { recursive: true, force: true })
 }
 
 console.log("runtime service lifecycle cleanup tests passed")