Harden managed service timeout cleanup

Author: chubes4

packages/cli/src/commands/recipe-run-finalizer.ts

@@ -353,6 +353,7 @@ function classifyRunResourceFailure(status: RuntimeRunRecord["status"], failure:
 
 function classifyRecipePhaseFailure(phase: string): string {
   switch (phase) {
+    case "provision_runtime_services":
     case "runtime_startup":
     case "run_blueprint_steps":
       return "startup"

packages/cli/src/commands/recipe-run.ts

@@ -29,7 +29,7 @@ import { RecipePhaseError } from "./recipe-run-phases.js"
 import { markPreviewLeaseAvailable, markPreviewLeaseFailed, markPreviewLeaseReleased, startPreviewLeaseRecipeRun } from "./preview-lease.js"
 import { importRecipeSiteSeeds } from "./recipe-site-seeds.js"
 import { applyRecipeRuntimeSetup, cleanupInputMountBaselines, prepareRecipeRuntimeSetup, recipeRunDependencyOverlay, recipeRunExtraPlugin, recipeRunStagedFile, rewriteInputMountPathArgs } from "./recipe-runtime-setup.js"
-import { provisionRuntimeServices, runtimeServiceEvidenceFromError, type RuntimeServiceEvidence } from "../runtime-services.js"
+import { provisionRuntimeServices, provisionRuntimeServicesForRecipe, runtimeServiceEvidenceFromError, type RuntimeServiceEvidence } from "../runtime-services.js"
 import { distributionStartupProbeFailure, executeRecipeCollectWorkloadResult, executeRecipeWorkflowStep, recipeAdvisoryFailure, recipeBrowserEvidence, recipeStepFailure, recipeWorkflowArgsEvidence, recipeWorkflowStepIsAdvisory, runDistributionSetupArtifacts, runDistributionStartupProbes, runRecipeProbes, withRecipeExecutionPhase } from "./recipe-run-workflow-evidence.js"
 import type { RecipeAdvisoryFailure, RecipeBrowserEvidence, RecipeDiagnosticArtifactRef, RecipeEffectiveRecipeArtifact, RecipeExecutionResult, RecipeFuzzCaseCommandRef, RecipeFuzzCaseResult, RecipeFuzzCaseStatus, RecipeFuzzRunResult, RecipeInterruptionController, RecipePhaseEvidence, RecipePhaseName, RecipePhpWasmRuntimeDiagnostic, RecipeRunCommandOutput, RecipeRunComponentContract, RecipeRunDeclaredArtifact, RecipeRunDistributionSetupArtifact, RecipeRunDistributionStartupProbe, RecipeRunFixtureDatabase, RecipeRunOptions, RecipeRunOutput, RecipeRunProbe, RecipeRunProvenance, RecipeRunStagedFile, RecipeRuntimeDiagnostic, RecipeStepFailure, RecipeValidateOptions, RecipeValidateOutput } from "./recipe-run-types.js"
 
@@ -162,7 +162,13 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
   let startupDurationMs: number | undefined
   let cleanupEvidence: RunResourceCleanupEvidence | undefined
   let managedServices: Awaited<ReturnType<typeof provisionRuntimeServices>> | undefined
+  let managedServicesReleased = false
   let serviceEvidence: RuntimeServiceEvidence[] = []
+  const releaseManagedServices = async (): Promise<void> => {
+    if (!managedServices || managedServicesReleased) return
+    await managedServices.release()
+    managedServicesReleased = true
+  }
   let runtimeDestroyed = false
   const destroyActiveRuntime = async (): Promise<void> => {
     if (!runtime || runtimeDestroyed) {
@@ -183,7 +189,11 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
     interruption?.throwIfInterrupted()
 
     runRecord = await runRegistry.update(runRecord.runId, { status: "booting" })
-    managedServices = await phaseTracker.run("provision_runtime_services", { services: recipe.inputs?.services?.map(({ id, kind }) => ({ id, kind })) ?? [] }, async () => await awaitRecipe("runtime-services.provision", () => provisionRuntimeServices(recipe.inputs?.services ?? [], { signal: interruption?.signal })))
+    managedServices = await phaseTracker.run("provision_runtime_services", { services: recipe.inputs?.services?.map(({ id, kind }) => ({ id, kind })) ?? [] }, async () => await provisionRuntimeServicesForRecipe(
+      recipe.inputs?.services ?? [],
+      async (provisioning) => await awaitRecipe("runtime-services.provision", provisioning),
+      { signal: interruption?.signal, onEvidence: (evidence) => { serviceEvidence = evidence } },
+    ))
     serviceEvidence = managedServices.evidence
     Object.assign(runtimeEnv, managedServices.env)
     const runtimeEnvironment = {
@@ -367,7 +377,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
           interruption.clear()
         }
       })
-      await managedServices?.release()
+      await releaseManagedServices()
       await cleanupRecipePreparedSources(workspaceMounts, extraPlugins, stagedFiles, overlays, dependencyOverlays)
       await cleanupInputMountBaselines(inputMountBaselinePaths)
     })
@@ -507,7 +517,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
     }
 
     cleanupEvidence = await runManagedServiceCleanup(runRegistry, runRecord, serviceEvidence, true, async () => {
-      await managedServices?.release()
+      await releaseManagedServices()
       await cleanupRecipePreparedSources(workspaceMounts, extraPlugins, stagedFiles, overlays, dependencyOverlays)
       await cleanupInputMountBaselines(inputMountBaselinePaths)
     })
@@ -548,6 +558,10 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
       },
     })
   } finally {
+    if (managedServices && !managedServicesReleased) {
+      await managedServices.release().catch(() => undefined)
+      await runRegistry.update(runRecord.runId, { metadata: { managedRuntimeServices: managedServices.evidence } }).catch(() => undefined)
+    }
     cancellationWatcher?.dispose()
   }
 }

packages/cli/src/recipe-validation.ts

@@ -702,7 +702,11 @@ export async function validateWorkspaceRecipeSemantics(recipe: WorkspaceRecipe,
 
 function validateRecipeRuntimeServices(recipe: WorkspaceRecipe, addIssue: (code: string, path: string, message: string) => void): void {
   const ids = new Set<string>()
-  const environment = new Set<string>(Object.keys(recipe.inputs?.runtimeEnv ?? {}))
+  const environment = new Set<string>([
+    ...Object.keys(recipe.distribution?.env ?? {}),
+    ...Object.keys(recipe.inputs?.runtimeEnv ?? {}),
+    ...(recipe.inputs?.secretEnv ?? []),
+  ])
   for (const [index, service] of (recipe.inputs?.services ?? []).entries()) {
     const path = `$.inputs.services[${index}]`
     if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(service.id)) addIssue("invalid-runtime-service-id", `${path}.id`, "Runtime service ids must be stable identifiers.")

packages/cli/src/runtime-services.ts

@@ -78,7 +78,7 @@ export async function provisionRuntimeServices(services: WorkspaceRecipeRuntimeS
       provisioned.push(managed)
     }
   } catch (error) {
-    await releaseServices(provisioned)
+    await releaseServices(provisioned).catch(() => undefined)
     if (error instanceof RuntimeServiceProvisionError) throw error
     throw new RuntimeServiceProvisionError("Managed runtime service provisioning failed", evidence)
   }
@@ -101,6 +101,37 @@ export async function provisionRuntimeServices(services: WorkspaceRecipeRuntimeS
   }
 }
 
+export async function provisionRuntimeServicesForRecipe(
+  services: WorkspaceRecipeRuntimeService[],
+  guard: <T>(promise: Promise<T>) => Promise<T>,
+  options: { signal?: AbortSignal; dependencies?: RuntimeServiceDependencies; onEvidence?: (evidence: RuntimeServiceEvidence[]) => void } = {},
+): Promise<Awaited<ReturnType<typeof provisionRuntimeServices>>> {
+  const controller = new AbortController()
+  const abort = () => controller.abort()
+  options.signal?.addEventListener("abort", abort, { once: true })
+  if (options.signal?.aborted) controller.abort()
+  const provisioning = provisionRuntimeServices(services, { signal: controller.signal, dependencies: options.dependencies })
+  try {
+    return await guard(provisioning)
+  } catch (error) {
+    controller.abort()
+    try {
+      const provisioned = await provisioning
+      try {
+        await provisioned.release()
+      } finally {
+        options.onEvidence?.(provisioned.evidence)
+      }
+    } catch (provisionError) {
+      const evidence = runtimeServiceEvidenceFromError(provisionError)
+      if (evidence) options.onEvidence?.(evidence)
+    }
+    throw error
+  } finally {
+    options.signal?.removeEventListener("abort", abort)
+  }
+}
+
 const mysqlDockerProvider: RuntimeServiceProvider = {
   name: "docker",
   kind: "mysql",
@@ -118,7 +149,7 @@ async function provisionMysqlDockerService(service: WorkspaceRecipeRuntimeServic
   evidenceList.push(evidence)
   const container = `wp-codebox-${service.id}-${dependencies.randomBytes(6).toString("hex")}`
   const password = dependencies.randomBytes(24).toString("base64url")
-  const childEnvironment = { PATH: process.env.PATH, MYSQL_DATABASE: "runtime", MYSQL_USER: "runtime", MYSQL_PASSWORD: password, MYSQL_ROOT_PASSWORD: password }
+  const childEnvironment = { ...process.env, MYSQL_DATABASE: "runtime", MYSQL_USER: "runtime", MYSQL_PASSWORD: password, MYSQL_ROOT_PASSWORD: password }
   const runArgs = ["run", "--detach", "--rm", "--name", container, "--publish", "127.0.0.1::3306", "--tmpfs", "/var/lib/mysql", "--env", "MYSQL_DATABASE", "--env", "MYSQL_USER", "--env", "MYSQL_PASSWORD", "--env", "MYSQL_ROOT_PASSWORD", MYSQL_IMAGE]
   let started = false
   try {
@@ -164,14 +195,25 @@ async function releaseService(container: string, evidence: RuntimeServiceEvidenc
     await dependencies.execute("docker", ["rm", "--force", container], { signal, timeout: 30_000 })
     evidence.lifecycle = "released"
     evidence.teardown = "completed"
-  } catch {
+  } catch (error) {
+    if (dockerContainerIsAbsent(error)) {
+      evidence.lifecycle = "released"
+      evidence.teardown = "completed"
+      return
+    }
     evidence.lifecycle = "failed"
     evidence.teardown = "failed"
     evidence.diagnostic = { code: "teardown-failed" }
     throw new Error(`Managed runtime service teardown failed: ${evidence.id}`)
   }
 }
 
+function dockerContainerIsAbsent(error: unknown): boolean {
+  if (!(error instanceof Error)) return false
+  const stderr = "stderr" in error && typeof error.stderr === "string" ? error.stderr : ""
+  return /No such container/i.test(`${error.message}\n${stderr}`)
+}
+
 export function parseLoopbackPort(output: string): number {
   const match = output.trim().match(/^127\.0\.0\.1:(\d+)$/m)
   const port = match ? Number(match[1]) : NaN

tests/runtime-services.test.ts

@@ -1,7 +1,9 @@
 import assert from "node:assert/strict"
 import { createServer } from "node:net"
-import { parseLoopbackPort, provisionRuntimeServices, RuntimeServiceProvisionError, runtimeServiceEvidenceFromError, runtimeServicePlan, waitForMysqlProtocol, type RuntimeServiceDependencies } from "../packages/cli/src/runtime-services.ts"
-import { validateWorkspaceRecipeJsonSchema } from "../packages/runtime-core/src/recipe-schema.ts"
+import { parseLoopbackPort, provisionRuntimeServices, provisionRuntimeServicesForRecipe, RuntimeServiceProvisionError, runtimeServiceEvidenceFromError, runtimeServicePlan, waitForMysqlProtocol, type RuntimeServiceDependencies } from "../packages/cli/src/runtime-services.ts"
+import { planWorkspaceRecipe } from "../packages/cli/src/recipe-dry-run.ts"
+import { validateWorkspaceRecipeSemantics } from "../packages/cli/src/recipe-validation.ts"
+import { validateWorkspaceRecipeJsonSchema, type WorkspaceRecipe } from "../packages/runtime-core/src/index.ts"
 
 const service = { id: "test-db", kind: "mysql", outputs: { host: "DB_HOST", port: "DB_PORT", password: "DB_PASSWORD" } } as const
 const plan = runtimeServicePlan([service])
@@ -13,6 +15,22 @@ const valid = validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-
 assert.equal(valid.valid, true)
 const unsafe = validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [{ ...service, outputs: { port: "bad-name" } }] }, workflow: { steps: [{ command: "wordpress.run-php" }] } })
 assert.equal(unsafe.valid, false)
+const recipe: WorkspaceRecipe = { schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [service] }, workflow: { steps: [{ command: "wordpress.run-php", args: ["code=echo 'ok';"] }] } }
+assert.deepEqual(await validateWorkspaceRecipeSemantics(recipe, "recipe.json"), [])
+const dryRun = await planWorkspaceRecipe(recipe, process.cwd(), { recipePath: "recipe.json" }, {
+  defaultWordPressVersion: "latest",
+  resolveExecutionSpec: async (step) => ({ command: step.command, args: step.args ?? [] }),
+})
+assert.deepEqual(dryRun.services, plan)
+const collisions: WorkspaceRecipe = {
+  ...recipe,
+  distribution: { name: "fixture", wordpress: { root: "/wordpress" }, env: { DB_HOST: "distribution" } },
+  inputs: { runtimeEnv: { DB_PORT: "3306" }, secretEnv: ["DB_PASSWORD"], services: [service] },
+}
+assert.deepEqual(
+  (await validateWorkspaceRecipeSemantics(collisions, "recipe.json")).map((issue) => issue.code),
+  ["duplicate-runtime-service-env", "duplicate-runtime-service-env", "duplicate-runtime-service-env"],
+)
 
 const server = createServer((socket) => socket.end(Buffer.from([1, 0, 0, 0, 10])))
 await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve))
@@ -44,8 +62,12 @@ assert.equal(provisioned.env.DB_PORT, "41001")
 assert.equal(provisioned.env.DB_PASSWORD, Buffer.alloc(24, 7).toString("base64url"))
 const runCall = calls.find((call) => call.args[0] === "run")
 assert.ok(runCall?.args.includes("MYSQL_PASSWORD"))
+assert.ok(runCall?.args.includes("127.0.0.1::3306"), "Docker publishes MySQL on a loopback ephemeral port")
+assert.deepEqual(runCall?.args.slice(runCall.args.indexOf("--tmpfs"), runCall.args.indexOf("--tmpfs") + 2), ["--tmpfs", "/var/lib/mysql"])
+assert.equal(runCall?.args.includes("--volume") || runCall?.args.includes("--mount"), false, "Docker uses no persistent volume")
 assert.equal(runCall?.args.some((arg) => arg.includes(provisioned.env.DB_PASSWORD)), false, "credentials never enter Docker argv")
 assert.equal(JSON.stringify(provisioned.evidence).includes(provisioned.env.DB_PASSWORD), false, "credentials never enter evidence")
+assert.equal(runCall?.env?.DOCKER_HOST, process.env.DOCKER_HOST, "Docker provider context is preserved")
 assert.equal(calls[0]?.args[0], "image", "the provider checks the image before starting the service")
 await provisioned.release()
 await provisioned.release()
@@ -58,6 +80,40 @@ await provisionedBeforeAbort.release()
 const cleanupCall = calls.filter((call) => call.args[0] === "rm").at(-1)
 assert.equal(cleanupCall?.signal, undefined, "teardown has an independent cleanup context after interruption")
 
+let finishLateProvisioning: (() => void) | undefined
+let lateRemoval = false
+const lateDependencies: RuntimeServiceDependencies = {
+  ...dependencies,
+  async execute(command, args, options) {
+    if (args[0] === "run") await new Promise<void>((resolve) => { finishLateProvisioning = resolve })
+    if (args[0] === "rm") lateRemoval = true
+    return dependencies.execute(command, args, options)
+  },
+}
+const guardedProvisioning = provisionRuntimeServicesForRecipe([service], async () => {
+  await new Promise<void>((resolve) => setTimeout(resolve, 5))
+  throw new Error("recipe timeout")
+}, { dependencies: lateDependencies })
+while (!finishLateProvisioning) await new Promise<void>((resolve) => setTimeout(resolve, 1))
+finishLateProvisioning()
+await assert.rejects(guardedProvisioning, /recipe timeout/)
+assert.equal(lateRemoval, true, "a timeout waits for and tears down late provisioning")
+
+const absentDependencies: RuntimeServiceDependencies = {
+  ...dependencies,
+  async execute(command, args, options) {
+    if (args[0] === "rm") {
+      const error = new Error("docker rm failed") as Error & { stderr: string }
+      error.stderr = `Error response from daemon: No such container: fixture`
+      throw error
+    }
+    return dependencies.execute(command, args, options)
+  },
+}
+const alreadyAbsent = await provisionRuntimeServices([service], { dependencies: absentDependencies })
+await alreadyAbsent.release()
+assert.equal(alreadyAbsent.evidence[0]?.teardown, "completed", "an already absent container is idempotently released")
+
 let failedCleanup = false
 const failingDependencies: RuntimeServiceDependencies = {
   ...dependencies,