Ensure disposable service teardown survives interruption

Author: chubes4

package.json

@@ -202,6 +202,7 @@
     "test:provider-runtime-contracts": "tsx tests/provider-runtime-contracts.test.ts",
     "test:runtime-requirements-readiness": "tsx tests/runtime-requirements-readiness.test.ts",
     "test:runtime-services": "tsx tests/runtime-services.test.ts",
+    "test:runtime-services-lifecycle": "tsx tests/runtime-services-lifecycle.test.ts",
     "test:disposable-mysql-mysqli-e2e": "tsx tests/disposable-mysql-mysqli.integration.test.ts",
     "test:runtime-contract-manifest": "tsx tests/runtime-contract-manifest.test.ts",
     "test:runtime-contract-package-exports": "tsx tests/runtime-contract-package-exports.test.ts",

packages/cli/src/commands/recipe-run.ts

@@ -371,6 +371,7 @@ async function runRecipe(options: RecipeRunOptions, interruption?: RecipeInterru
       await cleanupRecipePreparedSources(workspaceMounts, extraPlugins, stagedFiles, overlays, dependencyOverlays)
       await cleanupInputMountBaselines(inputMountBaselinePaths)
     })
+    runRecord = await runRegistry.update(runRecord.runId, { metadata: { managedRuntimeServices: serviceEvidence } })
     runRecord = await runRegistry.read(runRecord.runId)
     interruption?.throwIfInterrupted()
 
@@ -510,6 +511,7 @@ async function runRecipe(options: RecipeRunOptions, interruption?: RecipeInterru
       await cleanupRecipePreparedSources(workspaceMounts, extraPlugins, stagedFiles, overlays, dependencyOverlays)
       await cleanupInputMountBaselines(inputMountBaselinePaths)
     })
+    runRecord = await runRegistry.update(runRecord.runId, { metadata: { managedRuntimeServices: serviceEvidence } })
     runRecord = await runRegistry.read(runRecord.runId)
     const fuzzRunResult = recipeFuzzRunResult(recipe, executions)
     return await finalizeRecoveredRecipeFailure({

packages/cli/src/runtime-services.ts

@@ -37,14 +37,24 @@ export interface RuntimeServiceDependencies {
   randomBytes(size: number): Buffer
 }
 
+export interface RuntimeServiceProvider {
+  readonly name: string
+  readonly kind: string
+  readonly version: string
+  provision(service: WorkspaceRecipeRuntimeService, dependencies: RuntimeServiceDependencies, signal: AbortSignal | undefined, evidence: RuntimeServiceEvidence[]): Promise<ManagedRuntimeService>
+}
+
 const defaultDependencies: RuntimeServiceDependencies = {
   execute: async (command, args, options) => await execFileAsync(command, args, options),
   waitForReady: waitForMysqlProtocol,
   randomBytes,
 }
 
 export function runtimeServicePlan(services: WorkspaceRecipeRuntimeService[]): Array<{ id: string; kind: string; provider: string; version: string; bind: "loopback"; port: "ephemeral"; persistentVolume: false; outputs: Record<string, string> }> {
-  return services.map((service) => ({ id: service.id, kind: service.kind, provider: "docker", version: MYSQL_IMAGE, bind: "loopback", port: "ephemeral", persistentVolume: false, outputs: service.outputs }))
+  return services.map((service) => {
+    const provider = runtimeServiceProvider(service.kind)
+    return { id: service.id, kind: service.kind, provider: provider.name, version: provider.version, bind: "loopback", port: "ephemeral", persistentVolume: false, outputs: service.outputs }
+  })
 }
 
 export async function provisionRuntimeServices(services: WorkspaceRecipeRuntimeService[], options: { signal?: AbortSignal; dependencies?: RuntimeServiceDependencies } = {}): Promise<{ env: Record<string, string>; evidence: RuntimeServiceEvidence[]; release(): Promise<void> }> {
@@ -53,7 +63,7 @@ export async function provisionRuntimeServices(services: WorkspaceRecipeRuntimeS
   const evidence: RuntimeServiceEvidence[] = []
   try {
     for (const service of services) {
-      const managed = await provisionRuntimeService(service, dependencies, options.signal, evidence)
+      const managed = await runtimeServiceProvider(service.kind).provision(service, dependencies, options.signal, evidence)
       provisioned.push(managed)
     }
   } catch (error) {
@@ -71,15 +81,21 @@ export async function provisionRuntimeServices(services: WorkspaceRecipeRuntimeS
   }
 }
 
-async function provisionRuntimeService(service: WorkspaceRecipeRuntimeService, dependencies: RuntimeServiceDependencies, signal: AbortSignal | undefined, evidenceList: RuntimeServiceEvidence[]): Promise<ManagedRuntimeService> {
+const mysqlDockerProvider: RuntimeServiceProvider = {
+  name: "docker",
+  kind: "mysql",
+  version: MYSQL_IMAGE,
+  provision: provisionMysqlDockerService,
+}
+
+function runtimeServiceProvider(kind: string): RuntimeServiceProvider {
+  if (kind === mysqlDockerProvider.kind) return mysqlDockerProvider
+  throw new Error(`Unsupported managed runtime service kind: ${kind}`)
+}
+
+async function provisionMysqlDockerService(service: WorkspaceRecipeRuntimeService, dependencies: RuntimeServiceDependencies, signal: AbortSignal | undefined, evidenceList: RuntimeServiceEvidence[]): Promise<ManagedRuntimeService> {
   const evidence: RuntimeServiceEvidence = { id: service.id, kind: service.kind, provider: "docker", version: MYSQL_IMAGE, readiness: "pending", lifecycle: "provisioning" }
   evidenceList.push(evidence)
-  if (service.kind !== "mysql") {
-    evidence.lifecycle = "failed"
-    evidence.diagnostic = { code: "provision-failed" }
-    throw new RuntimeServiceProvisionError(`Unsupported managed runtime service kind: ${service.kind}`, evidenceList)
-  }
-
   const container = `wp-codebox-${service.id}-${dependencies.randomBytes(6).toString("hex")}`
   const password = dependencies.randomBytes(24).toString("base64url")
   const childEnvironment = { PATH: process.env.PATH, MYSQL_DATABASE: "runtime", MYSQL_USER: "runtime", MYSQL_PASSWORD: password, MYSQL_ROOT_PASSWORD: password }
@@ -96,7 +112,7 @@ async function provisionRuntimeService(service: WorkspaceRecipeRuntimeService, d
     evidence.readiness = "ready"
     evidence.lifecycle = "provisioned"
     const values: Record<string, string> = { host: "127.0.0.1", port: String(port), username: "runtime", password, database: "runtime" }
-    return { env: Object.fromEntries(Object.entries(service.outputs).map(([output, name]) => [name, values[output] ?? ""])), evidence, async release() { await releaseService(container, evidence, dependencies, signal) } }
+    return { env: Object.fromEntries(Object.entries(service.outputs).map(([output, name]) => [name, values[output] ?? ""])), evidence, async release() { await releaseService(container, evidence, dependencies) } }
   } catch (error) {
     evidence.readiness = "failed"
     evidence.lifecycle = "failed"

scripts/smoke-manifest.ts

@@ -130,6 +130,7 @@ export const smokeGroups = {
     commands: [
       npmScript("build"),
       npmScript("test:runtime-services"),
+      npmScript("test:runtime-services-lifecycle"),
       npmScript("test:disposable-mysql-mysqli-e2e"),
     ],
   },

tests/runtime-services-lifecycle.test.ts

@@ -0,0 +1,34 @@
+import assert from "node:assert/strict"
+import { mkdtemp, rm } from "node:fs/promises"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { RuntimeRunRegistry } from "../packages/runtime-core/src/run-registry.ts"
+import { runRecipeCleanup } from "../packages/cli/src/commands/recipe-run-finalizer.ts"
+import type { RuntimeServiceEvidence } from "../packages/cli/src/runtime-services.ts"
+
+async function assertTerminalCleanup(terminal: string): Promise<void> {
+  const directory = await mkdtemp(join(tmpdir(), "wp-codebox-service-lifecycle-"))
+  try {
+    const registry = new RuntimeRunRegistry(directory)
+    const run = await registry.create({ runId: `service-${terminal}`, status: "running", metadata: {} })
+    const evidence: RuntimeServiceEvidence[] = [{ id: "mysql", kind: "mysql", provider: "test", version: "test", readiness: "ready", lifecycle: "provisioned" }]
+    let releases = 0
+    await runRecipeCleanup(registry, run, async () => {
+      releases += 1
+      evidence[0]!.lifecycle = "released"
+      evidence[0]!.teardown = "completed"
+    })
+    await registry.update(run.runId, { metadata: { managedRuntimeServices: evidence, terminal } })
+    const completed = await registry.read(run.runId)
+    assert.equal(releases, 1, `${terminal} releases services once`)
+    assert.deepEqual(completed.metadata.managedRuntimeServices, evidence, `${terminal} persists final service evidence`)
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+}
+
+for (const terminal of ["runtime-creation-failure", "workflow-failure", "timeout", "cancellation", "interruption", "success"]) {
+  await assertTerminalCleanup(terminal)
+}
+
+console.log("runtime service lifecycle cleanup tests passed")

tests/runtime-services.test.ts

@@ -22,11 +22,11 @@ await waitForMysqlProtocol("127.0.0.1", address.port, 250)
 await new Promise<void>((resolve) => server.close(() => resolve()))
 await assert.rejects(waitForMysqlProtocol("127.0.0.1", address.port, 25), /readiness timed out/)
 
-const calls: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []
+const calls: Array<{ args: string[]; env?: NodeJS.ProcessEnv; signal?: AbortSignal }> = []
 const dependencies: RuntimeServiceDependencies = {
   randomBytes: (size) => Buffer.alloc(size, 7),
   async execute(_command, args, options) {
-    calls.push({ args, env: options.env })
+    calls.push({ args, env: options.env, signal: options.signal })
     if (args[0] === "port") return { stdout: "127.0.0.1:41001\n" }
     return { stdout: "" }
   },
@@ -42,6 +42,13 @@ await provisioned.release()
 await provisioned.release()
 assert.equal(calls.filter((call) => call.args[0] === "rm").length, 1, "release is idempotent")
 
+const interruptedAfterProvision = new AbortController()
+const provisionedBeforeAbort = await provisionRuntimeServices([service], { dependencies, signal: interruptedAfterProvision.signal })
+interruptedAfterProvision.abort()
+await provisionedBeforeAbort.release()
+const cleanupCall = calls.filter((call) => call.args[0] === "rm").at(-1)
+assert.equal(cleanupCall?.signal, undefined, "teardown has an independent cleanup context after interruption")
+
 let failedCleanup = false
 const failingDependencies: RuntimeServiceDependencies = {
   ...dependencies,