fix: diagnose unavailable MySQL runtime providers (#2116)

* fix: diagnose unavailable runtime service providers

* fix: clean up unavailable provider processes

* fix: preserve primary provider diagnostics

Author: chubes4

packages/cli/src/runtime-services.ts

@@ -30,7 +30,11 @@ export interface RuntimeServiceEvidence {
   readiness: "pending" | "ready" | "failed"
   lifecycle: "provisioning" | "provisioned" | "released" | "failed"
   teardown?: "completed" | "failed"
-  diagnostic?: { code: "readiness-failed" | "provision-failed" | "teardown-failed" | "interrupted" }
+  diagnostic?: {
+    code: "readiness-failed" | "provision-failed" | "provider-unavailable" | "teardown-failed" | "interrupted"
+    command?: string
+    cause?: { code: string; message: string }
+  }
   controls?: RuntimeServiceControlResult[]
   memory?: { budgetMiB: number; observedRssMiB?: number }
 }
@@ -300,12 +304,19 @@ async function provisionMysqlDockerService(service: WorkspaceRecipeRuntimeServic
   } catch (error) {
     evidence.readiness = "failed"
     evidence.lifecycle = "failed"
-    evidence.diagnostic = { code: signal?.aborted ? "interrupted" : started ? "readiness-failed" : "provision-failed" }
+    evidence.diagnostic = signal?.aborted
+      ? { code: "interrupted" }
+      : providerUnavailableDiagnostic(error, "docker") ?? { code: started ? "readiness-failed" : "provision-failed" }
     if (started) await releaseService(container, evidence, dependencies, undefined).catch(() => undefined)
     throw new RuntimeServiceProvisionError(`Managed runtime service failed: ${service.id}`, evidenceList)
   }
 }
 
+function providerUnavailableDiagnostic(error: unknown, command: string): RuntimeServiceEvidence["diagnostic"] | undefined {
+  if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT") return undefined
+  return { code: "provider-unavailable", command, cause: { code: "ENOENT", message: "Provider command executable was not found" } }
+}
+
 const NATIVE_MARIADB_ROOT_PREFIX = "wp-codebox-mariadb-"
 const NATIVE_MARIADB_START_ATTEMPTS = 5
 const NATIVE_MARIADB_START_TIMEOUT_MS = 30_000
@@ -1403,6 +1414,7 @@ async function waitForMysqlDatabase(container: string, engine: keyof typeof MYSQ
       return
     } catch (error) {
       if (signal?.aborted) throw error
+      if (providerUnavailableDiagnostic(error, "docker")) throw error
       await abortableDelay(100, signal)
     }
   }
@@ -1438,7 +1450,7 @@ async function releaseService(container: string, evidence: RuntimeServiceEvidenc
     }
     evidence.lifecycle = "failed"
     evidence.teardown = "failed"
-    evidence.diagnostic = { code: "teardown-failed" }
+    if (evidence.diagnostic?.code !== "provider-unavailable") evidence.diagnostic = { code: "teardown-failed" }
     throw new Error(`Managed runtime service teardown failed: ${evidence.id}`)
   }
 }
@@ -1548,9 +1560,10 @@ export function executeRuntimeServiceProcess(command: string, args: string[], op
     const child = spawn(command, args, {
       env: options.env,
       signal: options.signal,
-      timeout: options.timeout,
       stdio: ["pipe", "pipe", "pipe"],
     })
+    const timeout = setTimeout(() => child.kill(), options.timeout)
+    timeout.unref()
     const stdout: Buffer[] = []
     const stderr: Buffer[] = []
     let stdoutBytes = 0
@@ -1571,11 +1584,13 @@ export function executeRuntimeServiceProcess(command: string, args: string[], op
     child.once("error", (error) => {
       if (settled) return
       settled = true
+      clearTimeout(timeout)
       reject(error)
     })
     child.once("close", (code) => {
       if (settled) return
       settled = true
+      clearTimeout(timeout)
       const boundedStdout = Buffer.concat(stdout).toString("utf8")
       const boundedStderr = Buffer.concat(stderr).toString("utf8")
       if (overflow) {

tests/runtime-services.test.ts

@@ -1,10 +1,11 @@
 import assert from "node:assert/strict"
+import { spawnSync } from "node:child_process"
 import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"
 import { createServer } from "node:net"
 import { tmpdir } from "node:os"
 import { join } from "node:path"
 import { runRecipeBuildCommand } from "../packages/cli/src/commands/recipe-build.ts"
-import { parseLoopbackPort, provisionRuntimeServices, provisionRuntimeServicesForRecipe, RuntimeServiceProvisionError, runtimeServiceEvidenceFromError, runtimeServicePlan, waitForMysqlProtocol, type RuntimeServiceDependencies } from "../packages/cli/src/runtime-services.ts"
+import { executeRuntimeServiceProcess, parseLoopbackPort, provisionRuntimeServices, provisionRuntimeServicesForRecipe, RuntimeServiceProvisionError, runtimeServiceEvidenceFromError, runtimeServicePlan, waitForMysqlProtocol, type RuntimeServiceDependencies } from "../packages/cli/src/runtime-services.ts"
 import { planWorkspaceRecipe } from "../packages/cli/src/recipe-dry-run.ts"
 import { validateWorkspaceRecipeSemantics } from "../packages/cli/src/recipe-validation.ts"
 import { buildWordPressPhpunitRecipe } from "../packages/runtime-core/src/recipe-builders.ts"
@@ -139,6 +140,67 @@ await provisioned.release()
 await provisioned.release()
 assert.equal(calls.filter((call) => call.args[0] === "rm").length, 1, "release is idempotent")
 
+const missingProviderDependencies: RuntimeServiceDependencies = {
+  ...dependencies,
+  execute: async (_command, args, options) => await executeRuntimeServiceProcess("wp-codebox-provider-command-that-does-not-exist", args, options),
+}
+await assert.rejects(provisionRuntimeServices([service], { dependencies: missingProviderDependencies }), (error: unknown) => {
+  assert.ok(error instanceof RuntimeServiceProvisionError)
+  assert.deepEqual(error.evidence[0]?.diagnostic, {
+    code: "provider-unavailable",
+    command: "docker",
+    cause: { code: "ENOENT", message: "Provider command executable was not found" },
+  })
+  return true
+})
+const missingProviderChild = spawnSync(process.execPath, ["--import", "tsx", "--input-type=module", "-e", `
+  import { executeRuntimeServiceProcess } from "./packages/cli/src/runtime-services.ts";
+  await executeRuntimeServiceProcess("wp-codebox-provider-command-that-does-not-exist", [], { timeout: 300_000 }).catch(() => undefined);
+`], { cwd: process.cwd(), encoding: "utf8", timeout: 5_000 })
+assert.equal(missingProviderChild.error, undefined, `spawn ENOENT child exits without retaining its process timeout: ${missingProviderChild.error?.message ?? ""}`)
+assert.equal(missingProviderChild.status, 0, missingProviderChild.stderr)
+
+const failedStartDependencies: RuntimeServiceDependencies = {
+  ...dependencies,
+  async execute(command, args, options) {
+    if (args[0] === "run") throw new Error("provider start failed")
+    return dependencies.execute(command, args, options)
+  },
+}
+await assert.rejects(provisionRuntimeServices([service], { dependencies: failedStartDependencies }), (error: unknown) => error instanceof RuntimeServiceProvisionError && error.evidence[0]?.diagnostic?.code === "provision-failed")
+
+const failedReadinessDependencies: RuntimeServiceDependencies = {
+  ...dependencies,
+  async waitForReady() { throw new Error("provider readiness failed") },
+}
+await assert.rejects(provisionRuntimeServices([service], { dependencies: failedReadinessDependencies }), (error: unknown) => error instanceof RuntimeServiceProvisionError && error.evidence[0]?.diagnostic?.code === "readiness-failed")
+
+const unavailableDuringReadinessDependencies: RuntimeServiceDependencies = {
+  ...dependencies,
+  async execute(command, args, options) {
+    if (args[0] === "exec" || args[0] === "rm") return await executeRuntimeServiceProcess("wp-codebox-provider-command-that-does-not-exist", args, options)
+    return dependencies.execute(command, args, options)
+  },
+}
+await assert.rejects(provisionRuntimeServices([service], { dependencies: unavailableDuringReadinessDependencies }), (error: unknown) => {
+  assert.ok(error instanceof RuntimeServiceProvisionError)
+  assert.equal(error.evidence[0]?.diagnostic?.code, "provider-unavailable", "cleanup failure does not replace the primary provider failure")
+  assert.equal(error.evidence[0]?.teardown, "failed", "failed cleanup remains visible in service evidence")
+  return true
+})
+
+let ordinaryReadinessAttempts = 0
+const retryingReadinessDependencies: RuntimeServiceDependencies = {
+  ...dependencies,
+  async execute(command, args, options) {
+    if (args[0] === "exec" && ordinaryReadinessAttempts++ === 0) throw new Error("provider readiness command failed")
+    return dependencies.execute(command, args, options)
+  },
+}
+const retriedReadiness = await provisionRuntimeServices([service], { dependencies: retryingReadinessDependencies })
+assert.equal(ordinaryReadinessAttempts, 2, "ordinary Docker readiness failures remain retryable")
+await retriedReadiness.release()
+
 const emptyRootCalls: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []
 const emptyRootDependencies: RuntimeServiceDependencies = {
   ...dependencies,