Run distribution startup probes (#1237)

Author: chubes4

package.json

@@ -114,6 +114,7 @@
     "test:generic-ability-runtime-run": "tsx tests/generic-ability-runtime-run.test.ts",
     "test:provider-runtime-contracts": "tsx tests/provider-runtime-contracts.test.ts",
     "test:recipe-source-packages": "tsx tests/recipe-source-packages.test.ts",
+    "test:distribution-startup-probes": "tsx tests/distribution-startup-probes.test.ts",
     "test:path-policy-parity": "tsx tests/path-policy-parity.test.ts",
     "test:php-path-policy-parity": "php scripts/php-path-policy-parity-smoke.php",
     "test:production-boundary-enforcement": "tsx tests/production-boundary-enforcement.test.ts",

packages/cli/src/commands/recipe-declared-artifacts.ts

@@ -3,7 +3,7 @@ import { DEFAULT_CAPTURED_ARTIFACT_MAX_BYTES, STRUCTURED_ARTIFACT_SCHEMA, TYPED_
 import { stripUndefined } from "@automattic/wp-codebox-core/internals"
 import { appendRecipeRuntimeEvidenceFiles } from "../recipe-evidence.js"
 import { serializeRecipeRunError, RecipeDeclaredArtifactFailureError, RecipeProbeFailureError } from "./recipe-run-output.js"
-import type { RecipeRunDeclaredArtifact, RecipeRunFixtureDatabase, RecipeRunProbe } from "./recipe-run-types.js"
+import type { RecipeRunDeclaredArtifact, RecipeRunDistributionStartupProbe, RecipeRunFixtureDatabase, RecipeRunProbe } from "./recipe-run-types.js"
 
 const DECLARED_ARTIFACT_CAPTURE_MAX_BYTES = DEFAULT_CAPTURED_ARTIFACT_MAX_BYTES
 const declaredArtifactContents = new WeakMap<RecipeRunDeclaredArtifact, Buffer>()
@@ -208,14 +208,19 @@ export function recipeDeclaredArtifactFailure(declaredArtifacts: RecipeRunDeclar
   return declaredArtifacts.some((artifact) => artifact.required && artifact.status !== "collected") ? new RecipeDeclaredArtifactFailureError(declaredArtifacts) : undefined
 }
 
-export function recipeRuntimeEvidenceFiles(fixtureDatabases: RecipeRunFixtureDatabase[], probes: RecipeRunProbe[], declaredArtifacts: RecipeRunDeclaredArtifact[]): Array<{ filename: string; kind: string; value: unknown }> {
+export function recipeRuntimeEvidenceFiles(fixtureDatabases: RecipeRunFixtureDatabase[], distributionStartupProbes: RecipeRunDistributionStartupProbe[], probes: RecipeRunProbe[], declaredArtifacts: RecipeRunDeclaredArtifact[]): Array<{ filename: string; kind: string; value: unknown }> {
   return [
     ...(fixtureDatabases.length > 0 ? [{ filename: "fixture-databases.json", kind: "fixture-database-results", value: { schema: "wp-codebox/fixture-database-results/v1", fixtures: fixtureDatabases } }] : []),
+    ...(distributionStartupProbes.length > 0 ? [{ filename: "distribution-startup-probes.json", kind: "distribution-startup-probe-results", value: { schema: "wp-codebox/distribution-startup-probe-results/v1", passed: !distributionStartupProbeFailure(distributionStartupProbes), probes: distributionStartupProbes } }] : []),
     ...(probes.length > 0 ? [{ filename: "recipe-probes.json", kind: "recipe-probe-results", value: { schema: "wp-codebox/recipe-probe-results/v1", passed: !recipeProbeFailure(probes), probes } }] : []),
     ...(declaredArtifacts.length > 0 ? [{ filename: "recipe-declared-artifacts.json", kind: "recipe-declared-artifact-results", value: { schema: "wp-codebox/recipe-declared-artifact-results/v1", passed: !recipeDeclaredArtifactFailure(declaredArtifacts), artifacts: declaredArtifacts } }] : []),
   ]
 }
 
+function distributionStartupProbeFailure(probes: RecipeRunDistributionStartupProbe[]): boolean {
+  return probes.some((probe) => probe.status === "failed")
+}
+
 export function recipeProbeFailure(probes: RecipeRunProbe[]): RecipeProbeFailureError | undefined {
   return probes.some((probe) => probe.status === "failed" && !probe.allowFailure) ? new RecipeProbeFailureError(probes) : undefined
 }

packages/cli/src/commands/recipe-run-finalizer.ts

@@ -42,6 +42,7 @@ interface RecipeRunCommonOutputFields {
   stagedFiles?: RecipeRunStagedFile[]
   fixtureDatabases?: RecipeRunFixtureDatabase[]
   siteSeeds?: RecipeRunSiteSeed[]
+  distributionStartupProbes?: RecipeRunOutput["distributionStartupProbes"]
   probes?: RecipeRunProbe[]
   declaredArtifacts?: RecipeRunDeclaredArtifact[]
   phaseEvidence?: RecipePhaseEvidence[]
@@ -97,6 +98,7 @@ export function completedRecipeOutputFields(args: {
   stagedFiles: RecipeRunStagedFile[]
   fixtureDatabases: RecipeRunFixtureDatabase[]
   siteSeeds: RecipeRunOutput["siteSeeds"]
+  distributionStartupProbes: NonNullable<RecipeRunOutput["distributionStartupProbes"]>
   probes: RecipeRunProbe[]
   declaredArtifacts: RecipeRunDeclaredArtifact[]
   phaseEvidence: RecipePhaseEvidence[]
@@ -111,6 +113,7 @@ export function completedRecipeOutputFields(args: {
     stagedFiles: args.stagedFiles,
     fixtureDatabases: args.fixtureDatabases,
     siteSeeds: args.siteSeeds,
+    ...(args.distributionStartupProbes.length > 0 ? { distributionStartupProbes: args.distributionStartupProbes } : {}),
     probes: args.probes,
     declaredArtifacts: args.declaredArtifacts,
     phaseEvidence: args.phaseEvidence,

packages/cli/src/commands/recipe-run-types.ts

@@ -49,6 +49,7 @@ export interface RecipeRunOutput {
   stagedFiles?: RecipeRunStagedFile[]
   fixtureDatabases?: RecipeRunFixtureDatabase[]
   siteSeeds?: RecipeRunSiteSeed[]
+  distributionStartupProbes?: RecipeRunDistributionStartupProbe[]
   probes?: RecipeRunProbe[]
   declaredArtifacts?: RecipeRunDeclaredArtifact[]
   phaseEvidence?: RecipePhaseEvidence[]
@@ -149,7 +150,7 @@ export interface RecipeDiagnosticArtifactRef {
   sha256?: string
 }
 
-export type RecipePhaseName = "runtime_startup" | "mount_plugins" | "activate_plugins" | "run_blueprint_steps" | "import_fixture_databases" | "run_workloads" | "run_probes" | "collect_artifacts"
+export type RecipePhaseName = "runtime_startup" | "mount_plugins" | "activate_plugins" | "run_blueprint_steps" | "import_fixture_databases" | "run_distribution_startup_probes" | "run_workloads" | "run_probes" | "collect_artifacts"
 
 export interface RecipePhaseEvidence {
   schema: "wp-codebox/recipe-phase-evidence/v1"
@@ -243,6 +244,21 @@ export interface RecipeRunProbe {
   metadata?: Record<string, unknown>
 }
 
+export interface RecipeRunDistributionStartupProbe {
+  schema: "wp-codebox/distribution-startup-probe-result/v1"
+  index: number
+  name: string
+  type: "http" | "browser" | "wp-cli" | "php"
+  status: "passed" | "failed" | "skipped"
+  command?: string
+  args?: string[]
+  exitCode?: number
+  stdout?: string
+  stderr?: string
+  reason?: string
+  metadata?: Record<string, unknown>
+}
+
 export interface RecipeRunDeclaredArtifact {
   schema: "wp-codebox/recipe-declared-artifact-result/v1"
   index: number

packages/cli/src/commands/recipe-run-workflow-evidence.ts

@@ -1,11 +1,11 @@
-import { type ArtifactBundle, type ArtifactManifestFile, type ExecutionResult, type Runtime, type WorkspaceRecipe, type WorkspaceRecipeProbe } from "@automattic/wp-codebox-core"
+import { type ArtifactBundle, type ArtifactManifestFile, type ExecutionResult, type Runtime, type WorkspaceRecipe, type WorkspaceRecipeDistributionStartupProbe, type WorkspaceRecipeProbe } from "@automattic/wp-codebox-core"
 import { stripUndefined } from "@automattic/wp-codebox-core/internals"
 import { recipeExecutionSpec, sandboxWorkspaceContract } from "../agent-sandbox.js"
 import { executeAgentFanoutFromArgs } from "../agent-fanout.js"
 import { recipeWorkflowSteps, type RecipeWorkflowPhase } from "../recipe-validation.js"
 import { artifactManifestFilesByPath } from "./recipe-run-benchmark-artifacts.js"
 import { serializeRecipeRunError } from "./recipe-run-output.js"
-import type { RecipeAdvisoryFailure, RecipeBrowserEvidence, RecipeBrowserEvidenceFileRef, RecipeExecutionResult, RecipeRunOptions, RecipeRunProbe } from "./recipe-run-types.js"
+import type { RecipeAdvisoryFailure, RecipeBrowserEvidence, RecipeBrowserEvidenceFileRef, RecipeExecutionResult, RecipeRunDistributionStartupProbe, RecipeRunOptions, RecipeRunProbe } from "./recipe-run-types.js"
 
 export function withRecipeExecutionPhase(execution: ExecutionResult, recipePhase: RecipeWorkflowPhase, recipeStepIndex: number, recipeCommand?: string): RecipeExecutionResult {
   return {
@@ -177,6 +177,46 @@ export async function runRecipeProbes(recipe: WorkspaceRecipe, recipeDirectory:
   return results
 }
 
+export async function runDistributionStartupProbes(recipe: WorkspaceRecipe, runtime: Runtime, executions: RecipeExecutionResult[]): Promise<RecipeRunDistributionStartupProbe[]> {
+  const results: RecipeRunDistributionStartupProbe[] = []
+  for (const [index, probe] of (recipe.distribution?.startupProbes ?? []).entries()) {
+    if (probe.type === "http" || probe.type === "browser") {
+      results.push(stripUndefined({
+        schema: "wp-codebox/distribution-startup-probe-result/v1" as const,
+        index,
+        name: probe.name,
+        type: probe.type,
+        status: "skipped" as const,
+        reason: "Distribution startup probe type is planned but not executable by this runtime primitive.",
+        metadata: probe.metadata,
+      }))
+      continue
+    }
+
+    const execution = await executeDistributionStartupProbe(runtime, probe, index)
+    executions.push(execution)
+    results.push(stripUndefined({
+      schema: "wp-codebox/distribution-startup-probe-result/v1" as const,
+      index,
+      name: probe.name,
+      type: probe.type,
+      status: execution.exitCode === 0 ? "passed" as const : "failed" as const,
+      command: execution.command,
+      args: execution.args,
+      exitCode: execution.exitCode,
+      stdout: execution.stdout,
+      stderr: execution.stderr,
+      metadata: probe.metadata,
+    }))
+  }
+  return results
+}
+
+export function distributionStartupProbeFailure(probes: RecipeRunDistributionStartupProbe[]): Error | undefined {
+  const failed = probes.find((probe) => probe.status === "failed")
+  return failed ? new Error(`Distribution startup probe "${failed.name}" failed with exit code ${failed.exitCode ?? "unknown"}.`) : undefined
+}
+
 async function executeRecipeProbe(runtime: Runtime, probe: WorkspaceRecipeProbe, recipeDirectory: string, index: number): Promise<RecipeExecutionResult> {
   try {
     const execution = await runtime.execute(await recipeExecutionSpec(probe.step, recipeDirectory))
@@ -187,6 +227,19 @@ async function executeRecipeProbe(runtime: Runtime, probe: WorkspaceRecipeProbe,
   }
 }
 
+async function executeDistributionStartupProbe(runtime: Runtime, probe: WorkspaceRecipeDistributionStartupProbe, index: number): Promise<RecipeExecutionResult> {
+  const spec = probe.type === "wp-cli"
+    ? { command: "wordpress.wp-cli", args: [`command=${probe.command ?? ""}`] }
+    : { command: "wordpress.run-php", args: [`code=${probe.code ?? ""}`] }
+  try {
+    const execution = await runtime.execute(spec)
+    return withRecipeExecutionPhase(execution, "setup", index, `distribution.startupProbe:${probe.name}`)
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error)
+    throw new Error(`Distribution startup probe "${probe.name}" failed before producing a result: ${message}`, { cause: error })
+  }
+}
+
 function parseProbeJson(stdout: string): unknown | undefined {
   const trimmed = stdout.trim()
   if (!trimmed) {