Merge pull request #856 from Automattic/fix/issues-851-854-recipe-evidence

Fix recipe evidence completion contracts

Author: chubes4

docs/recipe-contract.md

@@ -138,6 +138,34 @@ come from the command catalog:
 npm run wp-codebox -- commands --json
 ```
 
+Use `allowFailure: true` or `advisory: true` for evidence-only workflow steps.
+Failed advisory steps are reported in `advisoryFailures` and do not make an
+otherwise successful recipe return `success: false`.
+
+```json
+{
+  "command": "wordpress.browser-actions",
+  "args": ["url=/", "steps-json=[...]"],
+  "advisory": true
+}
+```
+
+## Recipe Output Evidence
+
+`recipe-run --json` returns `wp-codebox/recipe-run/v1`. Browser command sidecars
+are promoted into `browserEvidence` so callers can discover stable evidence
+without hardcoding artifact paths or parsing command stdout.
+
+Each `browserEvidence` entry includes the workflow phase/index, command,
+summary file, artifact file refs, summary payload, and `scriptResult` when the
+browser command produced one. The same browser evidence is mirrored into
+`latest-runtime.json` under the run artifact pointer.
+
+`--preview-hold <duration>` records held-preview lifecycle metadata in the
+artifact bundle and returns after recipe work finishes. Use
+`--preview-hold-blocking` only for operator workflows that need the CLI process
+to keep a live preview server open for the hold duration.
+
 ## WordPress PHPUnit Runtime
 
 `wordpress.phpunit` is the lightweight WP Codebox equivalent of plugin PHPUnit

packages/cli/src/commands/recipe-run-artifact-pointers.ts

@@ -2,7 +2,7 @@ import { mkdir, writeFile } from "node:fs/promises"
 import { join, relative } from "node:path"
 import { stripUndefined, type ArtifactBundle, type RuntimeInfo } from "@automattic/wp-codebox-core"
 import type { RunOutput } from "../runtime-command-wrappers.js"
-import type { RecipeArtifactPointerCommandStatus, RecipeArtifactPointerState, RecipePhaseEvidence } from "./recipe-run-types.js"
+import type { RecipeArtifactPointerCommandStatus, RecipeArtifactPointerState, RecipeBrowserEvidence, RecipePhaseEvidence } from "./recipe-run-types.js"
 
 export class RecipeArtifactPointerTracker {
   private command: string | undefined
@@ -11,6 +11,7 @@ export class RecipeArtifactPointerTracker {
   private artifacts: ArtifactBundle | undefined
   private failure: RunOutput["error"] | undefined
   private phases: RecipePhaseEvidence[] = []
+  private browserEvidence: RecipeBrowserEvidence[] = []
 
   constructor(private readonly directory: string | undefined, private readonly runId: string, private readonly recipePath: string, private readonly startedAt: string) {}
 
@@ -23,8 +24,9 @@ export class RecipeArtifactPointerTracker {
     this.commandStatus = state.commandStatus ?? this.commandStatus
     this.runtime = state.runtime ?? this.runtime
     this.artifacts = state.artifacts ?? this.artifacts
-    this.failure = state.failure ?? this.failure
+    this.failure = state.failure ?? (state.commandStatus === "completed" || state.commandStatus === "running" ? undefined : this.failure)
     this.phases = state.phases ?? this.phases
+    this.browserEvidence = state.browserEvidence ?? this.browserEvidence
 
     const pointer = stripUndefined({
       schema: "wp-codebox/recipe-run-artifact-pointer/v1",
@@ -39,6 +41,7 @@ export class RecipeArtifactPointerTracker {
       commandStatus: this.commandStatus,
       failure: this.failure,
       failurePhase: recipeArtifactPointerFailurePhase(this.failure, this.phases),
+      browserEvidence: this.browserEvidence.length > 0 ? this.browserEvidence : undefined,
       paths: recipeArtifactPointerPaths(this.directory, this.runtime, this.artifacts),
     })
 

packages/cli/src/commands/recipe-run-interruption.ts

@@ -130,6 +130,9 @@ export function createRecipeInterruptionController(): RecipeInterruptionControll
       controller.dispose()
       process.kill(process.pid, metadata.signal)
     },
+    clear() {
+      metadata = undefined
+    },
   }
 
   return controller

packages/cli/src/commands/recipe-run-output.ts

@@ -51,7 +51,7 @@ export class RecipeDeclaredArtifactFailureError extends Error {
 
 export function exitAfterPlaygroundCliBootFailure(output: RecipeRunCommandOutput): void {
   if (output.schema === "wp-codebox/recipe-run/v1" && hasSerializedErrorCode(output.error, "wp-codebox-playground-cli-exited")) {
-    process.exit(output.success ? 0 : 1)
+    process.exitCode = output.success ? 0 : 1
   }
 }
 
@@ -74,10 +74,26 @@ function hasSerializedErrorCode(error: RunOutput["error"] | undefined, code: str
 
 export function exitAfterRecipeRunTimeout(output: RecipeRunCommandOutput): void {
   if (output.schema === "wp-codebox/recipe-run/v1" && output.error?.code === "recipe-run-timeout") {
-    process.exit(output.success ? 0 : 1)
+    process.exitCode = output.success ? 0 : 1
   }
 }
 
+export async function writeRecipeJsonOutput(output: unknown): Promise<void> {
+  await writeStdout(`${JSON.stringify(output, null, 2)}\n`)
+}
+
+function writeStdout(contents: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    process.stdout.write(contents, (error) => {
+      if (error) {
+        reject(error)
+        return
+      }
+      resolve()
+    })
+  })
+}
+
 export function printJsonFailureDiagnostic(output: { success: boolean; error?: { message?: string }; logs?: string[] }): void {
   if (output.success) {
     return

packages/cli/src/commands/recipe-run-types.ts

@@ -12,6 +12,7 @@ export interface RecipeRunOptions {
   previewPublicUrl?: string
   previewPort?: number
   previewBind?: string
+  previewHoldBlocking: boolean
   timeoutMs: number
   json: boolean
   dryRun: boolean
@@ -50,6 +51,8 @@ export interface RecipeRunOutput {
   probes?: RecipeRunProbe[]
   declaredArtifacts?: RecipeRunDeclaredArtifact[]
   phaseEvidence?: RecipePhaseEvidence[]
+  advisoryFailures?: RecipeAdvisoryFailure[]
+  browserEvidence?: RecipeBrowserEvidence[]
   diagnostics?: RecipeRuntimeDiagnostic[]
   validation?: {
     issues: RecipeValidationIssue[]
@@ -72,6 +75,37 @@ export type RecipeExecutionResult = ExecutionResult & {
   recipePhase?: RecipeWorkflowPhase
   recipeStepIndex?: number
   recipeCommand?: string
+  recipeAdvisory?: boolean
+}
+
+export interface RecipeAdvisoryFailure {
+  schema: "wp-codebox/recipe-advisory-failure/v1"
+  phase: RecipeWorkflowPhase
+  index: number
+  command: string
+  status: "failed"
+  error: RunOutput["error"]
+}
+
+export interface RecipeBrowserEvidenceFileRef {
+  path: string
+  kind?: string
+  contentType?: string
+  sha256?: { algorithm: "sha256"; value: string }
+}
+
+export interface RecipeBrowserEvidence {
+  schema: "wp-codebox/recipe-browser-evidence/v1"
+  phase?: RecipeWorkflowPhase
+  index?: number
+  command: string
+  status: "completed" | "failed"
+  requestedUrl?: string
+  finalUrl?: string
+  summaryFile?: RecipeBrowserEvidenceFileRef
+  files: Record<string, RecipeBrowserEvidenceFileRef | RecipeBrowserEvidenceFileRef[]>
+  summary?: unknown
+  scriptResult?: unknown
 }
 
 export type RecipeArtifactPointerCommandStatus = "queued" | "running" | "completed" | "failed"
@@ -83,6 +117,7 @@ export interface RecipeArtifactPointerState {
   artifacts?: ArtifactBundle
   failure?: RunOutput["error"]
   phases?: RecipePhaseEvidence[]
+  browserEvidence?: RecipeBrowserEvidence[]
 }
 
 export type RecipePhaseName = "runtime_startup" | "mount_plugins" | "activate_plugins" | "run_blueprint_steps" | "import_fixture_databases" | "run_workloads" | "run_probes" | "collect_artifacts"
@@ -212,4 +247,5 @@ export interface RecipeInterruptionController {
   interruptible<T>(promise: Promise<T>): Promise<T>
   throwIfInterrupted(): void
   propagateIfInterrupted(): void
+  clear(): void
 }