Merge pull request #1740 from Automattic/fix/editor-validate-browser-evidence

Surface editor validation browser evidence

Author: chubes4

packages/cli/src/commands/recipe-run-workflow-evidence.ts

@@ -88,6 +88,10 @@ function recipeBrowserEvidenceForExecution(execution: RecipeExecutionResult, man
   }
 
   const parsed = parseJsonObject(execution.stdout)
+  if (!parsed && command === "wordpress.editor-validate-blocks") {
+    const evidence = recipeBrowserEvidenceFromEditorValidateBlocksExecution(execution, manifestFiles)
+    return evidence ? [evidence] : []
+  }
   if (!parsed) {
     return []
   }
@@ -99,10 +103,39 @@ function recipeBrowserEvidenceForExecution(execution: RecipeExecutionResult, man
     })
   }
 
-  const evidence = recipeBrowserEvidenceFromParsedExecution(execution, command, parsed, manifestFiles, recipe)
+  const evidence = command === "wordpress.editor-validate-blocks"
+    ? recipeBrowserEvidenceFromEditorValidateBlocksExecution(execution, manifestFiles, parsed)
+    : recipeBrowserEvidenceFromParsedExecution(execution, command, parsed, manifestFiles, recipe)
   return evidence ? [evidence] : []
 }
 
+function recipeBrowserEvidenceFromEditorValidateBlocksExecution(execution: RecipeExecutionResult, manifestFiles: Map<string, ArtifactManifestFile>, parsed?: Record<string, unknown>): RecipeBrowserEvidence | undefined {
+  const files = recipeBrowserEvidenceFiles({
+    validateBlocks: "files/browser/editor-validate-blocks.json",
+    summary: "files/browser/editor-validate-blocks-summary.json",
+  }, manifestFiles)
+  const summaryFile = browserEvidenceFileRef("files/browser/editor-validate-blocks-summary.json", manifestFiles)
+  if (Object.keys(files).length === 0 && !summaryFile) {
+    return undefined
+  }
+
+  const validation = parsed ?? (typeof execution.result?.stdout === "string" ? parseJsonObject(execution.result.stdout) : undefined)
+  const summary = stripUndefined({
+    editorValidateBlocks: validation,
+  })
+  return stripUndefined({
+    schema: "wp-codebox/recipe-browser-evidence/v1",
+    phase: execution.recipePhase,
+    index: execution.recipeStepIndex,
+    command: "wordpress.editor-validate-blocks",
+    metadata: execution.recipeStepMetadata,
+    status: execution.exitCode === 0 ? "completed" : "failed",
+    summaryFile,
+    files,
+    summary: Object.keys(summary).length > 0 ? summary : undefined,
+  }) as RecipeBrowserEvidence
+}
+
 function recipeBrowserEvidenceFromParsedExecution(execution: RecipeExecutionResult, command: string, parsed: Record<string, unknown>, manifestFiles: Map<string, ArtifactManifestFile>, recipe: WorkspaceRecipe | undefined): RecipeBrowserEvidence | undefined {
   const files = recipeBrowserEvidenceFiles(parsed.files, manifestFiles)
   const summaryFile = browserEvidenceFileRef(stringValue((parsed.files as Record<string, unknown> | undefined)?.summary), manifestFiles)
@@ -169,7 +202,7 @@ function browserEvidenceFileRef(path: string | undefined, manifestFiles: Map<str
 }
 
 function recipeCommandProducesBrowserEvidence(command: string): boolean {
-  return command.startsWith("wordpress.browser-") || command === "wordpress.editor-canvas-probe" || command === "wordpress.html-capture" || command === "wordpress.visual-compare"
+  return command.startsWith("wordpress.browser-") || command === "wordpress.editor-canvas-probe" || command === "wordpress.editor-validate-blocks" || command === "wordpress.html-capture" || command === "wordpress.visual-compare"
 }
 
 export async function executeRecipeWorkflowStep(runtime: Runtime, workflowStep: ReturnType<typeof recipeWorkflowSteps>[number], recipeDirectory: string, sandboxWorkspace?: ReturnType<typeof sandboxWorkspaceContract>, artifactRoot?: string, options?: RecipeRunOptions, inputMountPathMap: readonly InputMountPathMapping[] = []): Promise<RecipeExecutionResult> {

tests/recipe-browser-evidence.test.ts

@@ -14,6 +14,8 @@ writeFileSync(manifestPath, JSON.stringify({
     { path: "files/browser/visual-compare/diff.png", kind: "browser-visual-diff-screenshot", contentType: "image/png", sha256: "diff-digest" },
     { path: "files/browser/visual-compare/visual-diff.json", kind: "browser-visual-diff", contentType: "application/json", sha256: "visual-diff-digest" },
     { path: "files/browser/visual-compare/summary.json", kind: "browser-summary", contentType: "application/json", sha256: "summary-digest" },
+    { path: "files/browser/editor-validate-blocks.json", kind: "editor-validate-blocks", contentType: "application/json", sha256: "editor-validate-digest" },
+    { path: "files/browser/editor-validate-blocks-summary.json", kind: "browser-summary", contentType: "application/json", sha256: "editor-summary-digest" },
   ],
 }, null, 2))
 
@@ -59,3 +61,37 @@ assert.equal(evidence[0]?.files.candidateScreenshot?.path, "files/browser/visual
 assert.equal(evidence[0]?.files.diffScreenshot?.path, "files/browser/visual-compare/diff.png")
 assert.equal(evidence[0]?.files.visualDiff?.path, "files/browser/visual-compare/visual-diff.json")
 assert.equal(evidence[0]?.summaryFile?.path, "files/browser/visual-compare/summary.json")
+
+const editorEvidence = await recipeBrowserEvidence({ manifestPath } as never, [{
+  id: "editor-validate-1",
+  command: "wordpress.editor-validate-blocks",
+  recipeCommand: "wordpress.editor-validate-blocks",
+  recipePhase: "steps",
+  recipeStepIndex: 3,
+  recipeStepMetadata: { fixture_id: "fixture-alpha" },
+  args: ["target=front-page"],
+  exitCode: 0,
+  stdout: `${JSON.stringify({
+    total_blocks: 2,
+    valid_blocks: 2,
+    invalid_blocks: 0,
+    validation_method: "wp.blocks.validateBlock",
+    validation_provider: "wordpress-block-editor",
+    results: [],
+  })}\n`,
+  stderr: "",
+  startedAt: "2026-01-01T00:00:00.000Z",
+  finishedAt: "2026-01-01T00:00:00.001Z",
+}], undefined)
+
+assert.equal(editorEvidence.length, 1)
+assert.equal(editorEvidence[0]?.command, "wordpress.editor-validate-blocks")
+assert.equal(editorEvidence[0]?.phase, "steps")
+assert.equal(editorEvidence[0]?.index, 3)
+assert.deepEqual(editorEvidence[0]?.metadata, { fixture_id: "fixture-alpha" })
+assert.equal(editorEvidence[0]?.status, "completed")
+assert.equal(editorEvidence[0]?.files.validateBlocks?.path, "files/browser/editor-validate-blocks.json")
+assert.equal(editorEvidence[0]?.files.validateBlocks?.kind, "editor-validate-blocks")
+assert.equal(editorEvidence[0]?.files.summary?.path, "files/browser/editor-validate-blocks-summary.json")
+assert.equal(editorEvidence[0]?.summaryFile?.path, "files/browser/editor-validate-blocks-summary.json")
+assert.equal((editorEvidence[0]?.summary as { editorValidateBlocks?: { total_blocks?: number } } | undefined)?.editorValidateBlocks?.total_blocks, 2)