Preserve verification failure evidence (#1890)

Author: chubes4

.github/scripts/run-agent-task/execute-native-agent-task.mjs

@@ -589,6 +589,19 @@ function underRoot(root, path) {
   return contained !== ".." && !contained.startsWith(`..${String.fromCharCode(47)}`) && !isAbsolute(contained)
 }
 
+async function appliedWorkspaceEvidence(refs, artifactsPath) {
+  const root = await realpath(resolve(artifactsPath))
+  const evidence = {}
+  for (const [kind, key] of [["codebox-patch", "patch"], ["codebox-changed-files", "changed_files"]]) {
+    const ref = refs.find((entry) => entry.kind === kind && typeof entry.path === "string" && entry.path)
+    if (!ref) continue
+    const path = await realpath(resolve(root, ref.path)).catch((error) => error?.code === "ENOENT" ? "" : Promise.reject(error))
+    if (!path || !underRoot(root, path)) continue
+    evidence[key] = { artifact_path: relative(root, path).replaceAll("\\", "/") }
+  }
+  return Object.keys(evidence).length === 2 ? evidence : undefined
+}
+
 async function canonicalReviewerTranscript(nativeRuntimeResult, artifactsPath) {
   const publicCore = await import(pathToFileURL(join(codeboxRoot, "packages/runtime-core/dist/public.js")).href)
   const refs = publicCore.normalizePublicArtifactRefDTOs(nativeRuntimeResult)
@@ -792,6 +805,7 @@ if (toolObservability) {
 }
 assertNoRuntimeSourcePaths(runtimeResult, privateRuntimeSourceRootForSanitization)
 let workspaceApply = { status: "no-op", changedFiles: [] }
+let workspaceEvidence
 let runnerWorkspaceCore = null
 let downstreamFailure = null
 if (execution.code === 0 && runtimeResult.success === true && request.runner_workspace?.enabled) {
@@ -800,6 +814,7 @@ if (execution.code === 0 && runtimeResult.success === true && request.runner_wor
     const publicCore = await import(pathToFileURL(join(codeboxRoot, "packages/runtime-core/dist/public.js")).href)
     const refs = publicCore.normalizePublicArtifactRefDTOs(runtimeResult)
       .filter((ref) => ref.kind === "codebox-patch" || ref.kind === "codebox-changed-files")
+    workspaceEvidence = await appliedWorkspaceEvidence(refs, artifactsPath)
     const trustedArtifacts = await trustedArtifactApplyRefs(trustedApplyArtifactRoot, refs)
     workspaceApply = await runnerWorkspaceCore.applyRunnerWorkspacePatch({ artifactRoot: trustedArtifacts.root, artifactRefs: trustedArtifacts.refs, workspaceRoot: workspace, writablePaths, seedIdentity: runnerWorkspaceSeedSnapshot?.provenance.identity })
   } catch (error) {
@@ -835,7 +850,21 @@ if (execution.code === 0 && request.run_agent && !request.dry_run && !downstream
 const verificationPassed = verification.every((check) => check.success)
 if (!verificationPassed) {
   const artifactError = verification.find((check) => check.artifact_error)?.artifact_error
-  downstreamFailure ??= { stage: "verification", message: artifactError?.message || "Runner workspace verification did not pass.", ...(artifactError ? { artifact_error: artifactError } : {}) }
+  const failedCheck = verification.find((check) => !check.success)
+  const diagnostic = failedCheck ? {
+    kind: failedCheck.kind,
+    command: failedCheck.command,
+    exit_code: failedCheck.exit_code,
+    stdout_tail: bounded(failedCheck.stdout, MAX_WORKFLOW_OUTPUT_BYTES),
+    stderr_tail: bounded(failedCheck.stderr, MAX_WORKFLOW_OUTPUT_BYTES),
+  } : undefined
+  downstreamFailure ??= {
+    stage: "verification",
+    message: artifactError?.message || "Runner workspace verification did not pass.",
+    ...(artifactError ? { artifact_error: artifactError } : {}),
+    ...(diagnostic ? { diagnostic } : {}),
+    ...(workspaceApply.status === "applied" && workspaceEvidence ? { evidence: workspaceEvidence } : {}),
+  }
 }
 const runtimeRecord = record(runtimeResult)
 const agentResult = record(runtimeRecord.agent_task_run_result)

.github/scripts/run-agent-task/prepare-agent-task-upload.mjs

@@ -163,7 +163,14 @@ function projectWorkflowResult(value) {
   const reviewerEvidence = record(result.reviewer_evidence)
   const verification = Array.isArray(result.verification) ? result.verification.slice(0, MAX_REVIEW_COLLECTION_ENTRIES).map((value) => {
     const check = record(value)
-    return projectReviewValue(Object.fromEntries(["kind", "command", "description", "success", "exit_code", "stdout_truncated", "stderr_truncated", "artifact", "artifact_error"].flatMap((key) => check[key] === undefined ? [] : [[key, check[key]]])))
+    const diagnostic = check.success === false ? {
+      stdout_tail: check.stdout,
+      stderr_tail: check.stderr,
+    } : {}
+    return projectReviewValue(Object.fromEntries(Object.entries({
+      ...Object.fromEntries(["kind", "command", "description", "success", "exit_code", "stdout_truncated", "stderr_truncated", "artifact", "artifact_error"].flatMap((key) => check[key] === undefined ? [] : [[key, check[key]]])),
+      ...diagnostic,
+    }).filter(([, entry]) => entry !== undefined)))
   }) : undefined
   return projectReviewValue(Object.fromEntries(Object.entries({
     schema: result.schema,

tests/execute-native-agent-task-lifecycle.test.mjs

@@ -37,7 +37,7 @@ async function run(mode = "success") {
         : mode === "artifact-oversized"
           ? `echo verification >> .codebox/order && node -e 'require("node:fs").writeFileSync(".codebox/agent-task-artifacts/completion-report.json", Buffer.alloc(4 * 1024 * 1024 + 1, 120))'`
           : `echo verification >> .codebox/order && printf '%s\\n' '{"ok":true}' > .codebox/agent-task-artifacts/completion-report.json${mode === "artifact-command-fail" ? " && false" : mode === "artifact-readonly" ? " && chmod 444 .codebox/agent-task-artifacts/completion-report.json" : ""}`
-  const verificationCommand = artifactModes.has(mode) && mode !== "artifact-drift" ? artifactCommand : (mode === "verify-fail" ? "false" : "echo verification >> .codebox/order")
+  const verificationCommand = artifactModes.has(mode) && mode !== "artifact-drift" ? artifactCommand : (mode === "verify-fail" ? "printf 'verification stdout\\n'; printf 'verification stderr\\n' >&2; false" : "echo verification >> .codebox/order")
   const driftCommand = mode === "artifact-drift"
     ? artifactCommand
     : mode === "artifact-replaced"
@@ -303,6 +303,25 @@ assert.equal(failedVerification.code, 1)
 assert(!failedVerification.order.includes("publish"))
 assert.equal(failedVerification.result.failure.stage, "verification")
 assert.equal(failedVerification.result.runtime_result.success, true, "verification failures retain the normalized runtime result")
+assert.equal(failedVerification.result.failure.diagnostic.stdout_tail, "verification stdout\n")
+assert.equal(failedVerification.result.failure.diagnostic.stderr_tail, "verification stderr\n")
+assert.equal(failedVerification.result.failure.evidence.patch.artifact_path, "files/patch.diff")
+assert.equal(failedVerification.result.failure.evidence.changed_files.artifact_path, "files/changed-files.json")
+await execFileAsync(process.execPath, [uploader], {
+  cwd: failedVerification.workspace,
+  env: {
+    ...process.env,
+    AGENT_TASK_WORKSPACE: failedVerification.workspace,
+    AGENT_TASK_REQUEST_PATH: join(failedVerification.workspace, ".codebox", "agent-task-request.json"),
+    AGENT_TASK_UPLOAD_PATH: join(failedVerification.workspace, ".codebox", "agent-task-upload"),
+  },
+})
+const uploadedFailedVerification = JSON.parse(await readFile(join(failedVerification.workspace, ".codebox", "agent-task-upload", ".codebox", "agent-task-workflow-result.json"), "utf8"))
+assert.equal(uploadedFailedVerification.verification.find((check) => !check.success).stdout_tail, "verification stdout\n")
+assert.equal(uploadedFailedVerification.verification.find((check) => !check.success).stderr_tail, "verification stderr\n")
+assert.equal(uploadedFailedVerification.failure.diagnostic.stderr_tail, "verification stderr\n")
+assert.match(await readFile(join(failedVerification.workspace, ".codebox", "agent-task-upload", ".codebox", "agent-task-artifacts", "apply-failure", "rejected.patch"), "utf8"), /-before/)
+assert.match(await readFile(join(failedVerification.workspace, ".codebox", "agent-task-upload", ".codebox", "agent-task-artifacts", "apply-failure", "changed-files.json"), "utf8"), /README\.md/)
 
 const denied = await run("deny")
 assert.equal(denied.code, 1)