Project reviewer-safe workflow results (#1815)

Author: chubes4

.github/scripts/run-agent-task/prepare-agent-task-upload.mjs

@@ -9,6 +9,8 @@ const MAX_UPLOAD_FILE_BYTES = 4 * 1024 * 1024
 const MAX_TRANSCRIPT_EXECUTIONS = 64
 const MAX_REVIEW_TEXT_BYTES = 32 * 1024
 const MAX_TOOL_ARGUMENT_KEYS = 32
+const MAX_REVIEW_COLLECTION_ENTRIES = 64
+const MAX_REVIEW_VALUE_DEPTH = 8
 const workspace = resolve(process.env.AGENT_TASK_WORKSPACE || process.cwd())
 const uploadPath = resolve(process.env.AGENT_TASK_UPLOAD_PATH || join(workspace, ".codebox", "agent-task-upload"))
 const requestPath = resolve(process.env.AGENT_TASK_REQUEST_PATH || join(workspace, ".codebox", "agent-task-request.json"))
@@ -136,7 +138,10 @@ async function stageTextFile(source, destination, options = {}) {
   const contents = openedMetadata.isFile() && openedMetadata.size <= MAX_UPLOAD_FILE_BYTES ? await handle.readFile() : null
   await handle.close()
   if (!contents || contents.includes(0) || !isUtf8(contents)) return false
-  const sanitized = options.compactNativeInput ? compactNativeInput(contents.toString("utf8")) : sanitizeText(contents.toString("utf8"))
+  const sourceText = contents.toString("utf8")
+  const sanitized = options.projectWorkflowResult
+    ? `${JSON.stringify(projectWorkflowResult(parseJsonOrEmpty(sourceText)), null, 2)}\n`
+    : options.compactNativeInput ? compactNativeInput(sourceText) : sanitizeText(sourceText)
   const text = redact(sanitizeSeedSnapshotJson(sanitized))
   assertNoRuntimeSourcePaths(text, privateUploadRoots, "Runtime source or workspace paths must never be persisted in artifact uploads.")
   assertNoSeedSnapshotPaths(text)
@@ -173,6 +178,51 @@ function boundedText(value) {
   return containsRuntimeSourceContent(text) ? "[redacted-source-content]" : text
 }
 
+function projectReviewValue(value, depth = 0) {
+  if (depth >= MAX_REVIEW_VALUE_DEPTH) return undefined
+  if (typeof value === "string") return boundedText(value)
+  if (typeof value === "number" || typeof value === "boolean" || value === null) return value
+  if (Array.isArray(value)) return value.slice(0, MAX_REVIEW_COLLECTION_ENTRIES).flatMap((entry) => {
+    const projected = projectReviewValue(entry, depth + 1)
+    return projected === undefined ? [] : [projected]
+  })
+  return Object.fromEntries(Object.entries(record(value)).slice(0, MAX_REVIEW_COLLECTION_ENTRIES).flatMap(([key, entry]) => {
+    const projectedKey = boundedText(key)
+    const projected = projectReviewValue(entry, depth + 1)
+    return projectedKey === undefined || projected === undefined ? [] : [[projectedKey, projected]]
+  }))
+}
+
+function projectWorkflowResult(value) {
+  const result = record(value)
+  const execution = record(result.execution)
+  const outputs = record(result.outputs)
+  const reviewerEvidence = record(result.reviewer_evidence)
+  const verification = Array.isArray(result.verification) ? result.verification.slice(0, MAX_REVIEW_COLLECTION_ENTRIES).map((value) => {
+    const check = record(value)
+    return projectReviewValue(Object.fromEntries(["kind", "command", "description", "success", "exit_code", "stdout_truncated", "stderr_truncated"].flatMap((key) => check[key] === undefined ? [] : [[key, check[key]]])))
+  }) : undefined
+  return projectReviewValue(Object.fromEntries(Object.entries({
+    schema: result.schema,
+    run_id: result.run_id,
+    status: result.status,
+    success: result.success,
+    request_path: result.request_path,
+    execution: Object.keys(execution).length ? { stdout_truncated: execution.stdout_truncated, stderr_truncated: execution.stderr_truncated } : undefined,
+    reviewer_evidence: reviewerEvidence.transcript === undefined ? undefined : { transcript: reviewerEvidence.transcript },
+    verification,
+    publication: result.publication,
+    transcript: result.transcript,
+    artifacts: result.artifacts,
+    outputs: outputs.projections === undefined ? undefined : { projections: outputs.projections },
+    access: result.access,
+    publication_verification: result.publication_verification,
+    publication_error: result.publication_error,
+    failure: result.failure,
+    projection_error: result.projection_error,
+  }).filter(([, entry]) => entry !== undefined)))
+}
+
 function safeTargetPath(value) {
   const path = safeRelativeArtifactPath(value)
   return path ? `workspace/${path}` : undefined
@@ -417,7 +467,7 @@ const declaredPaths = new Set(declaredArtifactPaths(result, declarations(request
 await rm(uploadPath, { recursive: true, force: true })
 await mkdir(uploadPath, { recursive: true })
 await stageTextFile(requestPath, join(uploadPath, ".codebox", "agent-task-request.json"))
-await stageTextFile(resultSource, join(uploadPath, ".codebox", "agent-task-workflow-result.json"))
+await stageTextFile(resultSource, join(uploadPath, ".codebox", "agent-task-workflow-result.json"), { projectWorkflowResult: true })
 await stageTextFile(join(workspace, ".codebox", "native-agent-task-input.json"), join(uploadPath, ".codebox", "native-agent-task-input.json"), { compactNativeInput: true })
 for (const path of declaredPaths) {
   const source = resolve(artifactsPath, path)

tests/runtime-sources-materialization.test.ts

@@ -159,8 +159,7 @@ await withTempDir("wp-codebox-runtime-source-upload-", async (directory) => {
   await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), `${JSON.stringify(diagnosticRegression.result, null, 2)}\n`)
   await execFileAsync(process.execPath, [script.pathname], { env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: privateRoot } })
   const stagedDiagnostic = await readFile(join(upload, ".codebox", "agent-task-workflow-result.json"), "utf8")
-  assert.match(stagedDiagnostic, /OpenAiProvider/)
-  assert.match(stagedDiagnostic, /WP_Agents_Registry/)
+  assert.doesNotMatch(stagedDiagnostic, /OpenAiProvider|WP_Agents_Registry/, "raw runtime diagnostics are not part of the reviewer-safe result")
   await writeFile(join(artifacts, "safe.json"), "Diagnostic snippet: <?php OpenAiProvider function was unavailable in WP_Agents_Registry.")
   await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), JSON.stringify({ typed_artifacts: [{ name: "reviewer-report", type: "report", artifact: { path: "safe.json" } }] }))
   await execFileAsync(process.execPath, [script.pathname], { env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: privateRoot } })
@@ -170,7 +169,8 @@ await withTempDir("wp-codebox-runtime-source-upload-", async (directory) => {
   await writeFile(join(artifacts, "files", "transcript.json"), transcriptSource)
   const transcriptDigest = createHash("sha256").update(transcriptSource).digest("hex")
   const reviewerEvidence = { transcript: { schema: "wp-codebox/agent-transcript/v1", kind: "codebox-transcript", path: "files/transcript.json", source_sha256: transcriptDigest, size_bytes: Buffer.byteLength(transcriptSource) } }
-  await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), JSON.stringify({ reviewer_evidence: reviewerEvidence, runtime_result: { agent_task_run_result: { refs: { transcripts: [{ kind: "codebox-transcript", path: "files/transcript.json", sha256: transcriptDigest }] } } }, typed_artifacts: [{ name: "reviewer-report", type: "report", artifact: { path: "prepared-plugins/agents-api/agents-api.php" } }] }))
+  const rawWorkflowPayload = "raw-provider-payload-sentinel"
+  await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), JSON.stringify({ schema: "wp-codebox/agent-task-workflow-result/v1", status: "succeeded", success: true, reviewer_evidence: reviewerEvidence, runtime_result: { agent_task_run_result: { refs: { transcripts: [{ kind: "codebox-transcript", path: "files/transcript.json", sha256: transcriptDigest }] }, model_output: `Target code: <?php final class Runtime_Result_Leak {} ${rawWorkflowPayload}`, tool_call: { arguments: { content: rawWorkflowPayload }, result: { body: rawWorkflowPayload }, error: { message: rawWorkflowPayload } } } }, outputs: { engine_data: { provider_response: `<?php final class Engine_Data_Leak {} ${rawWorkflowPayload}`, private_path: `${privateRoot}/source.php` }, projections: { pr_url: "https://github.com/example/repository/pull/1" } }, publication: { schema: "wp-codebox/runner-workspace-publication-result/v1", success: true, status: "published", pull_request: { number: 1, url: "https://github.com/example/repository/pull/1" } }, typed_artifacts: [{ name: "reviewer-report", type: "report", artifact: { path: "prepared-plugins/agents-api/agents-api.php" } }] }))
   await execFileAsync(process.execPath, [script.pathname], { env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: privateRoot, OPENAI_API_KEY: "secret-transcript-value" } })
   const transcript = await readFile(join(upload, ".codebox", "agent-task-artifacts", "transcript.json"), "utf8")
   assert.doesNotMatch(transcript, /<\?php final class Target_Plugin/)
@@ -186,7 +186,12 @@ await withTempDir("wp-codebox-runtime-source-upload-", async (directory) => {
   assert.doesNotMatch(transcript, /raw-error|"args"|"provider"|"hash"/)
   assert.doesNotMatch(transcript, /private-runtime-source|secret-transcript-value|untrusted secret sentinel|\/Users\/example/)
   const stagedWorkflowResult = await readFile(join(upload, ".codebox", "agent-task-workflow-result.json"), "utf8")
+  assert.match(stagedWorkflowResult, /"status": "succeeded"/)
+  assert.match(stagedWorkflowResult, /"publication"/)
+  assert.match(stagedWorkflowResult, /"pr_url": "https:\/\/github\.com\/example\/repository\/pull\/1"/)
+  assert.match(stagedWorkflowResult, /"reviewer_evidence"/)
   assert.match(stagedWorkflowResult, /"projection_sha256": "[a-f0-9]{64}"/, "the staged result records the digest of the sanitized transcript projection")
+  assert.doesNotMatch(stagedWorkflowResult, /runtime_result|engine_data|Runtime_Result_Leak|Engine_Data_Leak|raw-provider-payload-sentinel|private-runtime-source/)
   const transcriptExclusions = await readFile(join(upload, ".codebox", "agent-task-artifacts", "exclusions.json"), "utf8")
   assert.match(transcriptExclusions, /canonical_transcripts/)
   assert.match(transcriptExclusions, /codebox-transcript/)