From 673c5bcfcece82074a4dc03af87c256a318c70f0 Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Wed, 15 Jul 2026 23:41:47 -0400 Subject: [PATCH] Project reviewer-safe workflow results --- .../prepare-agent-task-upload.mjs | 54 ++++++++++++++++++- tests/runtime-sources-materialization.test.ts | 11 ++-- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs b/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs index 97d52a29c..a8ea963e7 100644 --- a/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs +++ b/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs @@ -9,6 +9,8 @@ const MAX_UPLOAD_FILE_BYTES = 4 * 1024 * 1024 const MAX_TRANSCRIPT_EXECUTIONS = 64 const MAX_REVIEW_TEXT_BYTES = 32 * 1024 const MAX_TOOL_ARGUMENT_KEYS = 32 +const MAX_REVIEW_COLLECTION_ENTRIES = 64 +const MAX_REVIEW_VALUE_DEPTH = 8 const workspace = resolve(process.env.AGENT_TASK_WORKSPACE || process.cwd()) const uploadPath = resolve(process.env.AGENT_TASK_UPLOAD_PATH || join(workspace, ".codebox", "agent-task-upload")) const requestPath = resolve(process.env.AGENT_TASK_REQUEST_PATH || join(workspace, ".codebox", "agent-task-request.json")) @@ -136,7 +138,10 @@ async function stageTextFile(source, destination, options = {}) { const contents = openedMetadata.isFile() && openedMetadata.size <= MAX_UPLOAD_FILE_BYTES ? await handle.readFile() : null await handle.close() if (!contents || contents.includes(0) || !isUtf8(contents)) return false - const sanitized = options.compactNativeInput ? compactNativeInput(contents.toString("utf8")) : sanitizeText(contents.toString("utf8")) + const sourceText = contents.toString("utf8") + const sanitized = options.projectWorkflowResult + ? `${JSON.stringify(projectWorkflowResult(parseJsonOrEmpty(sourceText)), null, 2)}\n` + : options.compactNativeInput ? compactNativeInput(sourceText) : sanitizeText(sourceText) const text = redact(sanitizeSeedSnapshotJson(sanitized)) assertNoRuntimeSourcePaths(text, privateUploadRoots, "Runtime source or workspace paths must never be persisted in artifact uploads.") assertNoSeedSnapshotPaths(text) @@ -173,6 +178,51 @@ function boundedText(value) { return containsRuntimeSourceContent(text) ? "[redacted-source-content]" : text } +function projectReviewValue(value, depth = 0) { + if (depth >= MAX_REVIEW_VALUE_DEPTH) return undefined + if (typeof value === "string") return boundedText(value) + if (typeof value === "number" || typeof value === "boolean" || value === null) return value + if (Array.isArray(value)) return value.slice(0, MAX_REVIEW_COLLECTION_ENTRIES).flatMap((entry) => { + const projected = projectReviewValue(entry, depth + 1) + return projected === undefined ? [] : [projected] + }) + return Object.fromEntries(Object.entries(record(value)).slice(0, MAX_REVIEW_COLLECTION_ENTRIES).flatMap(([key, entry]) => { + const projectedKey = boundedText(key) + const projected = projectReviewValue(entry, depth + 1) + return projectedKey === undefined || projected === undefined ? [] : [[projectedKey, projected]] + })) +} + +function projectWorkflowResult(value) { + const result = record(value) + const execution = record(result.execution) + const outputs = record(result.outputs) + const reviewerEvidence = record(result.reviewer_evidence) + const verification = Array.isArray(result.verification) ? result.verification.slice(0, MAX_REVIEW_COLLECTION_ENTRIES).map((value) => { + const check = record(value) + return projectReviewValue(Object.fromEntries(["kind", "command", "description", "success", "exit_code", "stdout_truncated", "stderr_truncated"].flatMap((key) => check[key] === undefined ? [] : [[key, check[key]]]))) + }) : undefined + return projectReviewValue(Object.fromEntries(Object.entries({ + schema: result.schema, + run_id: result.run_id, + status: result.status, + success: result.success, + request_path: result.request_path, + execution: Object.keys(execution).length ? { stdout_truncated: execution.stdout_truncated, stderr_truncated: execution.stderr_truncated } : undefined, + reviewer_evidence: reviewerEvidence.transcript === undefined ? undefined : { transcript: reviewerEvidence.transcript }, + verification, + publication: result.publication, + transcript: result.transcript, + artifacts: result.artifacts, + outputs: outputs.projections === undefined ? undefined : { projections: outputs.projections }, + access: result.access, + publication_verification: result.publication_verification, + publication_error: result.publication_error, + failure: result.failure, + projection_error: result.projection_error, + }).filter(([, entry]) => entry !== undefined))) +} + function safeTargetPath(value) { const path = safeRelativeArtifactPath(value) return path ? `workspace/${path}` : undefined @@ -417,7 +467,7 @@ const declaredPaths = new Set(declaredArtifactPaths(result, declarations(request await rm(uploadPath, { recursive: true, force: true }) await mkdir(uploadPath, { recursive: true }) await stageTextFile(requestPath, join(uploadPath, ".codebox", "agent-task-request.json")) -await stageTextFile(resultSource, join(uploadPath, ".codebox", "agent-task-workflow-result.json")) +await stageTextFile(resultSource, join(uploadPath, ".codebox", "agent-task-workflow-result.json"), { projectWorkflowResult: true }) await stageTextFile(join(workspace, ".codebox", "native-agent-task-input.json"), join(uploadPath, ".codebox", "native-agent-task-input.json"), { compactNativeInput: true }) for (const path of declaredPaths) { const source = resolve(artifactsPath, path) diff --git a/tests/runtime-sources-materialization.test.ts b/tests/runtime-sources-materialization.test.ts index 1b304ca17..5aa0f2c8e 100644 --- a/tests/runtime-sources-materialization.test.ts +++ b/tests/runtime-sources-materialization.test.ts @@ -159,8 +159,7 @@ await withTempDir("wp-codebox-runtime-source-upload-", async (directory) => { await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), `${JSON.stringify(diagnosticRegression.result, null, 2)}\n`) await execFileAsync(process.execPath, [script.pathname], { env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: privateRoot } }) const stagedDiagnostic = await readFile(join(upload, ".codebox", "agent-task-workflow-result.json"), "utf8") - assert.match(stagedDiagnostic, /OpenAiProvider/) - assert.match(stagedDiagnostic, /WP_Agents_Registry/) + assert.doesNotMatch(stagedDiagnostic, /OpenAiProvider|WP_Agents_Registry/, "raw runtime diagnostics are not part of the reviewer-safe result") await writeFile(join(artifacts, "safe.json"), "Diagnostic snippet: { await writeFile(join(artifacts, "files", "transcript.json"), transcriptSource) const transcriptDigest = createHash("sha256").update(transcriptSource).digest("hex") const reviewerEvidence = { transcript: { schema: "wp-codebox/agent-transcript/v1", kind: "codebox-transcript", path: "files/transcript.json", source_sha256: transcriptDigest, size_bytes: Buffer.byteLength(transcriptSource) } } - await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), JSON.stringify({ reviewer_evidence: reviewerEvidence, runtime_result: { agent_task_run_result: { refs: { transcripts: [{ kind: "codebox-transcript", path: "files/transcript.json", sha256: transcriptDigest }] } } }, typed_artifacts: [{ name: "reviewer-report", type: "report", artifact: { path: "prepared-plugins/agents-api/agents-api.php" } }] })) + const rawWorkflowPayload = "raw-provider-payload-sentinel" + await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), JSON.stringify({ schema: "wp-codebox/agent-task-workflow-result/v1", status: "succeeded", success: true, reviewer_evidence: reviewerEvidence, runtime_result: { agent_task_run_result: { refs: { transcripts: [{ kind: "codebox-transcript", path: "files/transcript.json", sha256: transcriptDigest }] }, model_output: `Target code: { assert.doesNotMatch(transcript, /raw-error|"args"|"provider"|"hash"/) assert.doesNotMatch(transcript, /private-runtime-source|secret-transcript-value|untrusted secret sentinel|\/Users\/example/) const stagedWorkflowResult = await readFile(join(upload, ".codebox", "agent-task-workflow-result.json"), "utf8") + assert.match(stagedWorkflowResult, /"status": "succeeded"/) + assert.match(stagedWorkflowResult, /"publication"/) + assert.match(stagedWorkflowResult, /"pr_url": "https:\/\/github\.com\/example\/repository\/pull\/1"/) + assert.match(stagedWorkflowResult, /"reviewer_evidence"/) assert.match(stagedWorkflowResult, /"projection_sha256": "[a-f0-9]{64}"/, "the staged result records the digest of the sanitized transcript projection") + assert.doesNotMatch(stagedWorkflowResult, /runtime_result|engine_data|Runtime_Result_Leak|Engine_Data_Leak|raw-provider-payload-sentinel|private-runtime-source/) const transcriptExclusions = await readFile(join(upload, ".codebox", "agent-task-artifacts", "exclusions.json"), "utf8") assert.match(transcriptExclusions, /canonical_transcripts/) assert.match(transcriptExclusions, /codebox-transcript/)