From 9164635eb9792909afec7e1b99962a5b56eba26f Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Fri, 17 Jul 2026 00:06:31 -0400 Subject: [PATCH] fix: preserve agent task workflow JSON outputs --- .../execute-native-agent-task.mjs | 88 ++++++++++++--- .../prepare-agent-task-upload.mjs | 29 +++++ docs/agent-task-reusable-workflow.md | 13 ++- tests/agent-task-reusable-workflow.test.ts | 101 ++++++++++++++++++ 4 files changed, 217 insertions(+), 14 deletions(-) diff --git a/.github/scripts/run-agent-task/execute-native-agent-task.mjs b/.github/scripts/run-agent-task/execute-native-agent-task.mjs index 71ead885a..a4556ccc7 100644 --- a/.github/scripts/run-agent-task/execute-native-agent-task.mjs +++ b/.github/scripts/run-agent-task/execute-native-agent-task.mjs @@ -16,7 +16,7 @@ const codeboxRoot = resolve(process.env.WP_CODEBOX_WORKFLOW_ROOT || ".") const codeboxCliPath = process.env.WP_CODEBOX_CLI_PATH || join(codeboxRoot, "packages/cli/dist/index.js") const outputPath = process.env.GITHUB_OUTPUT const MAX_CAPTURE_BYTES = 32768 -const MAX_OUTPUT_CHARS = 8192 +const MAX_WORKFLOW_OUTPUT_BYTES = 8192 const secretValues = ["OPENAI_API_KEY", "MODEL_PROVIDER_SECRET_1", "MODEL_PROVIDER_SECRET_2", "MODEL_PROVIDER_SECRET_3", "MODEL_PROVIDER_SECRET_4", "MODEL_PROVIDER_SECRET_5", "GITHUB_TOKEN", "GH_TOKEN", "ACCESS_TOKEN", "EXTERNAL_PACKAGE_SOURCE_POLICY"].map((name) => process.env[name]).filter(Boolean) let privateRuntimeSourceRoot = "" let privateRuntimeSourceRootForSanitization = "" @@ -97,7 +97,53 @@ function capturedStream(limit = MAX_CAPTURE_BYTES) { function output(name, value) { if (!outputPath) return Promise.resolve() const rendered = typeof value === "string" ? value : JSON.stringify(value) - return appendFile(outputPath, `${name}<<__WP_CODEBOX_OUTPUT__\n${bounded(rendered, MAX_OUTPUT_CHARS)}\n__WP_CODEBOX_OUTPUT__\n`) + const safe = sanitizeRuntimeSourceText(redact(rendered), privateRuntimeSourceRootForSanitization) + const bytes = Buffer.byteLength(safe) + if (bytes > MAX_WORKFLOW_OUTPUT_BYTES) { + throw new Error(`${name} exceeds the ${MAX_WORKFLOW_OUTPUT_BYTES}-byte workflow output limit.`) + } + return appendFile(outputPath, `${name}<<__WP_CODEBOX_OUTPUT__\n${safe}\n__WP_CODEBOX_OUTPUT__\n`) +} + +function serializedJson(value) { + return JSON.stringify(value) +} + +function workflowOutputReference(name, rendered, artifactsPath) { + const bytes = Buffer.byteLength(rendered) + if (bytes <= MAX_WORKFLOW_OUTPUT_BYTES) return Promise.resolve({ rendered }) + + const artifactPath = `workflow-outputs/${name}.json` + const reference = { + schema: "wp-codebox/workflow-output-reference/v1", + kind: "codebox-workflow-output", + output: name, + artifact_path: artifactPath, + bytes, + sha256: createHash("sha256").update(rendered).digest("hex"), + } + return mkdir(join(artifactsPath, "workflow-outputs"), { recursive: true }) + .then(() => writeFile(join(artifactsPath, artifactPath), `${rendered}\n`)) + .then(() => ({ rendered: serializedJson(reference), reference })) +} + +function projectionOutputLimitError(name, bytes) { + const error = new Error(`output_projections.${name} serializes to ${bytes} bytes, exceeding the ${MAX_WORKFLOW_OUTPUT_BYTES}-byte workflow output limit. Store the canonical value as a declared artifact and project its artifact-relative reference.`) + error.code = "wp-codebox.agent-task.output-projection-too-large" + error.output_name = name + error.bytes = bytes + error.max_bytes = MAX_WORKFLOW_OUTPUT_BYTES + return error +} + +function validateProjectionOutputSize(value) { + for (const [name, projected] of Object.entries(value)) { + const projectedBytes = Buffer.byteLength(serializedJson(projected)) + if (projectedBytes > MAX_WORKFLOW_OUTPUT_BYTES) throw projectionOutputLimitError(name, projectedBytes) + } + const rendered = serializedJson(value) + const bytes = Buffer.byteLength(rendered) + if (bytes > MAX_WORKFLOW_OUTPUT_BYTES) throw projectionOutputLimitError("projected_outputs_json", bytes) } function safeEnvironment(extra = {}) { @@ -355,7 +401,7 @@ function failureClassification(error) { async function writeNormalizedFailure(error, request = {}) { const resultPath = join(workspace, ".codebox", "agent-task-workflow-result.json") const failure = failureClassification(error) - const message = bounded(error instanceof Error ? error.message : String(error), MAX_OUTPUT_CHARS) + const message = bounded(error instanceof Error ? error.message : String(error), MAX_WORKFLOW_OUTPUT_BYTES) const accessError = failure.classification === "policy" && /allowed_repos|ACCESS_TOKEN|GitHub token|Caller repository|authorized/i.test(message) const result = { schema: "wp-codebox/agent-task-workflow-result/v1", @@ -592,7 +638,7 @@ if (execution.code === 0 && runtimeResult.success === true && request.runner_wor .filter((ref) => ref.kind === "codebox-patch" || ref.kind === "codebox-changed-files") workspaceApply = await runnerWorkspaceCore.applyRunnerWorkspacePatch({ artifactRoot: artifactsPath, artifactRefs: refs, workspaceRoot: workspace, writablePaths, seedIdentity: runnerWorkspaceSeedSnapshot?.provenance.identity }) } catch (error) { - downstreamFailure = { stage: "apply", message: bounded(error instanceof Error ? error.message : String(error), MAX_OUTPUT_CHARS), ...(error?.evidence ? { evidence: error.evidence } : {}) } + downstreamFailure = { stage: "apply", message: bounded(error instanceof Error ? error.message : String(error), MAX_WORKFLOW_OUTPUT_BYTES), ...(error?.evidence ? { evidence: error.evidence } : {}) } } } runtimeResult = sanitizeRuntimeSourceValue(runtimeResult, runtimeSourceOutputRoots) @@ -637,18 +683,26 @@ if (execution.code === 0 && runtimeRecord.success === true && verificationPassed runtimeRecord.metadata = { ...record(runtimeRecord.metadata), runner_workspace_publication: publication } runtimeRecord.outputs = { ...record(runtimeRecord.outputs), runner_workspace_publication: publication } } catch (error) { - downstreamFailure = { stage: "publication", message: bounded(error instanceof Error ? error.message : String(error), MAX_OUTPUT_CHARS) } + downstreamFailure = { stage: "publication", message: bounded(error instanceof Error ? error.message : String(error), MAX_WORKFLOW_OUTPUT_BYTES) } } } if (request.success?.requires_pr === true && workspaceApply.status === "no-op" && !publication) { downstreamFailure ??= { stage: "no-op", message: "success_requires_pr cannot succeed for a no-op runner workspace task." } } let evaluatedProjections = {} -let projectionError = "" +let projectionError try { evaluatedProjections = projections(request.outputs?.projections, runtimeRecord) + validateProjectionOutputSize(evaluatedProjections) } catch (error) { - projectionError = error instanceof Error ? error.message : String(error) + projectionError = { + code: typeof error?.code === "string" ? error.code : "wp-codebox.agent-task.output-projection", + classification: "output-projection", + message: error instanceof Error ? error.message : String(error), + ...(typeof error?.output_name === "string" ? { output_name: error.output_name } : {}), + ...(Number.isSafeInteger(error?.bytes) ? { bytes: error.bytes } : {}), + ...(Number.isSafeInteger(error?.max_bytes) ? { max_bytes: error.max_bytes } : {}), + } } const publicationRequired = request.success?.requires_pr === true const publicationVerification = publicationRequired && execution.code === 0 && runtimeRecord.success === true @@ -662,6 +716,12 @@ const success = request.run_agent && !request.dry_run ? execution.code === 0 && runtimeRecord.success === true && verificationPassed && publicationPassed && !projectionError && !downstreamFailure : true const status = request.run_agent && !request.dry_run ? (success ? "succeeded" : "failed") : "skipped" +const workflowOutputs = { + transcript_json: await workflowOutputReference("transcript_json", serializedJson(agentResult.refs?.transcripts || []), artifactsPath), + engine_data_json: await workflowOutputReference("engine_data_json", serializedJson(redact(record(runtimeRecord.outputs))), artifactsPath), + projected_outputs_json: { rendered: serializedJson(projectionError ? { error: projectionError } : evaluatedProjections) }, + declared_artifacts_json: await workflowOutputReference("declared_artifacts_json", serializedJson(request.artifacts?.declarations || []), artifactsPath), +} const result = { schema: "wp-codebox/agent-task-workflow-result/v1", run_id: runId, @@ -680,6 +740,10 @@ const result = { engine_data: record(runtimeRecord.outputs), projections: evaluatedProjections, }, + ...(Object.values(workflowOutputs).some((output) => output.reference) ? { + workflow_output_artifacts: Object.fromEntries(Object.entries(workflowOutputs) + .flatMap(([name, output]) => output.reference ? [[name, output.reference]] : [])), + } : {}), access: { authorized: true, credential_mode: process.env.GITHUB_TOKEN ? "runner-access-token" : (process.env.OPENAI_API_KEY ? "runner-provider-credentials" : "runner-default-credentials"), policy: { allowed_repos: request.access.allowed_repos } }, ...(publicationRequired ? { publication_verification: publicationVerification } : {}), ...(publicationRequired && !publicationPassed ? { publication_error: "success_requires_pr requires a valid published runner-workspace pull request for target_repo." } : {}), @@ -691,12 +755,12 @@ const sanitizedResult = sanitizeRuntimeSourceValue(redact(result), privateRuntim assertNoRuntimeSourcePaths(sanitizedResult, privateRuntimeSourceRootForSanitization) await writeFile(resultPath, `${JSON.stringify(sanitizedResult, null, 2)}\n`) await output("job_status", status) -await output("transcript_json", JSON.stringify(agentResult.refs?.transcripts || [])) +await output("transcript_json", workflowOutputs.transcript_json.rendered) await output("transcript_summary", `${request.workload?.label || "Run Agent Task"}: ${status}`) -await output("engine_data_json", result.outputs.engine_data) -await output("projected_outputs_json", result.outputs.projections) +await output("engine_data_json", workflowOutputs.engine_data_json.rendered) +await output("projected_outputs_json", workflowOutputs.projected_outputs_json.rendered) await output("credential_mode", result.access.credential_mode) -await output("declared_artifacts_json", result.artifacts.declarations) +await output("declared_artifacts_json", workflowOutputs.declared_artifacts_json.rendered) await output("result_path", ".codebox/agent-task-workflow-result.json") if (!success) process.exitCode = 1 @@ -706,7 +770,7 @@ try { await executeNativeAgentTask() } catch (error) { await writeNormalizedFailure(error) - console.error(bounded(error instanceof Error ? error.message : String(error), MAX_OUTPUT_CHARS)) + console.error(bounded(error instanceof Error ? error.message : String(error), MAX_WORKFLOW_OUTPUT_BYTES)) process.exitCode = 1 } finally { await cleanupMaterializedSources() diff --git a/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs b/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs index 7ab6e0c76..cbb0d05c2 100644 --- a/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs +++ b/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs @@ -419,6 +419,24 @@ function declaredArtifactPaths(result, allowed) { return [...paths].sort() } +function workflowOutputArtifacts(result) { + return Object.entries(record(result.workflow_output_artifacts)).flatMap(([name, value]) => { + const reference = record(value) + const path = safeRelativeArtifactPath(reference.artifact_path) + if (reference.schema !== "wp-codebox/workflow-output-reference/v1" + || reference.kind !== "codebox-workflow-output" + || reference.output !== name + || !/^[A-Za-z0-9_.-]+$/.test(name) + || path !== `workflow-outputs/${name}.json` + || !Number.isSafeInteger(reference.bytes) + || reference.bytes <= 0 + || !/^[a-f0-9]{64}$/.test(reference.sha256)) { + throw new Error("Workflow output artifact reference is malformed.") + } + return [[path, reference]] + }) +} + async function exclusions(root, declaredPaths, transcript) { const counts = new Map() const count = (category) => counts.set(category, (counts.get(category) || 0) + 1) @@ -479,6 +497,7 @@ const request = parseJsonOrEmpty(await readFile(requestPath, "utf8").catch(() => const resultSource = join(workspace, ".codebox", "agent-task-workflow-result.json") const result = parseJsonOrEmpty(await readFile(resultSource, "utf8").catch(() => "{}")) const declaredPaths = new Set(declaredArtifactPaths(result, declarations(request))) +const workflowOutputArtifactsToStage = workflowOutputArtifacts(result) await rm(uploadPath, { recursive: true, force: true }) await mkdir(uploadPath, { recursive: true }) @@ -495,6 +514,16 @@ for (const path of declaredPaths) { } await stageTextFile(source, join(uploadPath, ".codebox", "agent-task-artifacts", path)) } +for (const [path, reference] of workflowOutputArtifactsToStage) { + const source = resolve(artifactsPath, path) + if (relative(artifactsPath, source).startsWith("..") || sourceCategory(path, source)) continue + const bytes = await readFile(source) + const canonical = bytes.subarray(bytes.length - 1)[0] === 10 ? bytes.subarray(0, -1) : bytes + if (canonical.length !== reference.bytes || digest(canonical) !== reference.sha256) { + throw new Error("Workflow output artifact does not match its reference.") + } + await stageTextFile(source, join(uploadPath, ".codebox", "agent-task-artifacts", path)) +} const transcript = await stageCanonicalTranscript(result) if (transcript?.provenance?.source_sha256) { const stagedResultPath = join(uploadPath, ".codebox", "agent-task-workflow-result.json") diff --git a/docs/agent-task-reusable-workflow.md b/docs/agent-task-reusable-workflow.md index 1a7714048..f11cf869d 100644 --- a/docs/agent-task-reusable-workflow.md +++ b/docs/agent-task-reusable-workflow.md @@ -66,7 +66,7 @@ This release-coherence contract fixes [#1759](https://github.com/Automattic/wp-c - `access_token_repos`: comma-separated repositories available to the supplied access token. - `allowed_repos`: JSON repository allowlist. It and `access_token_repos` must explicitly include `target_repo`. - `expected_artifacts`: JSON allowlist and collection metadata. Required runtime artifacts are derived only from `artifact_declarations` entries with `required: true` and must be declared output artifacts. -- `output_projections`: JSON object mapping output names to dot-delimited paths in the native result. A string value remains a required projection for compatibility. A descriptor value has the exact shape `{ "path": "result.path", "required": false }`; unresolved optional projections are omitted, while unresolved required projections fail the run. +- `output_projections`: JSON object mapping output names to dot-delimited paths in the native result. A string value remains a required projection for compatibility. A descriptor value has the exact shape `{ "path": "result.path", "required": false }`; unresolved optional projections are omitted, while unresolved required projections fail the run. Each evaluated projection and the combined projection object must serialize to 8,192 bytes or less. Oversized projections fail before success with `wp-codebox.agent-task.output-projection-too-large`; declare an artifact and project its artifact-relative reference instead. ## Access And Publication @@ -82,7 +82,7 @@ fail closed for every PR, issue, and comment operation outside that set. The checkout does not persist credentials. Verification, dependency, and drift commands run with a clean environment that excludes provider and GitHub secrets. Known secret values are redacted before result or artifact persistence; captured -stdout/stderr is capped at 32 KiB and workflow outputs at 8 KiB. +stdout/stderr is capped at 32 KiB and workflow outputs at 8,192 bytes. `EXTERNAL_PACKAGE_SOURCE_POLICY` is treated as a secret even when it contains only repository and path metadata. It is redacted from runner output, excluded @@ -137,6 +137,15 @@ runner token, so a fabricated publication result cannot satisfy the gate. - `credential_mode`: redacted credential source classification. - `declared_artifacts_json`: accepted typed artifact declarations. +Every structured workflow output is serialized as complete JSON. When a +system-owned structured value exceeds 8,192 bytes, its workflow output becomes a +`wp-codebox/workflow-output-reference/v1` object with an artifact-relative path, +byte count, and SHA-256; the canonical JSON is collected from +`.codebox/agent-task-artifacts/workflow-outputs/`. The workflow result records +these references in `workflow_output_artifacts`. User-declared projections are +validated instead of truncated, so a success result never omits or corrupts a +projection. + The result artifact includes the executable task input, normalized runtime result, evaluated projections, verification records, and runner-owned publication result. A single canonical `codebox-transcript` ref from the diff --git a/tests/agent-task-reusable-workflow.test.ts b/tests/agent-task-reusable-workflow.test.ts index 3df9c5e9b..032cb9d0f 100644 --- a/tests/agent-task-reusable-workflow.test.ts +++ b/tests/agent-task-reusable-workflow.test.ts @@ -412,4 +412,105 @@ for (const name of ["oversize.txt", "binary.bin", "linked-secret.txt"]) { await assert.rejects(readFile(join(uploadArtifactsPath, name), "utf8"), /ENOENT/) } +// Workflow output limits are measured in UTF-8 bytes, never by JS code units. +// Oversized system outputs become artifact references; caller projections fail +// before they can produce malformed GitHub Actions JSON. +const fakeCliPath = join(tmp, "fake-agent-task-cli.mjs") +const outputValueAtBytes = (bytes: number) => { + const overhead = Buffer.byteLength(JSON.stringify({ value: "" })) + return { value: "x".repeat(bytes - overhead) } +} +const readWorkflowOutput = (contents: string, name: string) => { + const match = contents.match(new RegExp(`${name}<<__WP_CODEBOX_OUTPUT__\\n([\\s\\S]*?)\\n__WP_CODEBOX_OUTPUT__`)) + assert.ok(match, `Missing ${name} output`) + return JSON.parse(match[1]) +} +const runWorkflowOutputCase = async (nativeOutputs: Record, outputProjections = {}) => { + const caseOutputPath = join(tmp, `github-output-${Math.random().toString(16).slice(2)}.txt`) + const caseRequest = { + ...request, + external_package_source: nativeSource, + runner_workspace: { enabled: false }, + validation_dependencies: "", + verification_commands: [], + drift_checks: [], + run_agent: true, + dry_run: false, + outputs: { projections: outputProjections }, + } + await writeFile(requestPath, `${JSON.stringify(caseRequest, null, 2)}\n`) + const nativeResultForOutput = { + schema: "wp-codebox/agent-task-run/v1", + success: true, + status: "succeeded", + agent_task_run_result: { schema: "wp-codebox/agent-task-run-result/v1", success: true, status: "succeeded" }, + outputs: nativeOutputs, + } + await writeFile(fakeCliPath, [ + 'import { writeFile } from "node:fs/promises"', + 'const path = process.argv[process.argv.indexOf("--result-file") + 1]', + `await writeFile(path, ${JSON.stringify(JSON.stringify(nativeResultForOutput))})`, + ].join("\n")) + const execution = execFileAsync("node", [executeNativeAgentTask], { + cwd: tmp, + env: { + ...process.env, + NODE_ENV: "test", + GITHUB_OUTPUT: caseOutputPath, + AGENT_TASK_REQUEST_PATH: requestPath, + AGENT_TASK_WORKSPACE: tmp, + WP_CODEBOX_WORKFLOW_ROOT: new URL("..", import.meta.url).pathname, + WP_CODEBOX_CLI_PATH: fakeCliPath, + WP_CODEBOX_TEST_SKIP_MATERIALIZATION: "true", + WP_CODEBOX_TEST_EXTERNAL_PACKAGE_PATH: nativePackagePath, + GITHUB_TOKEN: "test-caller-token", + EXTERNAL_PACKAGE_SOURCE_POLICY: '{"version":1,"repositories":{"automattic/example-agent-packages":["packages/example-agent.agent.json"]}}', + }, + }).catch(async (error: any) => { + const workflowResult = await readFile(resultPath, "utf8").catch(() => "missing") + error.message = `${error.message}\n${error.stderr || ""}\n${workflowResult}` + throw error + }) + return { execution, result: () => readFile(resultPath, "utf8").then(JSON.parse), outputs: () => readFile(caseOutputPath, "utf8") } +} + +const exactBoundaryEngineData = outputValueAtBytes(8192) +const exactBoundaryCase = await runWorkflowOutputCase(exactBoundaryEngineData) +await exactBoundaryCase.execution +assert.deepEqual(readWorkflowOutput(await exactBoundaryCase.outputs(), "engine_data_json"), exactBoundaryEngineData) +assert.equal((await exactBoundaryCase.result()).workflow_output_artifacts, undefined) + +const overBoundaryEngineData = outputValueAtBytes(8193) +const overBoundaryCase = await runWorkflowOutputCase(overBoundaryEngineData) +await overBoundaryCase.execution +const overBoundaryReference = readWorkflowOutput(await overBoundaryCase.outputs(), "engine_data_json") +assert.deepEqual(overBoundaryReference, (await overBoundaryCase.result()).workflow_output_artifacts.engine_data_json) +assert.deepEqual(JSON.parse(await readFile(join(tmp, ".codebox", "agent-task-artifacts", overBoundaryReference.artifact_path), "utf8")), overBoundaryEngineData) +await execFileAsync("node", [new URL("../.github/scripts/run-agent-task/prepare-agent-task-upload.mjs", import.meta.url).pathname], { + cwd: tmp, + env: { ...process.env, AGENT_TASK_WORKSPACE: tmp, AGENT_TASK_REQUEST_PATH: requestPath }, +}) +assert.deepEqual(JSON.parse(await readFile(join(tmp, ".codebox", "agent-task-upload", ".codebox", "agent-task-artifacts", overBoundaryReference.artifact_path), "utf8")), overBoundaryEngineData) + +const multibyteCase = await runWorkflowOutputCase({ value: "😀".repeat(2048) }) +await multibyteCase.execution +assert.equal(readWorkflowOutput(await multibyteCase.outputs(), "engine_data_json").schema, "wp-codebox/workflow-output-reference/v1") + +const nestedProjectionCase = await runWorkflowOutputCase( + { nested: { result: { value: "x".repeat(8193) } } }, + { nested_output: "outputs.nested" }, +) +await assert.rejects(nestedProjectionCase.execution) +const nestedProjectionResult = await nestedProjectionCase.result() +assert.equal(nestedProjectionResult.success, false) +assert.deepEqual(nestedProjectionResult.projection_error, { + code: "wp-codebox.agent-task.output-projection-too-large", + classification: "output-projection", + message: nestedProjectionResult.projection_error.message, + output_name: "nested_output", + bytes: Buffer.byteLength(JSON.stringify({ result: { value: "x".repeat(8193) } })), + max_bytes: 8192, +}) +assert.deepEqual(readWorkflowOutput(await nestedProjectionCase.outputs(), "projected_outputs_json"), { error: nestedProjectionResult.projection_error }) + console.log("agent task reusable workflow ok")