Transport native task results through a result file (#1764)

Author: chubes4

.github/scripts/run-agent-task/execute-native-agent-task.mjs

@@ -1,7 +1,8 @@
-import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"
+import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises"
 import { join, resolve } from "node:path"
 import { spawn } from "node:child_process"
 import { materializeExternalNativePackage, normalizeExternalPackageSource, parseExternalPackageSourcePolicy } from "./materialize-external-native-package.mjs"
+import { readNativeResult } from "./native-result-file.mjs"
 
 const requestPath = process.env.AGENT_TASK_REQUEST_PATH || ".codebox/agent-task-request.json"
 const workspace = resolve(process.env.AGENT_TASK_WORKSPACE || process.cwd())
@@ -191,6 +192,8 @@ const externalPackageSource = normalizeExternalPackageSource(request.external_pa
 const artifactsPath = join(workspace, ".codebox", "agent-task-artifacts")
 const runtimeInputPath = join(workspace, ".codebox", "native-agent-task-input.json")
 const resultPath = join(workspace, ".codebox", "agent-task-workflow-result.json")
+const controlledCodeboxPath = resolve(requestPath, "..")
+const nativeResultPath = join(controlledCodeboxPath, "native-agent-task-result.json")
 const runnerWorkspaceTools = [
   "workspace-read", "workspace-ls", "workspace-grep", "workspace-write", "workspace-edit", "workspace-apply-patch",
   "workspace-git-status", "workspace-git-diff", "workspace-git-add", "workspace-git-commit", "workspace-git-push",
@@ -266,18 +269,16 @@ await writeFile(runtimeInputPath, `${JSON.stringify(taskInput, null, 2)}\n`)
 
 let execution = { code: 0, stdout: "", stderr: "", stdout_truncated: false, stderr_truncated: false }
 if (request.run_agent && !request.dry_run) {
-  execution = await command("node", [codeboxCliPath, "agent-task-run", "--input-file", runtimeInputPath, "--json"], workspace, agentEnvironment())
+  execution = await command("node", [codeboxCliPath, "agent-task-run", "--input-file", runtimeInputPath, "--result-file", nativeResultPath], workspace, agentEnvironment())
 }
 
 // Public package bytes are embedded in the runtime recipe and consumed only by
 // the Playground bootstrap before the agent's tools are resolved.
 
-let runtimeResult = {}
-try {
-  runtimeResult = redact(JSON.parse(execution.stdout))
-} catch {
-  runtimeResult = { success: false, diagnostics: [{ code: "wp-codebox.agent-task.invalid-result", message: "Native agent-task-run did not return JSON.", stderr: execution.stderr }] }
-}
+const runtimeResult = request.run_agent && !request.dry_run
+  ? await readNativeResult(nativeResultPath, controlledCodeboxPath, secretValues, redact)
+  : {}
+await rm(nativeResultPath, { force: true })
 
 await redactArtifactFiles(artifactsPath)
 

.github/scripts/run-agent-task/native-result-file.mjs

@@ -0,0 +1,47 @@
+import { lstat, readFile } from "node:fs/promises"
+import { resolve } from "node:path"
+
+export const MAX_NATIVE_RESULT_BYTES = 1024 * 1024
+const TERMINAL_STATUSES = new Set(["succeeded", "failed", "no_op", "timeout", "provider_error", "unable_to_remediate"])
+
+export function nativeResultFailure(code, message) {
+  return { success: false, diagnostics: [{ code, message }] }
+}
+
+export async function readNativeResult(path, controlledDirectory, secretValues, redact) {
+  const resolvedPath = resolve(path)
+  const resolvedDirectory = resolve(controlledDirectory)
+  if (!resolvedPath.startsWith(`${resolvedDirectory}/`)) {
+    return nativeResultFailure("wp-codebox.agent-task.result-path", "Native agent-task result path is outside the controlled .codebox directory.")
+  }
+  try {
+    const info = await lstat(resolvedPath)
+    if (!info.isFile() || info.isSymbolicLink()) {
+      return nativeResultFailure("wp-codebox.agent-task.result-file", "Native agent-task result file must be a regular non-symlink file.")
+    }
+    if (info.size > MAX_NATIVE_RESULT_BYTES) {
+      return nativeResultFailure("wp-codebox.agent-task.result-too-large", `Native agent-task result exceeds the ${MAX_NATIVE_RESULT_BYTES}-byte limit.`)
+    }
+    const contents = await readFile(resolvedPath, "utf8")
+    if (Buffer.byteLength(contents) !== info.size) {
+      return nativeResultFailure("wp-codebox.agent-task.result-file", "Native agent-task result changed while it was being read.")
+    }
+    if (secretValues.some((secret) => contents.includes(secret))) {
+      return nativeResultFailure("wp-codebox.agent-task.result-secret", "Native agent-task result contains a configured secret.")
+    }
+    const result = JSON.parse(contents)
+    const taskResult = record(result)
+    const summary = record(taskResult.agent_task_run_result)
+    if (taskResult.schema !== "wp-codebox/agent-task-run/v1" || typeof taskResult.success !== "boolean" || !TERMINAL_STATUSES.has(taskResult.status) || summary.schema !== "wp-codebox/agent-task-run-result/v1" || typeof summary.success !== "boolean" || !TERMINAL_STATUSES.has(summary.status) || taskResult.success !== summary.success || taskResult.status !== summary.status) {
+      return nativeResultFailure("wp-codebox.agent-task.result-schema", "Native agent-task result did not match the required result schema.")
+    }
+    return redact(taskResult)
+  } catch (error) {
+    const code = error && typeof error === "object" && error.code === "ENOENT" ? "wp-codebox.agent-task.result-missing" : "wp-codebox.agent-task.result-malformed"
+    return nativeResultFailure(code, code.endsWith("missing") ? "Native agent-task did not produce a result file." : "Native agent-task result file was malformed.")
+  }
+}
+
+function record(value) {
+  return value && typeof value === "object" && !Array.isArray(value) ? value : {}
+}

README.md

@@ -1361,7 +1361,7 @@ Generic caller-owned `request.json` payloads may use this shape:
 }
 ```
 
-WP Codebox normalizes the task input, writes the private temporary recipe, runs `wp-codebox recipe-run`, then deletes temporary recipe/seed files. The JSON response keeps `schema: "wp-codebox/agent-task-run/v1"` and includes `agent_task_run_result` with `schema: "wp-codebox/agent-task-run-result/v1"`. Consumers read `agent_task_run_result.status`, `agent_task_run_result.success`, `agent_task_run_result.refs`, `agent_task_run_result.metadata`, and `failure_evidence` as the stable result envelope. Secret values are never accepted in the request or returned in the response; `secret_env` carries names only.
+WP Codebox normalizes the task input, writes the private temporary recipe, runs `wp-codebox recipe-run`, then deletes temporary recipe/seed files. The JSON response keeps `schema: "wp-codebox/agent-task-run/v1"` and includes `agent_task_run_result` with `schema: "wp-codebox/agent-task-run-result/v1"`. Consumers read `agent_task_run_result.status`, `agent_task_run_result.success`, `agent_task_run_result.refs`, `agent_task_run_result.metadata`, and `failure_evidence` as the stable result envelope. `--result-file <path>` atomically writes that final JSON with a private temporary file and a rename; callers that require structured output should use it instead of stdout. Secret values are never accepted in the request or returned in the response; `secret_env` carries names only.
 
 Failed `agent-task-run` responses also include `failure_evidence` with `schema: "wp-codebox/agent-task-run-failure-evidence/v1"`. This block is intentionally safe for parent orchestrators to persist alongside their own task failure record. It includes the best available `phase`, `command`, `exit_code`, message, stdout/stderr snippets, runtime and sandbox identifiers, artifact directory or bundle identifiers, recipe-run status/run ID, diagnostics, phase evidence, and the serialized error. If recipe-run fails before normal runtime artifacts exist or returns malformed output, `agent-task-run` still emits this fallback evidence block in the CLI JSON payload and references it from `evidence_refs` with kind `codebox-agent-task-failure-evidence`.
 

docs/agent-runtime-contract.md

@@ -143,7 +143,7 @@ WP Codebox rejects product ability paths that pass raw `code` or `code_file`. De
 }
 ```
 
-`agent_task_result` is the sandbox semantic result `wp-codebox/agent-task-result/v1`. `agent_task_run_result` is the caller-facing `wp-codebox/agent-task-run-result/v1` contract emitted by the CLI and WordPress ability surfaces. Orchestrators read `agent_task_run_result.status`, `agent_task_run_result.success`, `agent_task_run_result.refs`, and `agent_task_run_result.metadata` as the stable result envelope. Node consumers can also normalize older or nested envelopes with `normalizeAgentTaskRunResult()` from `@automattic/wp-codebox-core` or `wp-codebox-workspace/core`; the package exports `AGENT_TASK_RUN_RESULT_SCHEMA` and `AGENT_TASK_RUN_RESULT_JSON_SCHEMA` for contract checks. The normalized envelope groups stable artifact refs into `artifact_bundles`, `changed_files`, `patches`, `transcripts`, `logs`, and `runtimes`, and classifies terminal statuses including `succeeded`, `failed`, `no_op`, `timeout`, `provider_error`, and `unable_to_remediate`.
+`agent_task_result` is the sandbox semantic result `wp-codebox/agent-task-result/v1`. `agent_task_run_result` is the caller-facing `wp-codebox/agent-task-run-result/v1` contract emitted by the CLI and WordPress ability surfaces. Orchestrators read `agent_task_run_result.status`, `agent_task_run_result.success`, `agent_task_run_result.refs`, and `agent_task_run_result.metadata` as the stable result envelope. For process execution, pass `--result-file <path>`: the CLI writes the final envelope through a mode `0600` temporary file and atomic rename. stdout and stderr remain diagnostic streams and must not be used as required structured transport. Node consumers can also normalize older or nested envelopes with `normalizeAgentTaskRunResult()` from `@automattic/wp-codebox-core` or `wp-codebox-workspace/core`; the package exports `AGENT_TASK_RUN_RESULT_SCHEMA` and `AGENT_TASK_RUN_RESULT_JSON_SCHEMA` for contract checks. The normalized envelope groups stable artifact refs into `artifact_bundles`, `changed_files`, `patches`, `transcripts`, `logs`, and `runtimes`, and classifies terminal statuses including `succeeded`, `failed`, `no_op`, `timeout`, `provider_error`, and `unable_to_remediate`.
 
 Failed `agent-task-run` responses include `wp-codebox/agent-task-run-failure-evidence/v1` in `failure_evidence` when available. This block is safe for orchestrators to persist with job failure records and includes phase, command, exit code, redacted stdout/stderr snippets, runtime and sandbox identifiers, artifact references, diagnostics, and serialized error data.
 

docs/agent-task-reusable-workflow.md

@@ -183,4 +183,4 @@ workflow itself cannot inspect the caller's `uses:` declaration at runtime.
 
 The reusable workflow stages every uploaded file through a fail-closed policy. Only regular UTF-8 files of 4 MiB or less are uploaded, after configured secret values are redacted. Symlinks, special files, binary files, and files larger than 4 MiB are excluded from the upload staging directory. This applies to task artifacts and workflow request, input, and result files.
 
-Native task execution, validation-dependency installation, verification, drift checks, and GitHub pull-request validation retain at most 32 KiB from each stdout and stderr stream while continuing to drain both streams. Command results expose `stdout_truncated` and `stderr_truncated` when retained output is incomplete.
+Native task execution, validation-dependency installation, verification, drift checks, and GitHub pull-request validation retain at most 32 KiB from each stdout and stderr stream while continuing to drain both streams. Command results expose `stdout_truncated` and `stderr_truncated` when retained output is incomplete. Native `agent-task-run` structured output is instead read from a controlled `.codebox/native-agent-task-result.json` file outside the agent-writable target checkout. The executor accepts only a regular, non-symlink JSON file up to 1 MiB with the required result schemas, rejects configured secret values, redacts the accepted record before persistence, and removes the transient native result file after reading it.

packages/cli/src/commands/agent-task-run.ts

@@ -1,6 +1,6 @@
-import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"
+import { lstat, mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises"
 import { tmpdir } from "node:os"
-import { join } from "node:path"
+import { dirname, join } from "node:path"
 import { AGENT_TASK_RUN_REQUEST_SCHEMA, HEADLESS_AGENT_TASK_REQUEST_SCHEMA, artifactResultEnvelope, buildAgentTaskRecipe, DEFAULT_WORDPRESS_VERSION, headlessAgentTaskRequestToRunInput, normalizeAgentRuntimeWorkload, normalizeAgentTaskRunResult, normalizeAgentTerminalResult, normalizeArtifactResultTypedArtifacts, normalizeHeadlessAgentTaskRequest, normalizeHeadlessAgentTaskResult, normalizeTaskInput, parseCommandJson, parseCommandOptions, resolveEffectiveRuntimeToolPolicy, type AgentTaskRunInput, type AgentTaskRunResultSummary, type AgentTerminalResult, type ArtifactResultEnvelope, type HeadlessAgentTaskResult, type SandboxToolPolicySnapshot, type TypedArtifactDTO } from "@automattic/wp-codebox-core"
 import { stripUndefined } from "@automattic/wp-codebox-core/internals"
 import { runRecipeRunCommand } from "./recipe-run.js"
@@ -10,6 +10,7 @@ export type { AgentTaskRunInput } from "@automattic/wp-codebox-core"
 export interface AgentTaskRunOptions {
   inputPath: string
   json: boolean
+  resultFile?: string
   previewHoldSeconds?: string
   previewPublicUrl?: string
   previewPort?: string
@@ -79,9 +80,14 @@ export async function runAgentTaskRunCommand(args: string[]): Promise<number> {
   const options = parseAgentTaskRunOptions(args)
   const input = normalizeAgentTaskRunCliInput(JSON.parse(await readFile(options.inputPath, "utf8")))
   const output = await runAgentTask(input, options)
+  const jsonOutput = agentTaskRunJsonOutput(output)
+
+  if (options.resultFile) {
+    await writeAgentTaskRunResultFile(options.resultFile, jsonOutput)
+  }
 
   if (options.json) {
-    process.stdout.write(`${JSON.stringify(agentTaskRunJsonOutput(output), null, 2)}\n`)
+    process.stdout.write(`${JSON.stringify(jsonOutput, null, 2)}\n`)
     return agentTaskRunExitCode(output)
   }
 
@@ -391,7 +397,7 @@ function parseAgentTaskRunOptions(args: string[]): AgentTaskRunOptions {
     throw new Error(`Unknown agent-task-run option: ${positionals[0]}`)
   }
   for (const name of options.keys()) {
-    if (!["--input-file", "--json", "--format", "--preview-hold-seconds", "--preview-public-url", "--preview-port", "--preview-bind", "--preview-hold-blocking", "--preview-lease-json"].includes(name)) {
+    if (!["--input-file", "--json", "--format", "--result-file", "--preview-hold-seconds", "--preview-public-url", "--preview-port", "--preview-bind", "--preview-hold-blocking", "--preview-lease-json"].includes(name)) {
       throw new Error(`Unknown agent-task-run option: ${name}`)
     }
   }
@@ -402,6 +408,7 @@ function parseAgentTaskRunOptions(args: string[]): AgentTaskRunOptions {
   return {
     inputPath,
     json: options.get("--json") === true || stringOption(options, "--format") === "json",
+    resultFile: stringOption(options, "--result-file") || undefined,
     previewHoldSeconds: stringOption(options, "--preview-hold-seconds"),
     previewPublicUrl: stringOption(options, "--preview-public-url"),
     previewPort: stringOption(options, "--preview-port"),
@@ -411,6 +418,32 @@ function parseAgentTaskRunOptions(args: string[]): AgentTaskRunOptions {
   }
 }
 
+/** Write caller-owned structured output without exposing an incomplete result. */
+export async function writeAgentTaskRunResultFile(path: string, output: AgentTaskRunOutput | HeadlessAgentTaskResult): Promise<void> {
+  const parent = dirname(path)
+  await mkdir(parent, { recursive: true, mode: 0o700 })
+  const parentStat = await lstat(parent)
+  if (!parentStat.isDirectory() || parentStat.isSymbolicLink()) {
+    throw new Error("agent-task-run result-file parent must be a non-symlink directory")
+  }
+  try {
+    const targetStat = await lstat(path)
+    if (!targetStat.isFile() || targetStat.isSymbolicLink()) {
+      throw new Error("agent-task-run result-file must be a regular file when it already exists")
+    }
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error
+  }
+
+  const temporaryPath = `${path}.${process.pid}.${Date.now()}.tmp`
+  try {
+    await writeFile(temporaryPath, `${JSON.stringify(output, null, 2)}\n`, { encoding: "utf8", mode: 0o600, flag: "wx" })
+    await rename(temporaryPath, path)
+  } finally {
+    await rm(temporaryPath, { force: true })
+  }
+}
+
 function objectRecord(value: unknown): Record<string, unknown> | undefined {
   return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined
 }

packages/cli/src/output.ts

@@ -331,7 +331,7 @@ export function printHelp(): void {
   wp-codebox fuzz-minimize-case --input-file <path> [--format=json] [--dry-run]
   wp-codebox run-wordpress-workload --input-file <path> [--format=json] [--dry-run]
   wp-codebox run-agent-task --input-file <path> [--json] [--preview-hold-seconds <n>] [--preview-hold-blocking] [--preview-port <port>] [--preview-bind <host>] [--preview-public-url <url>] [--preview-lease-json <json>]
-  wp-codebox agent-task-run --input-file <path> [--json] [--preview-hold-seconds <n>] [--preview-hold-blocking] [--preview-port <port>] [--preview-bind <host>] [--preview-public-url <url>] [--preview-lease-json <json>]
+  wp-codebox agent-task-run --input-file <path> [--json] [--result-file <path>] [--preview-hold-seconds <n>] [--preview-hold-blocking] [--preview-port <port>] [--preview-bind <host>] [--preview-public-url <url>] [--preview-lease-json <json>]
   wp-codebox validate-blueprint --blueprint <json|file> [options]
   wp-codebox materialize-replay-package --snapshot <path> --output <dir> [--snapshot-ref <ref>] [--json]
   wp-codebox recipe-run --recipe <path> [options]
@@ -343,6 +343,7 @@ Options:
   --options <path>    Recipe builder options JSON file for recipe build.
   --output <path>     Recipe build output JSON path, or materialize-replay-package output directory.
   --input-file <path> Input JSON for public workload/fuzz commands or agent-task-run.
+  --result-file <path> Atomically write the final agent-task-run JSON result to a caller-owned file.
   --format=json       Emit machine-readable JSON; accepted by public workload/fuzz commands.
   --preview-hold-seconds <n>
                     Keep preview runtimes alive after run-agent-task/agent-task-run/recipe-run.