From f66b836c1eecfa3ec38fa6fe5c5459cf27d14cca Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Mon, 13 Jul 2026 20:57:06 -0400 Subject: [PATCH] Transport native task results through a result file --- .../execute-native-agent-task.mjs | 17 +++---- .../run-agent-task/native-result-file.mjs | 47 +++++++++++++++++++ README.md | 2 +- docs/agent-runtime-contract.md | 2 +- docs/agent-task-reusable-workflow.md | 2 +- packages/cli/src/commands/agent-task-run.ts | 41 ++++++++++++++-- packages/cli/src/output.ts | 3 +- tests/agent-task-contracts.test.ts | 25 +++++++++- tests/agent-task-reusable-workflow.test.ts | 32 ++++++++++++- 9 files changed, 152 insertions(+), 19 deletions(-) create mode 100644 .github/scripts/run-agent-task/native-result-file.mjs diff --git a/.github/scripts/run-agent-task/execute-native-agent-task.mjs b/.github/scripts/run-agent-task/execute-native-agent-task.mjs index 0e7aec80f..20bd922b9 100644 --- a/.github/scripts/run-agent-task/execute-native-agent-task.mjs +++ b/.github/scripts/run-agent-task/execute-native-agent-task.mjs @@ -1,7 +1,8 @@ -import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises" +import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises" import { join, resolve } from "node:path" import { spawn } from "node:child_process" import { materializeExternalNativePackage, normalizeExternalPackageSource, parseExternalPackageSourcePolicy } from "./materialize-external-native-package.mjs" +import { readNativeResult } from "./native-result-file.mjs" const requestPath = process.env.AGENT_TASK_REQUEST_PATH || ".codebox/agent-task-request.json" const workspace = resolve(process.env.AGENT_TASK_WORKSPACE || process.cwd()) @@ -191,6 +192,8 @@ const externalPackageSource = normalizeExternalPackageSource(request.external_pa const artifactsPath = join(workspace, ".codebox", "agent-task-artifacts") const runtimeInputPath = join(workspace, ".codebox", "native-agent-task-input.json") const resultPath = join(workspace, ".codebox", "agent-task-workflow-result.json") +const controlledCodeboxPath = resolve(requestPath, "..") +const nativeResultPath = join(controlledCodeboxPath, "native-agent-task-result.json") const runnerWorkspaceTools = [ "workspace-read", "workspace-ls", "workspace-grep", "workspace-write", "workspace-edit", "workspace-apply-patch", "workspace-git-status", "workspace-git-diff", "workspace-git-add", "workspace-git-commit", "workspace-git-push", @@ -266,18 +269,16 @@ await writeFile(runtimeInputPath, `${JSON.stringify(taskInput, null, 2)}\n`) let execution = { code: 0, stdout: "", stderr: "", stdout_truncated: false, stderr_truncated: false } if (request.run_agent && !request.dry_run) { - execution = await command("node", [codeboxCliPath, "agent-task-run", "--input-file", runtimeInputPath, "--json"], workspace, agentEnvironment()) + execution = await command("node", [codeboxCliPath, "agent-task-run", "--input-file", runtimeInputPath, "--result-file", nativeResultPath], workspace, agentEnvironment()) } // Public package bytes are embedded in the runtime recipe and consumed only by // the Playground bootstrap before the agent's tools are resolved. -let runtimeResult = {} -try { - runtimeResult = redact(JSON.parse(execution.stdout)) -} catch { - runtimeResult = { success: false, diagnostics: [{ code: "wp-codebox.agent-task.invalid-result", message: "Native agent-task-run did not return JSON.", stderr: execution.stderr }] } -} +const runtimeResult = request.run_agent && !request.dry_run + ? await readNativeResult(nativeResultPath, controlledCodeboxPath, secretValues, redact) + : {} +await rm(nativeResultPath, { force: true }) await redactArtifactFiles(artifactsPath) diff --git a/.github/scripts/run-agent-task/native-result-file.mjs b/.github/scripts/run-agent-task/native-result-file.mjs new file mode 100644 index 000000000..a12071b2d --- /dev/null +++ b/.github/scripts/run-agent-task/native-result-file.mjs @@ -0,0 +1,47 @@ +import { lstat, readFile } from "node:fs/promises" +import { resolve } from "node:path" + +export const MAX_NATIVE_RESULT_BYTES = 1024 * 1024 +const TERMINAL_STATUSES = new Set(["succeeded", "failed", "no_op", "timeout", "provider_error", "unable_to_remediate"]) + +export function nativeResultFailure(code, message) { + return { success: false, diagnostics: [{ code, message }] } +} + +export async function readNativeResult(path, controlledDirectory, secretValues, redact) { + const resolvedPath = resolve(path) + const resolvedDirectory = resolve(controlledDirectory) + if (!resolvedPath.startsWith(`${resolvedDirectory}/`)) { + return nativeResultFailure("wp-codebox.agent-task.result-path", "Native agent-task result path is outside the controlled .codebox directory.") + } + try { + const info = await lstat(resolvedPath) + if (!info.isFile() || info.isSymbolicLink()) { + return nativeResultFailure("wp-codebox.agent-task.result-file", "Native agent-task result file must be a regular non-symlink file.") + } + if (info.size > MAX_NATIVE_RESULT_BYTES) { + return nativeResultFailure("wp-codebox.agent-task.result-too-large", `Native agent-task result exceeds the ${MAX_NATIVE_RESULT_BYTES}-byte limit.`) + } + const contents = await readFile(resolvedPath, "utf8") + if (Buffer.byteLength(contents) !== info.size) { + return nativeResultFailure("wp-codebox.agent-task.result-file", "Native agent-task result changed while it was being read.") + } + if (secretValues.some((secret) => contents.includes(secret))) { + return nativeResultFailure("wp-codebox.agent-task.result-secret", "Native agent-task result contains a configured secret.") + } + const result = JSON.parse(contents) + const taskResult = record(result) + const summary = record(taskResult.agent_task_run_result) + if (taskResult.schema !== "wp-codebox/agent-task-run/v1" || typeof taskResult.success !== "boolean" || !TERMINAL_STATUSES.has(taskResult.status) || summary.schema !== "wp-codebox/agent-task-run-result/v1" || typeof summary.success !== "boolean" || !TERMINAL_STATUSES.has(summary.status) || taskResult.success !== summary.success || taskResult.status !== summary.status) { + return nativeResultFailure("wp-codebox.agent-task.result-schema", "Native agent-task result did not match the required result schema.") + } + return redact(taskResult) + } catch (error) { + const code = error && typeof error === "object" && error.code === "ENOENT" ? "wp-codebox.agent-task.result-missing" : "wp-codebox.agent-task.result-malformed" + return nativeResultFailure(code, code.endsWith("missing") ? "Native agent-task did not produce a result file." : "Native agent-task result file was malformed.") + } +} + +function record(value) { + return value && typeof value === "object" && !Array.isArray(value) ? value : {} +} diff --git a/README.md b/README.md index 0f70df932..f03fdb989 100644 --- a/README.md +++ b/README.md @@ -1361,7 +1361,7 @@ Generic caller-owned `request.json` payloads may use this shape: } ``` -WP Codebox normalizes the task input, writes the private temporary recipe, runs `wp-codebox recipe-run`, then deletes temporary recipe/seed files. The JSON response keeps `schema: "wp-codebox/agent-task-run/v1"` and includes `agent_task_run_result` with `schema: "wp-codebox/agent-task-run-result/v1"`. Consumers read `agent_task_run_result.status`, `agent_task_run_result.success`, `agent_task_run_result.refs`, `agent_task_run_result.metadata`, and `failure_evidence` as the stable result envelope. Secret values are never accepted in the request or returned in the response; `secret_env` carries names only. +WP Codebox normalizes the task input, writes the private temporary recipe, runs `wp-codebox recipe-run`, then deletes temporary recipe/seed files. The JSON response keeps `schema: "wp-codebox/agent-task-run/v1"` and includes `agent_task_run_result` with `schema: "wp-codebox/agent-task-run-result/v1"`. Consumers read `agent_task_run_result.status`, `agent_task_run_result.success`, `agent_task_run_result.refs`, `agent_task_run_result.metadata`, and `failure_evidence` as the stable result envelope. `--result-file ` atomically writes that final JSON with a private temporary file and a rename; callers that require structured output should use it instead of stdout. Secret values are never accepted in the request or returned in the response; `secret_env` carries names only. Failed `agent-task-run` responses also include `failure_evidence` with `schema: "wp-codebox/agent-task-run-failure-evidence/v1"`. This block is intentionally safe for parent orchestrators to persist alongside their own task failure record. It includes the best available `phase`, `command`, `exit_code`, message, stdout/stderr snippets, runtime and sandbox identifiers, artifact directory or bundle identifiers, recipe-run status/run ID, diagnostics, phase evidence, and the serialized error. If recipe-run fails before normal runtime artifacts exist or returns malformed output, `agent-task-run` still emits this fallback evidence block in the CLI JSON payload and references it from `evidence_refs` with kind `codebox-agent-task-failure-evidence`. diff --git a/docs/agent-runtime-contract.md b/docs/agent-runtime-contract.md index dd51e67db..6359c3bdb 100644 --- a/docs/agent-runtime-contract.md +++ b/docs/agent-runtime-contract.md @@ -143,7 +143,7 @@ WP Codebox rejects product ability paths that pass raw `code` or `code_file`. De } ``` -`agent_task_result` is the sandbox semantic result `wp-codebox/agent-task-result/v1`. `agent_task_run_result` is the caller-facing `wp-codebox/agent-task-run-result/v1` contract emitted by the CLI and WordPress ability surfaces. Orchestrators read `agent_task_run_result.status`, `agent_task_run_result.success`, `agent_task_run_result.refs`, and `agent_task_run_result.metadata` as the stable result envelope. Node consumers can also normalize older or nested envelopes with `normalizeAgentTaskRunResult()` from `@automattic/wp-codebox-core` or `wp-codebox-workspace/core`; the package exports `AGENT_TASK_RUN_RESULT_SCHEMA` and `AGENT_TASK_RUN_RESULT_JSON_SCHEMA` for contract checks. The normalized envelope groups stable artifact refs into `artifact_bundles`, `changed_files`, `patches`, `transcripts`, `logs`, and `runtimes`, and classifies terminal statuses including `succeeded`, `failed`, `no_op`, `timeout`, `provider_error`, and `unable_to_remediate`. +`agent_task_result` is the sandbox semantic result `wp-codebox/agent-task-result/v1`. `agent_task_run_result` is the caller-facing `wp-codebox/agent-task-run-result/v1` contract emitted by the CLI and WordPress ability surfaces. Orchestrators read `agent_task_run_result.status`, `agent_task_run_result.success`, `agent_task_run_result.refs`, and `agent_task_run_result.metadata` as the stable result envelope. For process execution, pass `--result-file `: the CLI writes the final envelope through a mode `0600` temporary file and atomic rename. stdout and stderr remain diagnostic streams and must not be used as required structured transport. Node consumers can also normalize older or nested envelopes with `normalizeAgentTaskRunResult()` from `@automattic/wp-codebox-core` or `wp-codebox-workspace/core`; the package exports `AGENT_TASK_RUN_RESULT_SCHEMA` and `AGENT_TASK_RUN_RESULT_JSON_SCHEMA` for contract checks. The normalized envelope groups stable artifact refs into `artifact_bundles`, `changed_files`, `patches`, `transcripts`, `logs`, and `runtimes`, and classifies terminal statuses including `succeeded`, `failed`, `no_op`, `timeout`, `provider_error`, and `unable_to_remediate`. Failed `agent-task-run` responses include `wp-codebox/agent-task-run-failure-evidence/v1` in `failure_evidence` when available. This block is safe for orchestrators to persist with job failure records and includes phase, command, exit code, redacted stdout/stderr snippets, runtime and sandbox identifiers, artifact references, diagnostics, and serialized error data. diff --git a/docs/agent-task-reusable-workflow.md b/docs/agent-task-reusable-workflow.md index 8d0284a85..5ba4a8093 100644 --- a/docs/agent-task-reusable-workflow.md +++ b/docs/agent-task-reusable-workflow.md @@ -183,4 +183,4 @@ workflow itself cannot inspect the caller's `uses:` declaration at runtime. The reusable workflow stages every uploaded file through a fail-closed policy. Only regular UTF-8 files of 4 MiB or less are uploaded, after configured secret values are redacted. Symlinks, special files, binary files, and files larger than 4 MiB are excluded from the upload staging directory. This applies to task artifacts and workflow request, input, and result files. -Native task execution, validation-dependency installation, verification, drift checks, and GitHub pull-request validation retain at most 32 KiB from each stdout and stderr stream while continuing to drain both streams. Command results expose `stdout_truncated` and `stderr_truncated` when retained output is incomplete. +Native task execution, validation-dependency installation, verification, drift checks, and GitHub pull-request validation retain at most 32 KiB from each stdout and stderr stream while continuing to drain both streams. Command results expose `stdout_truncated` and `stderr_truncated` when retained output is incomplete. Native `agent-task-run` structured output is instead read from a controlled `.codebox/native-agent-task-result.json` file outside the agent-writable target checkout. The executor accepts only a regular, non-symlink JSON file up to 1 MiB with the required result schemas, rejects configured secret values, redacts the accepted record before persistence, and removes the transient native result file after reading it. diff --git a/packages/cli/src/commands/agent-task-run.ts b/packages/cli/src/commands/agent-task-run.ts index 4631f7e07..1fafac248 100644 --- a/packages/cli/src/commands/agent-task-run.ts +++ b/packages/cli/src/commands/agent-task-run.ts @@ -1,6 +1,6 @@ -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { lstat, mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" -import { join } from "node:path" +import { dirname, join } from "node:path" import { AGENT_TASK_RUN_REQUEST_SCHEMA, HEADLESS_AGENT_TASK_REQUEST_SCHEMA, artifactResultEnvelope, buildAgentTaskRecipe, DEFAULT_WORDPRESS_VERSION, headlessAgentTaskRequestToRunInput, normalizeAgentRuntimeWorkload, normalizeAgentTaskRunResult, normalizeAgentTerminalResult, normalizeArtifactResultTypedArtifacts, normalizeHeadlessAgentTaskRequest, normalizeHeadlessAgentTaskResult, normalizeTaskInput, parseCommandJson, parseCommandOptions, resolveEffectiveRuntimeToolPolicy, type AgentTaskRunInput, type AgentTaskRunResultSummary, type AgentTerminalResult, type ArtifactResultEnvelope, type HeadlessAgentTaskResult, type SandboxToolPolicySnapshot, type TypedArtifactDTO } from "@automattic/wp-codebox-core" import { stripUndefined } from "@automattic/wp-codebox-core/internals" import { runRecipeRunCommand } from "./recipe-run.js" @@ -10,6 +10,7 @@ export type { AgentTaskRunInput } from "@automattic/wp-codebox-core" export interface AgentTaskRunOptions { inputPath: string json: boolean + resultFile?: string previewHoldSeconds?: string previewPublicUrl?: string previewPort?: string @@ -79,9 +80,14 @@ export async function runAgentTaskRunCommand(args: string[]): Promise { const options = parseAgentTaskRunOptions(args) const input = normalizeAgentTaskRunCliInput(JSON.parse(await readFile(options.inputPath, "utf8"))) const output = await runAgentTask(input, options) + const jsonOutput = agentTaskRunJsonOutput(output) + + if (options.resultFile) { + await writeAgentTaskRunResultFile(options.resultFile, jsonOutput) + } if (options.json) { - process.stdout.write(`${JSON.stringify(agentTaskRunJsonOutput(output), null, 2)}\n`) + process.stdout.write(`${JSON.stringify(jsonOutput, null, 2)}\n`) return agentTaskRunExitCode(output) } @@ -391,7 +397,7 @@ function parseAgentTaskRunOptions(args: string[]): AgentTaskRunOptions { throw new Error(`Unknown agent-task-run option: ${positionals[0]}`) } for (const name of options.keys()) { - if (!["--input-file", "--json", "--format", "--preview-hold-seconds", "--preview-public-url", "--preview-port", "--preview-bind", "--preview-hold-blocking", "--preview-lease-json"].includes(name)) { + if (!["--input-file", "--json", "--format", "--result-file", "--preview-hold-seconds", "--preview-public-url", "--preview-port", "--preview-bind", "--preview-hold-blocking", "--preview-lease-json"].includes(name)) { throw new Error(`Unknown agent-task-run option: ${name}`) } } @@ -402,6 +408,7 @@ function parseAgentTaskRunOptions(args: string[]): AgentTaskRunOptions { return { inputPath, json: options.get("--json") === true || stringOption(options, "--format") === "json", + resultFile: stringOption(options, "--result-file") || undefined, previewHoldSeconds: stringOption(options, "--preview-hold-seconds"), previewPublicUrl: stringOption(options, "--preview-public-url"), previewPort: stringOption(options, "--preview-port"), @@ -411,6 +418,32 @@ function parseAgentTaskRunOptions(args: string[]): AgentTaskRunOptions { } } +/** Write caller-owned structured output without exposing an incomplete result. */ +export async function writeAgentTaskRunResultFile(path: string, output: AgentTaskRunOutput | HeadlessAgentTaskResult): Promise { + const parent = dirname(path) + await mkdir(parent, { recursive: true, mode: 0o700 }) + const parentStat = await lstat(parent) + if (!parentStat.isDirectory() || parentStat.isSymbolicLink()) { + throw new Error("agent-task-run result-file parent must be a non-symlink directory") + } + try { + const targetStat = await lstat(path) + if (!targetStat.isFile() || targetStat.isSymbolicLink()) { + throw new Error("agent-task-run result-file must be a regular file when it already exists") + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + } + + const temporaryPath = `${path}.${process.pid}.${Date.now()}.tmp` + try { + await writeFile(temporaryPath, `${JSON.stringify(output, null, 2)}\n`, { encoding: "utf8", mode: 0o600, flag: "wx" }) + await rename(temporaryPath, path) + } finally { + await rm(temporaryPath, { force: true }) + } +} + function objectRecord(value: unknown): Record | undefined { return value && typeof value === "object" && !Array.isArray(value) ? value as Record : undefined } diff --git a/packages/cli/src/output.ts b/packages/cli/src/output.ts index a813b35e3..19384e680 100644 --- a/packages/cli/src/output.ts +++ b/packages/cli/src/output.ts @@ -331,7 +331,7 @@ export function printHelp(): void { wp-codebox fuzz-minimize-case --input-file [--format=json] [--dry-run] wp-codebox run-wordpress-workload --input-file [--format=json] [--dry-run] wp-codebox run-agent-task --input-file [--json] [--preview-hold-seconds ] [--preview-hold-blocking] [--preview-port ] [--preview-bind ] [--preview-public-url ] [--preview-lease-json ] - wp-codebox agent-task-run --input-file [--json] [--preview-hold-seconds ] [--preview-hold-blocking] [--preview-port ] [--preview-bind ] [--preview-public-url ] [--preview-lease-json ] + wp-codebox agent-task-run --input-file [--json] [--result-file ] [--preview-hold-seconds ] [--preview-hold-blocking] [--preview-port ] [--preview-bind ] [--preview-public-url ] [--preview-lease-json ] wp-codebox validate-blueprint --blueprint [options] wp-codebox materialize-replay-package --snapshot --output [--snapshot-ref ] [--json] wp-codebox recipe-run --recipe [options] @@ -343,6 +343,7 @@ Options: --options Recipe builder options JSON file for recipe build. --output Recipe build output JSON path, or materialize-replay-package output directory. --input-file Input JSON for public workload/fuzz commands or agent-task-run. + --result-file Atomically write the final agent-task-run JSON result to a caller-owned file. --format=json Emit machine-readable JSON; accepted by public workload/fuzz commands. --preview-hold-seconds Keep preview runtimes alive after run-agent-task/agent-task-run/recipe-run. diff --git a/tests/agent-task-contracts.test.ts b/tests/agent-task-contracts.test.ts index becb78787..f8029ce1e 100644 --- a/tests/agent-task-contracts.test.ts +++ b/tests/agent-task-contracts.test.ts @@ -1,12 +1,12 @@ import assert from "node:assert/strict" -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { mkdirSync, mkdtempSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" import { chdir, cwd } from "node:process" import { AGENT_TASK_RUN_REQUEST_SCHEMA, AGENT_TASK_RUN_RESULT_JSON_SCHEMA, AGENT_TASK_RUN_RESULT_SCHEMA, ARTIFACT_RESULT_ENVELOPE_SCHEMA, HEADLESS_AGENT_TASK_REQUEST_JSON_SCHEMA, HEADLESS_AGENT_TASK_REQUEST_SCHEMA, HEADLESS_AGENT_TASK_RESULT_JSON_SCHEMA, HEADLESS_AGENT_TASK_RESULT_SCHEMA, PREVIEW_LEASE_SCHEMA, TYPED_ARTIFACT_SCHEMA, buildAgentTaskRecipe, headlessAgentTaskRequestToRunInput, normalizeAgentRuntimeWorkload, normalizeAgentTaskRunResult, normalizeAgentTerminalResult, normalizeArtifactResultTypedArtifacts, normalizeHeadlessAgentTaskRequest, normalizeHeadlessAgentTaskResult, normalizeRecipeRunSummary, normalizeTaskInput } from "../packages/runtime-core/src/index.js" import { effectivePolicyCommands } from "../packages/runtime-core/src/contracts.js" import { commandCatalogOutput } from "../packages/cli/src/commands/discovery.js" -import { agentTaskResultFromRun, agentTaskRunExitCode, agentTaskRunJsonOutput, normalizeAgentTaskRunCliInput } from "../packages/cli/src/commands/agent-task-run.js" +import { agentTaskResultFromRun, agentTaskRunExitCode, agentTaskRunJsonOutput, normalizeAgentTaskRunCliInput, writeAgentTaskRunResultFile } from "../packages/cli/src/commands/agent-task-run.js" import { agentSandboxRunCode, resolveSandboxTaskCode } from "../packages/cli/src/agent-code.js" import { dryRunRecipe } from "../packages/cli/src/recipe-dry-run.js" import { recipePolicy } from "../packages/cli/src/recipe-validation.js" @@ -18,6 +18,27 @@ assert.equal(succeeded.schema, AGENT_TASK_RUN_RESULT_SCHEMA) assert.equal(succeeded.status, "succeeded") assert.equal(agentTaskRunExitCode({ success: true, agent_task_run_result: succeeded }), 0) +const resultFileDirectory = mkdtempSync(join(tmpdir(), "wp-codebox-agent-task-result-file-")) +const resultFilePath = join(resultFileDirectory, "result.json") +const atomicResult = { schema: "wp-codebox/agent-task-run/v1", success: true, status: "succeeded", padding: "x".repeat(40 * 1024) } +await writeAgentTaskRunResultFile(resultFilePath, atomicResult as never) +assert.deepEqual(JSON.parse(await (await import("node:fs/promises")).readFile(resultFilePath, "utf8")), atomicResult) +assert.equal(statSync(resultFilePath).mode & 0o777, 0o600) +const originalNow = Date.now +Date.now = () => 1 +const interruptedTemporaryPath = `${resultFilePath}.${process.pid}.1.tmp` +writeFileSync(interruptedTemporaryPath, "partial") +try { + await assert.rejects(writeAgentTaskRunResultFile(resultFilePath, atomicResult as never), /EEXIST/) +} finally { + Date.now = originalNow +} +assert.deepEqual(readdirSync(resultFileDirectory).filter((entry) => entry.includes(".tmp")), [], "Interrupted atomic writes must clean up temporary files") +mkdirSync(join(resultFileDirectory, "interrupted.json")) +await assert.rejects(writeAgentTaskRunResultFile(join(resultFileDirectory, "interrupted.json"), atomicResult as never), /regular file/) +assert.deepEqual(readdirSync(resultFileDirectory).filter((entry) => entry.includes(".tmp")), [], "Failed result-file writes must not leave partial temporary files") +rmSync(resultFileDirectory, { recursive: true, force: true }) + const succeededWithAccess = normalizeAgentTaskRunResult({ success: true, status: "completed", outputs: { preview_url: "https://preview.example.test", site_url: "https://site.example.test" } }, { exitStatus: 0 }) assert.equal(succeededWithAccess.runtime_access?.schema, "wp-codebox/runtime-access/v1") assert.equal(succeededWithAccess.runtime_access?.preview_url, "https://preview.example.test") diff --git a/tests/agent-task-reusable-workflow.test.ts b/tests/agent-task-reusable-workflow.test.ts index 1f7c32b46..53c68c353 100644 --- a/tests/agent-task-reusable-workflow.test.ts +++ b/tests/agent-task-reusable-workflow.test.ts @@ -1,10 +1,11 @@ import assert from "node:assert/strict" import { execFile } from "node:child_process" -import { mkdir, mkdtemp, readFile, readdir, symlink, writeFile } from "node:fs/promises" +import { mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" import { promisify } from "node:util" import { sha256BytesV1 } from "../.github/scripts/run-agent-task/materialize-external-native-package.mjs" +import { MAX_NATIVE_RESULT_BYTES, readNativeResult } from "../.github/scripts/run-agent-task/native-result-file.mjs" const execFileAsync = promisify(execFile) @@ -97,6 +98,35 @@ assert.match(docs, /not a WordPress Playground end-to-end test/) assert.doesNotMatch(docs.slice(0, docs.indexOf("## Runtime Coverage")), /docs-agent|wp-codebox\/docs-agent-runner-recipe\/v1|recipe_path|recipe_json|wp_codebox_ref|datamachine|data machine|data-machine|agents api|sandbox mounts|ability ids|provider internals|homeboy|require_app_token/i) const tmp = await mkdtemp(join(tmpdir(), "wp-codebox-agent-task-workflow-")) +const controlledCodeboxPath = join(tmp, ".codebox") +await mkdir(controlledCodeboxPath) +const nativeResultPath = join(controlledCodeboxPath, "native-agent-task-result.json") +const nativeResult = { + schema: "wp-codebox/agent-task-run/v1", + success: true, + status: "succeeded", + agent_task_run_result: { schema: "wp-codebox/agent-task-run-result/v1", success: true, status: "succeeded" }, + padding: "x".repeat(40 * 1024), +} +const redactNativeResult = (value: unknown) => value +await writeFile(nativeResultPath, JSON.stringify(nativeResult)) +const hostedTruncationRegression = await readNativeResult(nativeResultPath, controlledCodeboxPath, ["native-secret"], redactNativeResult) +assert.equal(hostedTruncationRegression.success, true, "A valid result larger than the 32 KiB stdout diagnostic limit must be read from the result file") +assert.equal((hostedTruncationRegression as Record).padding, nativeResult.padding) +assert.match(await readFile(new URL("../.github/scripts/run-agent-task/execute-native-agent-task.mjs", import.meta.url), "utf8"), /"--result-file", nativeResultPath/) +await writeFile(nativeResultPath, "{") +assert.equal((await readNativeResult(nativeResultPath, controlledCodeboxPath, [], redactNativeResult) as any).diagnostics[0].code, "wp-codebox.agent-task.result-malformed") +await writeFile(nativeResultPath, JSON.stringify({ ...nativeResult, status: "pending" })) +assert.equal((await readNativeResult(nativeResultPath, controlledCodeboxPath, [], redactNativeResult) as any).diagnostics[0].code, "wp-codebox.agent-task.result-schema") +await writeFile(nativeResultPath, JSON.stringify({ ...nativeResult, padding: "x".repeat(MAX_NATIVE_RESULT_BYTES) })) +assert.equal((await readNativeResult(nativeResultPath, controlledCodeboxPath, [], redactNativeResult) as any).diagnostics[0].code, "wp-codebox.agent-task.result-too-large") +await writeFile(nativeResultPath, JSON.stringify({ ...nativeResult, padding: "native-secret" })) +assert.equal((await readNativeResult(nativeResultPath, controlledCodeboxPath, ["native-secret"], redactNativeResult) as any).diagnostics[0].code, "wp-codebox.agent-task.result-secret") +await writeFile(join(controlledCodeboxPath, "outside.json"), JSON.stringify(nativeResult)) +await rm(nativeResultPath) +await symlink(join(controlledCodeboxPath, "outside.json"), nativeResultPath) +assert.equal((await readNativeResult(nativeResultPath, controlledCodeboxPath, [], redactNativeResult) as any).diagnostics[0].code, "wp-codebox.agent-task.result-file") +assert.equal((await readNativeResult(join(tmp, "outside.json"), controlledCodeboxPath, [], redactNativeResult) as any).diagnostics[0].code, "wp-codebox.agent-task.result-path") const nativePackageRepository = join(tmp, "native-package-repository") const nativePackagePath = join(nativePackageRepository, "packages", "example-agent.agent.json") await mkdir(join(nativePackageRepository, "packages"), { recursive: true })