diff --git a/.github/scripts/run-agent-task/execute-native-agent-task.mjs b/.github/scripts/run-agent-task/execute-native-agent-task.mjs index d5b5db1e3..f6327aa63 100644 --- a/.github/scripts/run-agent-task/execute-native-agent-task.mjs +++ b/.github/scripts/run-agent-task/execute-native-agent-task.mjs @@ -1,6 +1,6 @@ import { rmSync } from "node:fs" import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises" -import { join, resolve } from "node:path" +import { join, relative, resolve } from "node:path" import { spawn } from "node:child_process" import { materializeExternalNativePackage, materializeRuntimeSources, normalizeExternalPackageSource, normalizeRuntimeSources, parseExternalPackageSourcePolicy, validateRuntimeSourceModel } from "./materialize-external-native-package.mjs" import { readNativeResult } from "./native-result-file.mjs" @@ -14,6 +14,8 @@ const outputPath = process.env.GITHUB_OUTPUT const MAX_CAPTURE_BYTES = 32768 const MAX_OUTPUT_CHARS = 8192 const secretValues = ["OPENAI_API_KEY", "MODEL_PROVIDER_SECRET_1", "MODEL_PROVIDER_SECRET_2", "MODEL_PROVIDER_SECRET_3", "MODEL_PROVIDER_SECRET_4", "MODEL_PROVIDER_SECRET_5", "GITHUB_TOKEN", "GH_TOKEN", "ACCESS_TOKEN", "EXTERNAL_PACKAGE_SOURCE_POLICY"].map((name) => process.env[name]).filter(Boolean) +let privateRuntimeSourceRoot = "" +let privateRuntimeSourceRootForSanitization = "" function redact(value) { if (typeof value === "string") return secretValues.reduce((output, secret) => output.split(secret).join("[REDACTED]"), value) if (Array.isArray(value)) return value.map(redact) @@ -173,6 +175,45 @@ function projections(value, runtimeResult) { return output } +function workflowPath(path) { + const relativePath = relative(workspace, resolve(path)) + return relativePath && !relativePath.startsWith("..") ? relativePath.replaceAll("\\", "/") : ".codebox/agent-task-workflow-result.json" +} + +function failureClassification(error) { + const message = error instanceof Error ? error.message : String(error) + const code = typeof error?.code === "string" && error.code ? error.code : "" + if (code.includes(".policy")) return { code, classification: "policy" } + if (code && !/materializ|fetch|download|archive|entrypoint|git failed|spawn git/i.test(message)) return { code, classification: "native-agent-task" } + if (/policy|authorized|allowlisted|allowed_repos|ACCESS_TOKEN/i.test(message)) return { code: "wp-codebox.agent-task.policy", classification: "policy" } + if (/materializ|fetch|download|archive|entrypoint|git failed|spawn git/i.test(message)) return { code: "wp-codebox.agent-task.materialization", classification: "materialization" } + if (/approval|publication|pull request/i.test(message)) return { code: "wp-codebox.agent-task.approval", classification: "approval" } + if (/projection/i.test(message)) return { code: "wp-codebox.agent-task.output-projection", classification: "output-projection" } + return { code: "wp-codebox.agent-task.execution", classification: "execution" } +} + +async function writeNormalizedFailure(error, request = {}) { + const resultPath = join(workspace, ".codebox", "agent-task-workflow-result.json") + const failure = failureClassification(error) + const message = bounded(error instanceof Error ? error.message : String(error), MAX_OUTPUT_CHARS) + const accessError = failure.classification === "policy" && /allowed_repos|ACCESS_TOKEN|GitHub token|Caller repository|authorized/i.test(message) + const result = { + schema: "wp-codebox/agent-task-workflow-result/v1", + run_id: `${record(request).workload?.id || "agent-task"}-${process.env.GITHUB_RUN_ID || "local"}`.replace(/[^A-Za-z0-9._-]+/g, "-"), + status: "failed", + success: false, + request_path: workflowPath(requestPath), + failure: { ...failure, message }, + ...(accessError ? { access: { authorized: false, error: message } } : {}), + } + await mkdir(join(workspace, ".codebox"), { recursive: true }) + const sanitized = sanitizeRuntimeSourceValue(redact(result), privateRuntimeSourceRootForSanitization) + assertNoRuntimeSourcePaths(sanitized, privateRuntimeSourceRootForSanitization) + await writeFile(resultPath, `${JSON.stringify(sanitized, null, 2)}\n`) + await output("job_status", "failed") + await output("result_path", ".codebox/agent-task-workflow-result.json") +} + async function redactArtifactFiles(directory) { const { readdir, stat } = await import("node:fs/promises") for (const entry of await readdir(directory, { withFileTypes: true })) { @@ -189,6 +230,7 @@ async function redactArtifactFiles(directory) { } } +async function executeNativeAgentTask() { const request = JSON.parse(await readFile(requestPath, "utf8")) const verificationCommands = commandEntries(request.verification_commands, "verification_commands") const driftChecks = commandEntries(request.drift_checks, "drift_checks") @@ -202,8 +244,6 @@ const runtimeInputPath = join(workspace, ".codebox", "native-agent-task-input.js const resultPath = join(workspace, ".codebox", "agent-task-workflow-result.json") const controlledCodeboxPath = resolve(requestPath, "..") const nativeResultPath = join(controlledCodeboxPath, "native-agent-task-result.json") -let privateRuntimeSourceRoot = "" -let privateRuntimeSourceRootForSanitization = "" let cleaningPrivateRuntimeSources = false async function cleanupPrivateRuntimeSources() { if (cleaningPrivateRuntimeSources || !privateRuntimeSourceRoot) return @@ -232,11 +272,9 @@ await mkdir(artifactsPath, { recursive: true }) const accessError = accessFailure(request) if (accessError) { - const result = { schema: "wp-codebox/agent-task-workflow-result/v1", run_id: runId, status: "failed", success: false, request_path: requestPath, access: { authorized: false, error: accessError } } - await writeFile(resultPath, `${JSON.stringify(result, null, 2)}\n`) - await output("job_status", "failed") - process.exitCode = 1 - process.exit() + const error = new Error(accessError) + error.code = "wp-codebox.agent-task.policy" + throw error } const materializedPackage = request.run_agent && !request.dry_run @@ -404,3 +442,20 @@ await output("declared_artifacts_json", result.artifacts.declarations) await output("result_path", ".codebox/agent-task-workflow-result.json") if (!success) process.exitCode = 1 +} + +try { + await executeNativeAgentTask() +} catch (error) { + try { + await writeNormalizedFailure(error) + } finally { + if (privateRuntimeSourceRoot) { + const root = privateRuntimeSourceRoot + privateRuntimeSourceRoot = "" + await rm(root, { recursive: true, force: true }) + } + } + console.error(bounded(error instanceof Error ? error.message : String(error), MAX_OUTPUT_CHARS)) + process.exitCode = 1 +} diff --git a/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs b/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs index 069cce35f..68ceed0a8 100644 --- a/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs +++ b/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs @@ -8,83 +8,178 @@ const MAX_UPLOAD_FILE_BYTES = 4 * 1024 * 1024 const workspace = resolve(process.env.AGENT_TASK_WORKSPACE || process.cwd()) const uploadPath = resolve(process.env.AGENT_TASK_UPLOAD_PATH || join(workspace, ".codebox", "agent-task-upload")) const requestPath = resolve(process.env.AGENT_TASK_REQUEST_PATH || join(workspace, ".codebox", "agent-task-request.json")) +const artifactsPath = join(workspace, ".codebox", "agent-task-artifacts") const secretValues = ["OPENAI_API_KEY", "MODEL_PROVIDER_SECRET_1", "MODEL_PROVIDER_SECRET_2", "MODEL_PROVIDER_SECRET_3", "MODEL_PROVIDER_SECRET_4", "MODEL_PROVIDER_SECRET_5", "GITHUB_TOKEN", "GH_TOKEN", "ACCESS_TOKEN", "EXTERNAL_PACKAGE_SOURCE_POLICY"].map((name) => process.env[name]).filter(Boolean) const runtimeSourceRoot = process.env.WP_CODEBOX_RUNTIME_SOURCE_ROOT ? resolve(process.env.WP_CODEBOX_RUNTIME_SOURCE_ROOT) : "" const runtimeSourcePrefix = process.env.WP_CODEBOX_RUNTIME_SOURCE_PREFIX ? resolve(process.env.WP_CODEBOX_RUNTIME_SOURCE_PREFIX) : "" const runtimeSourceRoots = [runtimeSourceRoot, runtimeSourcePrefix].filter(Boolean) -const RUNTIME_SOURCE_TREE = /(^|\/)(prepared-plugins|agents-api|ai-provider-for-openai)(\/|$)/ -const RUNTIME_SOURCE_FILE = /^(agents-api\.php|plugin\.php)$/ +const SOURCE_TREE = /(^|\/)(prepared-plugins|prepared-source-packages|source-package[^/]*)(\/|$)/i +const SOURCE_FILE = /\.(?:php|phtml|js|mjs|cjs|jsx|ts|tsx)$/i const RUNTIME_SOURCE_CONTENT = /(?:Plugin Name:|WP_Agents_Registry|OpenAiProvider)/ function redact(value) { return secretValues.reduce((output, secret) => output.split(secret).join("[REDACTED]"), value) } +function sanitizeText(text) { + return sanitizeRuntimeSourceJson(text, runtimeSourceRoots) +} + +function compactNativeInput(text) { + const privateFields = new Set(["source_package_root", "component_contracts", "extra_plugins", "provider_plugins", "runtime_overlays", "prepared_sources"]) + const compact = (value) => { + if (Array.isArray(value)) return value.map(compact) + const entry = record(value) + if (!Object.keys(entry).length) return value + return Object.fromEntries(Object.entries(entry).flatMap(([key, item]) => privateFields.has(key) ? [] : [[key, compact(item)]])) + } + try { + return `${JSON.stringify(compact(JSON.parse(sanitizeText(text))), null, 2)}\n` + } catch { + return sanitizeText(text) + } +} + function isPrivateRuntimePath(value) { - if (!runtimeSourceRoot || typeof value !== "string") return false + if (!runtimeSourceRoots.length || typeof value !== "string") return false const path = resolve(value) - const contained = relative(runtimeSourceRoot, path) - return path === runtimeSourceRoot || (contained !== ".." && !contained.startsWith(`..${String.fromCharCode(47)}`) && !isAbsolute(contained)) + return runtimeSourceRoots.some((root) => { + const contained = relative(root, path) + return path === root || (contained !== ".." && !contained.startsWith(`..${String.fromCharCode(47)}`) && !isAbsolute(contained)) + }) } -function sanitizeText(text) { - return sanitizeRuntimeSourceJson(text, runtimeSourceRoots) +function safeRelativeArtifactPath(value) { + if (typeof value !== "string" || !value.trim() || isAbsolute(value)) return "" + const path = value.replace(/\\/g, "/").replace(/^\.\//, "") + if (path.split("/").some((part) => !part || part === "." || part === "..")) return "" + return path } -async function stageFile(source, destination) { - if (isPrivateRuntimePath(source)) { - throw new Error("Runtime source files must never be staged for artifact upload.") - } +function sourceCategory(path, absolutePath) { + if (isPrivateRuntimePath(absolutePath)) return "private-runtime" + if (SOURCE_TREE.test(path)) return "source-tree" + if (SOURCE_FILE.test(path)) return "source-file" + return "" +} + +async function stageTextFile(source, destination, options = {}) { const metadata = await lstat(source).catch(() => null) if (!metadata?.isFile() || metadata.size > MAX_UPLOAD_FILE_BYTES) return false - if (RUNTIME_SOURCE_TREE.test(source) || RUNTIME_SOURCE_FILE.test(source.split("/").pop() || "")) { - throw new Error("Prepared runtime plugin sources must never be staged for artifact upload.") - } const handle = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW).catch(() => null) if (!handle) return false const openedMetadata = await handle.stat() const contents = openedMetadata.isFile() && openedMetadata.size <= MAX_UPLOAD_FILE_BYTES ? await handle.readFile() : null await handle.close() if (!contents || contents.includes(0) || !isUtf8(contents)) return false - await mkdir(resolve(destination, ".."), { recursive: true }) - let text = contents.toString("utf8") - if (RUNTIME_SOURCE_CONTENT.test(text)) { - throw new Error("Prepared runtime plugin source contents must never be staged for artifact upload.") - } - text = sanitizeText(text) + const text = redact(options.compactNativeInput ? compactNativeInput(contents.toString("utf8")) : sanitizeText(contents.toString("utf8"))) assertNoRuntimeSourcePaths(text, runtimeSourceRoots, "Runtime source paths must never be persisted in artifact uploads.") - await writeFile(destination, redact(text)) + if (RUNTIME_SOURCE_CONTENT.test(text)) throw new Error("Prepared runtime plugin source contents must never be staged for artifact upload.") + await mkdir(resolve(destination, ".."), { recursive: true }) + await writeFile(destination, text) return true } -async function stageDirectory(source, destination) { - const metadata = await lstat(source).catch(() => null) - if (!metadata?.isDirectory()) return - for (const entry of await readdir(source, { withFileTypes: true })) { - const entrySource = join(source, entry.name) - const entryDestination = join(destination, entry.name) - if (entry.isDirectory()) await stageDirectory(entrySource, entryDestination) - else if (entry.isFile()) await stageFile(entrySource, entryDestination) +function record(value) { + return value && typeof value === "object" && !Array.isArray(value) ? value : {} +} + +function declarations(request) { + return (Array.isArray(record(request).artifacts?.declarations) ? record(request).artifacts.declarations : []) + .flatMap((declaration) => { + const entry = record(declaration) + return typeof entry.name === "string" && entry.name.trim() + ? [{ name: entry.name.trim(), type: typeof entry.type === "string" ? entry.type.trim() : "" }] + : [] + }) +} + +function declaredArtifactPaths(result, allowed) { + const paths = new Set() + const visit = (value) => { + if (Array.isArray(value)) return value.forEach(visit) + const entry = record(value) + if (!Object.keys(entry).length) return + const artifact = record(entry.artifact) + const path = safeRelativeArtifactPath(artifact.path) + const declared = allowed.some((candidate) => candidate.name === entry.name && (!candidate.type || candidate.type === entry.type)) + if (path && declared) paths.add(path) + Object.values(entry).forEach(visit) + } + visit(result) + return [...paths].sort() +} + +async function exclusions(root, declaredPaths) { + const counts = new Map() + const count = (category) => counts.set(category, (counts.get(category) || 0) + 1) + const visit = async (directory) => { + const entries = await readdir(directory, { withFileTypes: true }).catch(() => []) + for (const entry of entries) { + const source = join(directory, entry.name) + const path = relative(root, source).replaceAll("\\", "/") + if (entry.isDirectory()) await visit(source) + else if (entry.isFile()) { + const category = sourceCategory(path, source) + if (category) count(category) + else if (!declaredPaths.has(path)) count("undeclared-artifact") + } else count("special-file") + } } + await visit(root) + return [...counts.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([category, count]) => ({ category, count })) +} + +function runtimeProvenance(request) { + const sources = Array.isArray(record(request).runtime_sources) ? record(request).runtime_sources : [] + return sources.flatMap((source) => { + const entry = record(source) + if (typeof entry.role !== "string") return [] + const provenance = { role: entry.role } + if (record(entry.source).type === "https_zip") { + const sourceInfo = record(entry.source) + provenance.source = Object.fromEntries(["type", "url", "sha256", "archive_root"].flatMap((key) => typeof sourceInfo[key] === "string" ? [[key, sourceInfo[key]]] : [])) + } else Object.assign(provenance, ...["repository", "revision", "digest"].flatMap((key) => typeof entry[key] === "string" ? [{ [key]: entry[key] }] : [])) + return [provenance] + }) } -async function assertNoPrivateRuntimePaths(directory) { +async function finalScan(directory) { for (const entry of await readdir(directory, { withFileTypes: true })) { const path = join(directory, entry.name) - if (entry.isDirectory()) await assertNoPrivateRuntimePaths(path) + const relativePath = relative(uploadPath, path).replaceAll("\\", "/") + if (sourceCategory(relativePath, path)) throw new Error("Prepared runtime plugin sources must never be persisted in artifact uploads.") + if (entry.isDirectory()) await finalScan(path) else if (entry.isFile()) { - const contents = await readFile(path, "utf8") - assertNoRuntimeSourcePaths(contents, runtimeSourceRoots, "Runtime source paths must never be persisted in artifact uploads.") - if (RUNTIME_SOURCE_TREE.test(path) || RUNTIME_SOURCE_FILE.test(entry.name) || RUNTIME_SOURCE_CONTENT.test(contents)) throw new Error("Prepared runtime plugin sources must never be persisted in artifact uploads.") - } + const bytes = await readFile(path) + const text = isUtf8(bytes) ? bytes.toString("utf8") : "" + assertNoRuntimeSourcePaths(text, runtimeSourceRoots, "Runtime source paths must never be persisted in artifact uploads.") + if (RUNTIME_SOURCE_CONTENT.test(text)) throw new Error("Prepared runtime plugin source contents must never be persisted in artifact uploads.") + } else throw new Error("Only regular files may be persisted in artifact uploads.") } } +const parseJsonOrEmpty = (text) => { + try { return JSON.parse(text) } catch { return {} } +} +const request = parseJsonOrEmpty(await readFile(requestPath, "utf8").catch(() => "{}")) +const resultSource = join(workspace, ".codebox", "agent-task-workflow-result.json") +const result = parseJsonOrEmpty(await readFile(resultSource, "utf8").catch(() => "{}")) +const declaredPaths = new Set(declaredArtifactPaths(result, declarations(request))) + await rm(uploadPath, { recursive: true, force: true }) await mkdir(uploadPath, { recursive: true }) -await stageFile(requestPath, join(uploadPath, ".codebox", "agent-task-request.json")) -for (const path of [".codebox/agent-task-workflow-result.json", ".codebox/native-agent-task-input.json"]) { - await stageFile(join(workspace, path), join(uploadPath, path)) +await stageTextFile(requestPath, join(uploadPath, ".codebox", "agent-task-request.json")) +await stageTextFile(resultSource, join(uploadPath, ".codebox", "agent-task-workflow-result.json")) +await stageTextFile(join(workspace, ".codebox", "native-agent-task-input.json"), join(uploadPath, ".codebox", "native-agent-task-input.json"), { compactNativeInput: true }) +for (const path of declaredPaths) { + const source = resolve(artifactsPath, path) + if (relative(artifactsPath, source).startsWith("..") || sourceCategory(path, source)) { + throw new Error("Declared reviewer artifacts must not reference source files or private runtime internals.") + } + await stageTextFile(source, join(uploadPath, ".codebox", "agent-task-artifacts", path)) } -await stageDirectory(join(workspace, ".codebox", "agent-task-artifacts"), join(uploadPath, ".codebox", "agent-task-artifacts")) -await assertNoPrivateRuntimePaths(uploadPath) +await mkdir(join(uploadPath, ".codebox", "agent-task-artifacts"), { recursive: true }) +await writeFile(join(uploadPath, ".codebox", "agent-task-artifacts", "runtime-provenance.json"), `${JSON.stringify({ schema: "wp-codebox/agent-task-runtime-provenance/v1", sources: runtimeProvenance(request) }, null, 2)}\n`) +await writeFile(join(uploadPath, ".codebox", "agent-task-artifacts", "exclusions.json"), `${JSON.stringify({ schema: "wp-codebox/agent-task-upload-exclusions/v1", exclusions: await exclusions(artifactsPath, declaredPaths) }, null, 2)}\n`) +await finalScan(uploadPath) diff --git a/.github/workflows/run-agent-task.yml b/.github/workflows/run-agent-task.yml index de22853a3..093da3f51 100644 --- a/.github/workflows/run-agent-task.yml +++ b/.github/workflows/run-agent-task.yml @@ -299,9 +299,8 @@ jobs: with: name: codebox-agent-task-request-${{ github.run_id }} path: | - workspace/.codebox/agent-task-upload/.codebox/agent-task-request.json - workspace/.codebox/agent-task-upload/.codebox/agent-task-workflow-result.json - workspace/.codebox/agent-task-upload/.codebox/native-agent-task-input.json + workspace/.codebox/agent-task-upload + include-hidden-files: true - name: Upload Codebox task result if: always() @@ -309,6 +308,6 @@ jobs: with: name: codebox-agent-task-result-${{ github.run_id }} path: | - workspace/.codebox/agent-task-upload/.codebox/agent-task-workflow-result.json - workspace/.codebox/agent-task-upload/.codebox/agent-task-artifacts + workspace/.codebox/agent-task-upload if-no-files-found: ignore + include-hidden-files: true diff --git a/fixtures/agent-task-upload-run-29306539573.json b/fixtures/agent-task-upload-run-29306539573.json new file mode 100644 index 000000000..5af3957cf --- /dev/null +++ b/fixtures/agent-task-upload-run-29306539573.json @@ -0,0 +1,15 @@ +{ + "run_id": "29306539573", + "workflow": "Build With WordPress Skills Agent", + "raw_layout": { + "request": ".codebox/agent-task-request.json", + "result": "workspace/.codebox/agent-task-workflow-result.json", + "native_input": "workspace/.codebox/native-agent-task-input.json", + "runtime_source": "workspace/.codebox/agent-task-artifacts/prepared-plugins/agents-api/agents-api.php" + }, + "observed": { + "native_execution": "failed", + "upload_preparation": "failed-on-runtime-source", + "uploaded": [".codebox/agent-task-request.json"] + } +} diff --git a/tests/agent-task-reusable-workflow.test.ts b/tests/agent-task-reusable-workflow.test.ts index 96934d826..7bc744a36 100644 --- a/tests/agent-task-reusable-workflow.test.ts +++ b/tests/agent-task-reusable-workflow.test.ts @@ -38,7 +38,7 @@ assert.match(workflow, /Install WP Codebox runtime/) assert.match(workflow, /Checkout target workspace/) assert.match(workflow, /Execute native agent task/) assert.match(workflow, /execute-native-agent-task\.mjs/) -assert.match(workflow, /agent-task-artifacts/) +assert.match(workflow, /workspace\/\.codebox\/agent-task-upload/) assert.match(workflow, /prepare-agent-task-upload\.mjs/) assert.match(workflow, /agent-task-upload/) assert.match(workflow, /if: always\(\)/) @@ -291,6 +291,68 @@ await assert.rejects(execFileAsync("node", [new URL("../.github/scripts/run-agen }, }), /verification_commands\[0\]\.command/) +await writeFile(requestPath, "{\n") +await assert.rejects(execFileAsync("node", [executeNativeAgentTask], { + cwd: tmp, + env: { + ...process.env, + GITHUB_OUTPUT: outputPath, + AGENT_TASK_REQUEST_PATH: requestPath, + AGENT_TASK_WORKSPACE: tmp, + WP_CODEBOX_WORKFLOW_ROOT: new URL("..", import.meta.url).pathname, + EXTERNAL_PACKAGE_SOURCE_POLICY: '{"version":1,"repositories":{"automattic/example-agent-packages":["packages/example-agent.agent.json"]}}', + }, +})) +const malformedParseResult = JSON.parse(await readFile(resultPath, "utf8")) +assert.equal(malformedParseResult.failure.classification, "execution") +await execFileAsync("node", [new URL("../.github/scripts/run-agent-task/prepare-agent-task-upload.mjs", import.meta.url).pathname], { + cwd: tmp, + env: { ...process.env, AGENT_TASK_WORKSPACE: tmp, AGENT_TASK_REQUEST_PATH: requestPath }, +}) +assert.ok(await readFile(join(tmp, ".codebox", "agent-task-upload", ".codebox", "agent-task-workflow-result.json"), "utf8")) + +// Every lifecycle failure publishes the same safe review envelope, including +// failures before artifact materialization has created an artifact directory. +const assertEarlyFailureUpload = async (name: string, environment: Record, expectedClassification: string) => { + const failureRoot = await mkdtemp(join(tmpdir(), `wp-codebox-agent-task-${name}-`)) + const failureCodebox = join(failureRoot, ".codebox") + const failureRequestPath = join(failureCodebox, "agent-task-request.json") + await mkdir(failureCodebox, { recursive: true }) + await writeFile(failureRequestPath, `${JSON.stringify({ ...request, run_agent: name === "materialization", dry_run: false }, null, 2)}\n`) + await assert.rejects(execFileAsync(process.execPath, [executeNativeAgentTask], { + cwd: failureRoot, + env: { + ...process.env, + GITHUB_OUTPUT: join(failureRoot, "github-output.txt"), + AGENT_TASK_REQUEST_PATH: failureRequestPath, + AGENT_TASK_WORKSPACE: failureRoot, + WP_CODEBOX_WORKFLOW_ROOT: new URL("..", import.meta.url).pathname, + EXTERNAL_PACKAGE_SOURCE_POLICY: '{"version":1,"repositories":{"automattic/example-agent-packages":["packages/example-agent.agent.json"]}}', + GITHUB_TOKEN: "test-caller-token", + ...environment, + }, + })) + const failureResultPath = join(failureCodebox, "agent-task-workflow-result.json") + const failureResult = JSON.parse(await readFile(failureResultPath, "utf8")) + assert.equal(failureResult.status, "failed") + assert.equal(failureResult.success, false) + assert.equal(failureResult.failure.classification, expectedClassification) + assert.equal(await readFile(join(failureCodebox, "agent-task-artifacts", "exclusions.json"), "utf8").catch(() => "missing"), "missing", "Early failures must not create source or artifact trees") + await execFileAsync("node", [new URL("../.github/scripts/run-agent-task/prepare-agent-task-upload.mjs", import.meta.url).pathname], { + cwd: failureRoot, + env: { ...process.env, AGENT_TASK_WORKSPACE: failureRoot, AGENT_TASK_REQUEST_PATH: failureRequestPath, AGENT_TASK_UPLOAD_PATH: join(failureCodebox, "agent-task-upload") }, + }) + const uploadRoot = join(failureCodebox, "agent-task-upload", ".codebox") + assert.deepEqual(JSON.parse(await readFile(join(uploadRoot, "agent-task-workflow-result.json"), "utf8")).failure, failureResult.failure) + assert.ok(await readFile(join(uploadRoot, "agent-task-request.json"), "utf8")) + assert.ok(await readFile(join(uploadRoot, "agent-task-artifacts", "exclusions.json"), "utf8")) + assert.equal((await readdir(uploadRoot, { recursive: true })).some((path) => /prepared-|source-package|\.php$|\.m?js$/i.test(path)), false, "Failure uploads must exclude sources") +} + +await assertEarlyFailureUpload("source-policy", { EXTERNAL_PACKAGE_SOURCE_POLICY: "{}" }, "policy") +await assertEarlyFailureUpload("materialization", { PATH: "" }, "materialization") +await assertEarlyFailureUpload("approval", { GITHUB_TOKEN: "", EXPLICIT_ACCESS_TOKEN_CONFIGURED: "false" }, "policy") + // A serialized task request cannot stand in for a native run. Exercise the real // package-staging and canonical agents/chat harnesses instead of fabricating CLI // JSON or publication output in this workflow test. @@ -321,8 +383,9 @@ await execFileAsync("node", [new URL("../.github/scripts/run-agent-task/prepare- env: { ...process.env, AGENT_TASK_WORKSPACE: tmp, OPENAI_API_KEY: "secret-agent-value", GITHUB_TOKEN: "secret-github-value", EXTERNAL_PACKAGE_SOURCE_POLICY: '{"private":"policy"}' }, }) const uploadArtifactsPath = join(tmp, ".codebox", "agent-task-upload", ".codebox", "agent-task-artifacts") -assert.match(await readFile(join(uploadArtifactsPath, "safe.txt"), "utf8"), /\[REDACTED\]/) -assert.doesNotMatch(await readFile(join(uploadArtifactsPath, "safe.txt"), "utf8"), /secret-github-value/) +const exclusions = await readFile(join(uploadArtifactsPath, "exclusions.json"), "utf8") +assert.match(exclusions, /"category": "undeclared-artifact"/) +assert.doesNotMatch(exclusions, /secret-agent-value|secret-github-value/) assert.doesNotMatch(await readFile(join(tmp, ".codebox", "agent-task-upload", ".codebox", "agent-task-request.json"), "utf8"), /secret-agent-value|secret-github-value|\{"private":"policy"\}/) for (const name of ["oversize.txt", "binary.bin", "linked-secret.txt"]) { await assert.rejects(readFile(join(uploadArtifactsPath, name), "utf8"), /ENOENT/) diff --git a/tests/runtime-sources-materialization.test.ts b/tests/runtime-sources-materialization.test.ts index 804c023c9..f280f28eb 100644 --- a/tests/runtime-sources-materialization.test.ts +++ b/tests/runtime-sources-materialization.test.ts @@ -13,6 +13,10 @@ const execFileAsync = promisify(execFile) const hostedRegression = JSON.parse(await readFile(new URL("../fixtures/agent-task-runtime-sources-run-29299109269.json", import.meta.url), "utf8")) assert.equal(hostedRegression.run_id, "29299109269") assert.deepEqual(hostedRegression.runtime_sources.map((source: { role: string }) => source.role), ["component", "provider_plugin", "bundled_library"]) +const uploadLayoutRegression = JSON.parse(await readFile(new URL("../fixtures/agent-task-upload-run-29306539573.json", import.meta.url), "utf8")) +assert.equal(uploadLayoutRegression.run_id, "29306539573") +assert.equal(uploadLayoutRegression.observed.upload_preparation, "failed-on-runtime-source") +assert.match(uploadLayoutRegression.raw_layout.runtime_source, /prepared-plugins\/agents-api\/agents-api\.php$/) const hostedPathRegression = JSON.parse(await readFile(new URL("../fixtures/agent-task-runtime-paths-run-29305012941.json", import.meta.url), "utf8")) for (const result of [hostedPathRegression.success, hostedPathRegression.failure]) { const sanitized = sanitizeRuntimeSourceValue(result, hostedPathRegression.runtime_root) @@ -122,7 +126,12 @@ await withTempDir("wp-codebox-runtime-source-upload-", async (directory) => { const suffixedPrivateRoot = `${privateRoot}-actual-mkdtemp-suffix` await mkdir(artifacts, { recursive: true }) await mkdir(privateRoot, { recursive: true }) + await mkdir(join(workspace, ".codebox"), { recursive: true }) await writeFile(join(privateRoot, "source.php"), " { metadata: { originalSource: privateRoot, nested: { preparedPath: privateRoot, requestedPath: privateRoot }, runtime_source: { role: "component", repository: "example/runtime", revision: "a".repeat(40), path: "plugin" } }, }], })) - await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), JSON.stringify({ callback_data: { task_path: privateRoot, nested: { result_path: privateRoot } } })) + await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), JSON.stringify({ status: "failed", success: false, callback_data: { task_path: privateRoot, nested: { result_path: privateRoot } }, typed_artifacts: [{ name: "reviewer-report", type: "report", artifact: { path: "safe.json" } }] })) const script = new URL("../.github/scripts/run-agent-task/prepare-agent-task-upload.mjs", import.meta.url) await execFileAsync(process.execPath, [script.pathname], { env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: privateRoot } }) const staged = await readFile(join(upload, ".codebox", "agent-task-artifacts", "safe.json"), "utf8") assert.doesNotMatch(staged, /private-runtime-source|private runtime source/) const stagedInput = await readFile(join(upload, ".codebox", "native-agent-task-input.json"), "utf8") assert.doesNotMatch(stagedInput, /private-runtime-source/) - assert.match(stagedInput, /"runtime_source"/) + assert.doesNotMatch(stagedInput, /component_contracts|source_package_root/) const stagedResult = await readFile(join(upload, ".codebox", "agent-task-workflow-result.json"), "utf8") assert.doesNotMatch(stagedResult, /private-runtime-source/) await writeFile(join(artifacts, "leak.json"), `runtime log ${privateRoot}/source.php`) + await writeFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), JSON.stringify({ status: "failed", success: false, typed_artifacts: [{ name: "reviewer-report", type: "report", artifact: { path: "leak.json" } }] })) await execFileAsync(process.execPath, [script.pathname], { env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: privateRoot } }) assert.doesNotMatch(await readFile(join(upload, ".codebox", "agent-task-artifacts", "leak.json"), "utf8"), /private-runtime-source/) assert.match(await readFile(join(upload, ".codebox", "agent-task-artifacts", "leak.json"), "utf8"), /\[runtime-source\]/) await rm(join(artifacts, "leak.json")) await mkdir(join(artifacts, "prepared-plugins", "agents-api"), { recursive: true }) await writeFile(join(artifacts, "prepared-plugins", "agents-api", "agents-api.php"), " { assert.match(output, /runtime_source_root<<__WP_CODEBOX_OUTPUT__\n\[runtime-source\]\n__WP_CODEBOX_OUTPUT__/, "executor must sanitize the private root in step output") await writeFile(join(codebox, "agent-task-artifacts", "exact-root-leak.json"), exactPrivateRuntimeRoot) await execFileAsync(process.execPath, [uploaderPath.pathname], { env: { ...environment, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: exactPrivateRuntimeRoot } }) - assert.doesNotMatch(await readFile(join(upload, ".codebox", "agent-task-artifacts", "exact-root-leak.json"), "utf8"), new RegExp(exactPrivateRuntimeRoot.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))) + assert.doesNotMatch(await readFile(join(upload, ".codebox", "agent-task-artifacts", "exclusions.json"), "utf8"), new RegExp(exactPrivateRuntimeRoot.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))) await rm(join(codebox, "agent-task-artifacts", "exact-root-leak.json")) await execFileAsync(process.execPath, [uploaderPath.pathname], { env: { ...environment, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: exactPrivateRuntimeRoot } }) const privateRuntimePrefix = exactPrivateRuntimeRoot - for (const path of [".codebox/agent-task-request.json", ".codebox/agent-task-workflow-result.json", ".codebox/agent-task-artifacts/safe.json"]) { + for (const path of [".codebox/agent-task-request.json", ".codebox/agent-task-workflow-result.json", ".codebox/agent-task-artifacts/exclusions.json"]) { assert.ok(!(await readFile(join(upload, path), "utf8")).includes(privateRuntimePrefix)) } - const downloadedArtifact = JSON.stringify(await Promise.all([".codebox/agent-task-request.json", ".codebox/agent-task-workflow-result.json", ".codebox/agent-task-artifacts/safe.json"].map((path) => readFile(join(upload, path), "utf8")))) + const downloadedArtifact = JSON.stringify(await Promise.all([".codebox/agent-task-request.json", ".codebox/agent-task-workflow-result.json", ".codebox/agent-task-artifacts/exclusions.json"].map((path) => readFile(join(upload, path), "utf8")))) assert.doesNotMatch(downloadedArtifact, /prepared-plugins|agents-api|ai-provider-for-openai|plugin\.php|private-runtime-source/) assert.match(downloadedArtifact, /\\"repository\\": \\"example\/source\\"/) assert.doesNotMatch(downloadedArtifact, /provider id is required/i)