diff --git a/.github/scripts/run-agent-task/execute-native-agent-task.mjs b/.github/scripts/run-agent-task/execute-native-agent-task.mjs index 548f3ffeb..bf31c5c7f 100644 --- a/.github/scripts/run-agent-task/execute-native-agent-task.mjs +++ b/.github/scripts/run-agent-task/execute-native-agent-task.mjs @@ -1,12 +1,13 @@ import { rmSync } from "node:fs" import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises" -import { join, relative, resolve } from "node:path" +import { isAbsolute, join, relative, resolve } from "node:path" import { pathToFileURL } from "node:url" import { spawn } from "node:child_process" -import { materializeExternalNativePackage, materializeRuntimeSources, normalizeExternalPackageSource, normalizeRuntimeSources, parseExternalPackageSourcePolicy, validateRuntimeSourceModel } from "./materialize-external-native-package.mjs" +import { canonicalExternalNativeAgentIdentity, materializeExternalNativePackage, materializeRuntimeSources, normalizeExternalPackageSource, normalizeRuntimeSources, parseExternalPackageSourcePolicy, sha256BytesV1, validateRuntimeSourceModel } from "./materialize-external-native-package.mjs" import { readNativeResult } from "./native-result-file.mjs" import { assertNoRuntimeSourcePaths, sanitizeRuntimeSourceJson, sanitizeRuntimeSourceText, sanitizeRuntimeSourceValue } from "./runtime-source-sanitizer.mjs" import { publishRunnerWorkspace } from "./runner-workspace-publisher.mjs" +import { createRunnerWorkspaceSeedSnapshot, RUNNER_WORKSPACE_SEED_EXCLUDES } from "./runner-workspace-seed-snapshot.mjs" const requestPath = process.env.AGENT_TASK_REQUEST_PATH || ".codebox/agent-task-request.json" const workspace = resolve(process.env.AGENT_TASK_WORKSPACE || process.cwd()) @@ -18,6 +19,42 @@ const MAX_OUTPUT_CHARS = 8192 const secretValues = ["OPENAI_API_KEY", "MODEL_PROVIDER_SECRET_1", "MODEL_PROVIDER_SECRET_2", "MODEL_PROVIDER_SECRET_3", "MODEL_PROVIDER_SECRET_4", "MODEL_PROVIDER_SECRET_5", "GITHUB_TOKEN", "GH_TOKEN", "ACCESS_TOKEN", "EXTERNAL_PACKAGE_SOURCE_POLICY"].map((name) => process.env[name]).filter(Boolean) let privateRuntimeSourceRoot = "" let privateRuntimeSourceRootForSanitization = "" +let runnerWorkspaceSeedSnapshot + +const SIGNAL_EXIT_CODES = { SIGINT: 130, SIGTERM: 143, SIGHUP: 129 } +let materializedSourceCleanup +function claimMaterializedSourcePaths() { + const paths = [] + if (runnerWorkspaceSeedSnapshot) { + paths.push(runnerWorkspaceSeedSnapshot.source) + runnerWorkspaceSeedSnapshot = undefined + } + if (privateRuntimeSourceRoot) { + paths.push(privateRuntimeSourceRoot) + privateRuntimeSourceRoot = "" + } + return paths +} +// Single idempotent cleanup coordinator for the private runtime source +// materialization root and the runner workspace seed snapshot. Every +// completion path (normal, failure, signal) awaits this coordinator; repeat +// invocations chain onto the in-flight cleanup instead of racing it. +function cleanupMaterializedSources() { + materializedSourceCleanup = (materializedSourceCleanup ?? Promise.resolve()) + .then(() => Promise.all(claimMaterializedSourcePaths().map((path) => rm(path, { recursive: true, force: true })))) + return materializedSourceCleanup +} +process.once("exit", () => { + // Bounded synchronous best-effort fallback for abrupt exits; on every + // awaited path the coordinator has already claimed these roots. + for (const path of claimMaterializedSourcePaths()) { + try { rmSync(path, { recursive: true, force: true, maxRetries: 0 }) } catch { /* best effort */ } + } +}) +for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) { + process.once(signal, () => { cleanupMaterializedSources().finally(() => process.exit(SIGNAL_EXIT_CODES[signal])) }) +} + function redact(value) { if (typeof value === "string") return secretValues.reduce((output, secret) => output.split(secret).join("[REDACTED]"), value) if (Array.isArray(value)) return value.map(redact) @@ -197,6 +234,55 @@ function requiredArtifacts(declarations) { }))) } +async function testRuntimeFixtures(externalPackageSource) { + if (process.env.NODE_ENV !== "test") return {} + const packagePath = string(process.env.WP_CODEBOX_TEST_EXTERNAL_PACKAGE_PATH) + const runtimeSourceInputs = string(process.env.WP_CODEBOX_TEST_RUNTIME_SOURCE_INPUTS) + if (!packagePath && !runtimeSourceInputs) return {} + const fixtures = {} + if (packagePath) { + const bytes = await readFile(packagePath) + if (sha256BytesV1(bytes) !== externalPackageSource.digest) throw new Error("Test external package fixture digest does not match the request.") + fixtures.materializedPackage = { bytes, descriptor: externalPackageSource, identity: canonicalExternalNativeAgentIdentity(bytes) } + } + if (runtimeSourceInputs) { + try { + fixtures.runtimeSourceInputs = JSON.parse(runtimeSourceInputs) + } catch { + throw new Error("WP_CODEBOX_TEST_RUNTIME_SOURCE_INPUTS must be valid JSON.") + } + } + return fixtures +} + +// Test-only interruption hook: inert in production (requires NODE_ENV=test and +// an explicit marker path). Publishes the seed snapshot location, then holds a +// bounded window so a harness can deliver a termination signal. +async function testPauseAfterSeedSnapshot(seedSnapshot) { + if (process.env.NODE_ENV !== "test") return + const markerPath = string(process.env.WP_CODEBOX_TEST_SEED_SNAPSHOT_PAUSE_FILE) + if (!markerPath) return + await writeFile(markerPath, `${JSON.stringify({ schema: "wp-codebox/test-seed-snapshot-pause/v1", seed_snapshot_source: seedSnapshot?.source ?? "" })}\n`) + await new Promise((resolvePause) => setTimeout(resolvePause, 120_000)) +} + +function runtimeSourceFixtureRoot(value) { + const paths = [] + const collect = (entry) => { + if (typeof entry === "string") { + if (isAbsolute(entry)) paths.push(resolve(entry)) + return + } + if (Array.isArray(entry)) return entry.forEach(collect) + if (entry && typeof entry === "object") Object.values(entry).forEach(collect) + } + collect(value) + if (paths.length === 0) return "" + + const shared = paths.map((path) => path.split("/").filter(Boolean)).reduce((prefix, parts) => prefix.filter((part, index) => parts[index] === part)) + return shared.length > 0 ? `/${shared.join("/")}` : "" +} + function workflowPath(path) { const relativePath = relative(workspace, resolve(path)) return relativePath && !relativePath.startsWith("..") ? relativePath.replaceAll("\\", "/") : ".codebox/agent-task-workflow-result.json" @@ -261,23 +347,12 @@ const externalPackagePolicy = parseExternalPackageSourcePolicy(string(process.en const externalPackageSource = normalizeExternalPackageSource(request.external_package_source, externalPackagePolicy) const runtimeSources = normalizeRuntimeSources(request.runtime_sources ?? [], externalPackagePolicy) const requestedModel = validateRuntimeSourceModel(request.model, runtimeSources) +const writablePaths = String(request.writable_paths || "").split(",").map((value) => value.trim()).filter(Boolean) const artifactsPath = join(workspace, ".codebox", "agent-task-artifacts") const runtimeInputPath = join(workspace, ".codebox", "native-agent-task-input.json") const resultPath = join(workspace, ".codebox", "agent-task-workflow-result.json") const controlledCodeboxPath = resolve(requestPath, "..") -const nativeResultPath = join(controlledCodeboxPath, "native-agent-task-result.json") -let cleaningPrivateRuntimeSources = false -async function cleanupPrivateRuntimeSources() { - if (cleaningPrivateRuntimeSources || !privateRuntimeSourceRoot) return - cleaningPrivateRuntimeSources = true - const root = privateRuntimeSourceRoot - privateRuntimeSourceRoot = "" - await rm(root, { recursive: true, force: true }) -} -process.once("exit", () => { if (privateRuntimeSourceRoot) rmSync(privateRuntimeSourceRoot, { recursive: true, force: true }) }) -for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) { - process.once(signal, () => { cleanupPrivateRuntimeSources().finally(() => process.exit(128)) }) -} + const nativeResultPath = join(controlledCodeboxPath, "native-agent-task-result.json") const runnerWorkspaceTools = [ "workspace_read", "workspace_ls", "workspace_grep", "workspace_write", "workspace_edit", "workspace_apply_patch", "workspace_show", "workspace_git_status", "workspace_git_diff", @@ -291,17 +366,21 @@ function runtimeMetadataForExecutionLocation(executionLocation) { await mkdir(artifactsPath, { recursive: true }) -const accessError = accessFailure(request) + const accessError = accessFailure(request) if (accessError) { const error = new Error(accessError) error.code = "wp-codebox.agent-task.policy" - throw error -} + throw error + } + + if (request.runner_workspace?.enabled) runnerWorkspaceSeedSnapshot = await createRunnerWorkspaceSeedSnapshot(workspace) + await testPauseAfterSeedSnapshot(runnerWorkspaceSeedSnapshot) -const skipMaterialization = process.env.NODE_ENV === "test" && process.env.WP_CODEBOX_TEST_SKIP_MATERIALIZATION === "true" +const testFixtures = await testRuntimeFixtures(externalPackageSource) +const skipMaterialization = process.env.NODE_ENV === "test" && (process.env.WP_CODEBOX_TEST_SKIP_MATERIALIZATION === "true" || Boolean(testFixtures.materializedPackage || testFixtures.runtimeSourceInputs)) const materializedPackage = request.run_agent && !request.dry_run && !skipMaterialization ? await materializeExternalNativePackage(externalPackageSource, { policy: externalPackagePolicy }) - : undefined + : testFixtures.materializedPackage const materializedRuntimeSources = request.run_agent && !request.dry_run && !skipMaterialization ? await materializeRuntimeSources(runtimeSources, { policy: externalPackagePolicy, forbiddenRoots: [workspace, artifactsPath] }) : undefined @@ -311,7 +390,8 @@ await output("runtime_source_root", privateRuntimeSourceRoot) const runtimeSourceInputs = (materializedRuntimeSources?.lowered ?? []).reduce((input, lowered) => { for (const [key, entries] of Object.entries(lowered)) input[key] = [...(input[key] ?? []), ...entries] return input -}, {}) +}, testFixtures.runtimeSourceInputs ?? {}) +const runtimeSourceOutputRoots = [privateRuntimeSourceRoot, runtimeSourceFixtureRoot(testFixtures.runtimeSourceInputs)] // Runtime source preparation must remain beside the private checkout, never in // the target artifact directory that is collected after the run. const privatePreparationRoot = privateRuntimeSourceRoot ? join(privateRuntimeSourceRoot, "prepared-runtime-sources") : "" @@ -358,27 +438,27 @@ const taskInput = { model: requestedModel.name, runner_workspace: { ...record(request.runner_workspace), allowed_repos: request.access.allowed_repos }, target_repo: request.target_repo, - writable_paths: request.writable_paths, - runner_workspace_policy: { allowed_repos: request.access.allowed_repos, writable_paths: request.writable_paths }, + writable_paths: writablePaths, + runner_workspace_policy: { allowed_repos: request.access.allowed_repos, writable_paths: writablePaths }, }, artifact_declarations: request.artifacts?.declarations || [], required_artifacts: requiredArtifacts(request.artifacts?.declarations), output_projections: [], - metadata: { workload: request.workload, ...(materializedPackage ? { imported_agent: materializedPackage.identity } : {}), runtime_sources: materializedRuntimeSources?.descriptors ?? [] }, + metadata: { workload: request.workload, ...(materializedPackage ? { imported_agent: materializedPackage.identity } : {}), runtime_sources: materializedRuntimeSources?.descriptors ?? [], ...(runnerWorkspaceSeedSnapshot ? { runner_workspace_seed: runnerWorkspaceSeedSnapshot.provenance } : {}) }, }, }, - // agent-task-run unwraps task_input before building the recipe. Keep the - // runner seed on that canonical input rather than on the request envelope. - workspaces: request.runner_workspace?.enabled ? [{ target: "/workspace", mode: "readwrite", sourceMode: "repo-backed", seed: { type: "directory", source: workspace, excludePaths: [".git/**", ".codebox/**", "node_modules/**", "vendor/**", "dist/**", "build/**", "coverage/**", ".cache/**"] } }] : [], + // agent-task-run unwraps task_input before building the recipe. The + // external snapshot prevents the recipe from ever mounting the checkout. + workspaces: runnerWorkspaceSeedSnapshot ? [{ target: "/workspace", mode: "readwrite", sourceMode: "repo-backed", seed: { type: "directory", source: runnerWorkspaceSeedSnapshot.source, excludePaths: RUNNER_WORKSPACE_SEED_EXCLUDES } }] : [], }, } await writeFile(executionInputPath, `${JSON.stringify(taskInput, null, 2)}\n`) let execution = { code: 0, stdout: "", stderr: "", stdout_truncated: false, stderr_truncated: false } -if (request.run_agent && !request.dry_run) { - execution = await command("node", [codeboxCliPath, "agent-task-run", "--input-file", executionInputPath, "--result-file", nativeResultPath], workspace, agentEnvironment()) -} + if (request.run_agent && !request.dry_run) { + execution = await command("node", [codeboxCliPath, "agent-task-run", "--input-file", executionInputPath, "--result-file", nativeResultPath], workspace, agentEnvironment()) + } // Public package bytes are embedded in the runtime recipe and consumed only by // the Playground bootstrap before the agent's tools are resolved. @@ -387,7 +467,7 @@ const nativeRuntimeResult = request.run_agent && !request.dry_run ? await readNativeResult(nativeResultPath, controlledCodeboxPath, secretValues, redact) : {} await rm(nativeResultPath, { force: true }) -const runtimeResult = sanitizeRuntimeSourceValue(nativeRuntimeResult, privateRuntimeSourceRootForSanitization) +let runtimeResult = sanitizeRuntimeSourceValue(nativeRuntimeResult, privateRuntimeSourceRootForSanitization) assertNoRuntimeSourcePaths(runtimeResult, privateRuntimeSourceRootForSanitization) let workspaceApply = { status: "no-op", changedFiles: [] } let runnerWorkspaceCore = null @@ -398,12 +478,13 @@ if (execution.code === 0 && runtimeResult.success === true && request.runner_wor const publicCore = await import(pathToFileURL(join(codeboxRoot, "packages/runtime-core/dist/public.js")).href) const refs = publicCore.normalizePublicArtifactRefDTOs(runtimeResult) .filter((ref) => ref.kind === "codebox-patch" || ref.kind === "codebox-changed-files") - workspaceApply = await runnerWorkspaceCore.applyRunnerWorkspacePatch({ artifactRoot: artifactsPath, artifactRefs: refs, workspaceRoot: workspace, writablePaths: String(request.writable_paths || "").split(",").map((value) => value.trim()).filter(Boolean) }) + workspaceApply = await runnerWorkspaceCore.applyRunnerWorkspacePatch({ artifactRoot: artifactsPath, artifactRefs: refs, workspaceRoot: workspace, writablePaths }) } catch (error) { downstreamFailure = { stage: "apply", message: bounded(error instanceof Error ? error.message : String(error), MAX_OUTPUT_CHARS) } } } -await cleanupPrivateRuntimeSources() +runtimeResult = sanitizeRuntimeSourceValue(runtimeResult, runtimeSourceOutputRoots) +privateRuntimeSourceRootForSanitization = runtimeSourceOutputRoots assertNoRuntimeSourcePaths(runtimeResult, privateRuntimeSourceRootForSanitization) await redactArtifactFiles(artifactsPath) @@ -511,15 +592,9 @@ if (!success) process.exitCode = 1 try { await executeNativeAgentTask() } catch (error) { - try { - await writeNormalizedFailure(error) - } finally { - if (privateRuntimeSourceRoot) { - const root = privateRuntimeSourceRoot - privateRuntimeSourceRoot = "" - await rm(root, { recursive: true, force: true }) - } - } + await writeNormalizedFailure(error) console.error(bounded(error instanceof Error ? error.message : String(error), MAX_OUTPUT_CHARS)) process.exitCode = 1 +} finally { + await cleanupMaterializedSources() } diff --git a/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs b/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs index cebcb7103..e5c1d9619 100644 --- a/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs +++ b/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs @@ -13,6 +13,7 @@ const secretValues = ["OPENAI_API_KEY", "MODEL_PROVIDER_SECRET_1", "MODEL_PROVID const runtimeSourceRoot = process.env.WP_CODEBOX_RUNTIME_SOURCE_ROOT ? resolve(process.env.WP_CODEBOX_RUNTIME_SOURCE_ROOT) : "" const runtimeSourcePrefix = process.env.WP_CODEBOX_RUNTIME_SOURCE_PREFIX ? resolve(process.env.WP_CODEBOX_RUNTIME_SOURCE_PREFIX) : "" const runtimeSourceRoots = [runtimeSourceRoot, runtimeSourcePrefix].filter(Boolean) +const privateUploadRoots = [...runtimeSourceRoots, workspace] const SOURCE_TREE = /(^|\/)(prepared-plugins|prepared-source-packages|source-package[^/]*)(\/|$)/i const SOURCE_FILE = /\.(?:php|phtml|js|mjs|cjs|jsx|ts|tsx)$/i const PHP_OPENING_TAG = /<\?(?:php|=)(?:\s|$)/i @@ -32,16 +33,32 @@ function redact(value) { } function sanitizeText(text) { - return sanitizeRuntimeSourceJson(text, runtimeSourceRoots) + return sanitizeRuntimeSourceJson(text, privateUploadRoots) } function compactNativeInput(text) { const privateFields = new Set(["source_package_root", "component_contracts", "extra_plugins", "provider_plugins", "runtime_overlays", "prepared_sources"]) - const compact = (value) => { + let seedProvenance = {} + try { + const parsed = JSON.parse(sanitizeText(text)) + seedProvenance = record(record(record(parsed).task_input).runtime_task?.input?.metadata).runner_workspace_seed + } catch {} + const compact = (value, key = "") => { if (Array.isArray(value)) return value.map(compact) const entry = record(value) if (!Object.keys(entry).length) return value - return Object.fromEntries(Object.entries(entry).flatMap(([key, item]) => privateFields.has(key) ? [] : [[key, compact(item)]])) + if (key === "seed" && typeof entry.source === "string") { + const provenance = record(seedProvenance) + return { + kind: "runner-workspace-seed", + digest: record(provenance.digest).sha256, + files: provenance.files, + bytes: provenance.bytes, + excludes: provenance.excludes, + excluded: provenance.excluded, + } + } + return Object.fromEntries(Object.entries(entry).flatMap(([childKey, item]) => privateFields.has(childKey) ? [] : [[childKey, compact(item, childKey)]])) } try { return `${JSON.stringify(compact(JSON.parse(sanitizeText(text))), null, 2)}\n` @@ -50,6 +67,38 @@ function compactNativeInput(text) { } } +function assertNoSeedSnapshotPaths(text) { + if (/wp-codebox-runner-workspace-seed-/i.test(text)) throw new Error("Temporary runner workspace seed paths must never be persisted in artifact uploads.") + try { + const visit = (value) => { + if (Array.isArray(value)) return value.forEach(visit) + const entry = record(value) + if (!Object.keys(entry).length) return + if (record(entry.seed).source && isAbsolute(record(entry.seed).source)) throw new Error("Absolute runner workspace seed paths must never be persisted in artifact uploads.") + Object.values(entry).forEach(visit) + } + visit(JSON.parse(text)) + } catch (error) { + if (error instanceof Error && /seed paths/.test(error.message)) throw error + } +} + +function sanitizeSeedSnapshotJson(text) { + try { + const compact = (value, key = "") => { + if (typeof value === "string") return value.replace(/\/?[^\s"']*wp-codebox-runner-workspace-seed-[^\s"']*/gi, "[runner-workspace-seed]") + if (Array.isArray(value)) return value.map((entry) => compact(entry)) + const entry = record(value) + if (!Object.keys(entry).length) return value + if (key === "seed" && typeof entry.source === "string") return { kind: "runner-workspace-seed" } + return Object.fromEntries(Object.entries(entry).map(([childKey, item]) => [childKey, compact(item, childKey)])) + } + return `${JSON.stringify(compact(JSON.parse(text)), null, 2)}\n` + } catch { + return text + } +} + function isPrivateRuntimePath(value) { if (!runtimeSourceRoots.length || typeof value !== "string") return false const path = resolve(value) @@ -82,8 +131,10 @@ async function stageTextFile(source, destination, options = {}) { const contents = openedMetadata.isFile() && openedMetadata.size <= MAX_UPLOAD_FILE_BYTES ? await handle.readFile() : null await handle.close() if (!contents || contents.includes(0) || !isUtf8(contents)) return false - const text = redact(options.compactNativeInput ? compactNativeInput(contents.toString("utf8")) : sanitizeText(contents.toString("utf8"))) - assertNoRuntimeSourcePaths(text, runtimeSourceRoots, "Runtime source paths must never be persisted in artifact uploads.") + const sanitized = options.compactNativeInput ? compactNativeInput(contents.toString("utf8")) : sanitizeText(contents.toString("utf8")) + const text = redact(sanitizeSeedSnapshotJson(sanitized)) + assertNoRuntimeSourcePaths(text, privateUploadRoots, "Runtime source or workspace paths must never be persisted in artifact uploads.") + assertNoSeedSnapshotPaths(text) if (containsRuntimeSourceContent(text)) throw new Error("Prepared runtime plugin source contents must never be staged for artifact upload.") await mkdir(resolve(destination, ".."), { recursive: true }) await writeFile(destination, text) @@ -163,7 +214,8 @@ async function finalScan(directory) { else if (entry.isFile()) { const bytes = await readFile(path) const text = isUtf8(bytes) ? bytes.toString("utf8") : "" - assertNoRuntimeSourcePaths(text, runtimeSourceRoots, "Runtime source paths must never be persisted in artifact uploads.") + assertNoRuntimeSourcePaths(text, privateUploadRoots, "Runtime source or workspace paths must never be persisted in artifact uploads.") + assertNoSeedSnapshotPaths(text) if (containsRuntimeSourceContent(text)) throw new Error("Prepared runtime plugin source contents must never be persisted in artifact uploads.") } else throw new Error("Only regular files may be persisted in artifact uploads.") } diff --git a/.github/scripts/run-agent-task/runner-workspace-seed-snapshot.mjs b/.github/scripts/run-agent-task/runner-workspace-seed-snapshot.mjs new file mode 100644 index 000000000..00ab454d0 --- /dev/null +++ b/.github/scripts/run-agent-task/runner-workspace-seed-snapshot.mjs @@ -0,0 +1,101 @@ +import { createHash } from "node:crypto" +import { chmod, copyFile, lstat, mkdir, mkdtemp, readdir, readFile, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join, relative, resolve } from "node:path" + +// Snapshot policy is intentionally allow-by-exception: disposable build trees and +// credential-bearing files never enter the agent-readable workspace. `.env.example` +// is the one documented environment-template exception. +export const RUNNER_WORKSPACE_SEED_EXCLUDES = [".git/**", ".codebox/**", "node_modules/**", "vendor/**", "dist/**", "build/**", "coverage/**", ".cache/**", ".env", ".env.*", ".npmrc", ".yarnrc.yml", ".pypirc", ".netrc", "auth.json", "id_rsa", "id_ed25519", "*.pem", "*.key", "credential files"] +const EXCLUDED_ROOT_NAMES = new Set(RUNNER_WORKSPACE_SEED_EXCLUDES.filter((pattern) => pattern.endsWith("/**")).map((pattern) => pattern.slice(0, -3))) +const MAX_FILES = 10_000 +const MAX_BYTES = 256 * 1024 * 1024 + +function snapshotError(message) { + const error = new Error(message) + error.code = "wp-codebox.agent-task.runner-workspace-snapshot" + return error +} + +function fileMode(stat) { + return stat.mode & 0o111 ? 0o755 : 0o644 +} + +function secretCategory(path) { + const name = path.split("/").at(-1)?.toLowerCase() || "" + if ((name === ".env" || name.startsWith(".env.")) && name !== ".env.example") return "environment" + if ([".npmrc", ".yarnrc.yml", ".pypirc", ".netrc", "auth.json", "credentials", "credentials.json", "secrets.json", "token.json"].includes(name)) return "credentials" + if (["id_rsa", "id_ed25519", "id_ecdsa", "id_dsa"].includes(name) || /\.(?:pem|key|p12|pfx)$/i.test(name)) return "private-key" + return "" +} + +export async function createRunnerWorkspaceSeedSnapshot(source) { + const sourceRoot = resolve(source) + const root = await mkdtemp(join(tmpdir(), "wp-codebox-runner-workspace-seed-")) + const digest = createHash("sha256") + let fileCount = 0 + let byteCount = 0 + const excluded = new Map() + const exclude = (category) => excluded.set(category, (excluded.get(category) || 0) + 1) + + try { + async function copyTree(currentSource, currentTarget) { + const entries = await readdir(currentSource, { withFileTypes: true }) + for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { + if (EXCLUDED_ROOT_NAMES.has(entry.name)) { + exclude("generated-tree") + continue + } + const input = join(currentSource, entry.name) + const output = join(currentTarget, entry.name) + const stat = await lstat(input) + const path = relative(sourceRoot, input).replaceAll("\\", "/") + const category = secretCategory(path) + if (category) { + exclude(category) + continue + } + if (stat.isSymbolicLink() || (!stat.isDirectory() && !stat.isFile())) { + throw snapshotError(`Runner workspace seed contains unsupported filesystem entry: ${path}`) + } + if (stat.isDirectory()) { + await mkdir(output, { recursive: true, mode: 0o755 }) + await chmod(output, 0o755) + digest.update(`directory\0${path}\n`) + await copyTree(input, output) + continue + } + fileCount += 1 + byteCount += stat.size + if (fileCount > MAX_FILES) throw snapshotError(`Runner workspace seed exceeds the ${MAX_FILES} file limit.`) + if (byteCount > MAX_BYTES) throw snapshotError(`Runner workspace seed exceeds the ${MAX_BYTES} byte limit.`) + const bytes = await readFile(input) + digest.update(`file\0${path}\0${fileMode(stat).toString(8)}\0${bytes.length}\n`) + digest.update(bytes) + await copyFile(input, output) + await chmod(output, fileMode(stat)) + } + } + + const sourceStat = await lstat(sourceRoot) + if (!sourceStat.isDirectory() || sourceStat.isSymbolicLink()) throw snapshotError("Runner workspace seed source must be a real directory.") + await copyTree(sourceRoot, root) + return { + source: root, + provenance: { + schema: "wp-codebox/runner-workspace-seed-snapshot/v1", + digest: { sha256: digest.digest("hex") }, + files: fileCount, + bytes: byteCount, + excludes: RUNNER_WORKSPACE_SEED_EXCLUDES, + excluded: { + files: [...excluded.values()].reduce((total, count) => total + count, 0), + categories: [...excluded.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([category, count]) => ({ category, count })), + }, + }, + } + } catch (error) { + await rm(root, { recursive: true, force: true }) + throw error + } +} diff --git a/.github/workflows/agent-task-contracts.yml b/.github/workflows/agent-task-contracts.yml index 6407cec54..cb3d4303b 100644 --- a/.github/workflows/agent-task-contracts.yml +++ b/.github/workflows/agent-task-contracts.yml @@ -17,6 +17,8 @@ on: - "tests/agent-task-*.test.ts" - "tests/runtime-sources-materialization.test.ts" - "tests/runtime-sources-playground-integration.test.ts" + - "tests/execute-native-agent-task-playground-e2e.test.ts" + - "tests/execute-native-agent-task-interruption.test.mjs" - "tests/redaction.test.ts" - "tests/production-boundary-enforcement.test.ts" - "tests/runtime-tool-policy.test.ts" @@ -36,6 +38,8 @@ on: - "tests/agent-task-*.test.ts" - "tests/runtime-sources-materialization.test.ts" - "tests/runtime-sources-playground-integration.test.ts" + - "tests/execute-native-agent-task-playground-e2e.test.ts" + - "tests/execute-native-agent-task-interruption.test.mjs" - "tests/redaction.test.ts" - "tests/production-boundary-enforcement.test.ts" - "tests/runtime-tool-policy.test.ts" @@ -55,6 +59,8 @@ jobs: - run: npm run build - run: npm run test:agent-task-contracts - run: npm run test:runtime-sources-playground-integration + - run: npm run test:native-agent-task-playground-e2e + - run: npm run test:native-agent-task-interruption - run: npm run test:redaction - run: npm run test:production-boundary-enforcement - run: npm run test:runtime-tool-policy diff --git a/docs/runner-workspace-seed-policy.md b/docs/runner-workspace-seed-policy.md new file mode 100644 index 000000000..7759edb25 --- /dev/null +++ b/docs/runner-workspace-seed-policy.md @@ -0,0 +1,5 @@ +# Runner Workspace Seed Policy + +Runner workspaces are copied from a bounded, disposable snapshot. The snapshot excludes generated trees and credential material before an agent can read it: `.env` and `.env.*` except `.env.example`, `.npmrc`, `.yarnrc.yml` (it can carry registry auth), `.pypirc`, `.netrc`, `auth.json`, common credential files, SSH private-key names, and private-key extensions. + +Snapshot provenance records only a digest, copied file and byte counts, the policy patterns, and aggregate exclusion categories/counts. It never records excluded path names or the temporary snapshot location. diff --git a/package.json b/package.json index 129227ed9..bfae37b1c 100644 --- a/package.json +++ b/package.json @@ -85,7 +85,7 @@ "generate:fanout-aggregation-contract": "tsx scripts/generate-fanout-aggregation-contract-fixture.ts", "smoke": "tsx scripts/run-smoke.ts", "test:redaction": "tsx tests/redaction.test.ts", - "test:agent-task-contracts": "tsx tests/agent-task-contracts.test.ts && npm run test:agent-task-workflow-interface && npm run test:runtime-sources-materialization && tsx tests/agent-task-reusable-workflow.test.ts", + "test:agent-task-contracts": "tsx tests/agent-task-contracts.test.ts && tsx tests/agent-task-canonical-evidence.test.ts && npm run test:agent-task-workflow-interface && npm run test:runtime-sources-materialization && tsx tests/agent-task-reusable-workflow.test.ts", "test:agent-task-workflow-interface": "tsx tests/run-agent-task-reusable-workflow-interface.test.ts", "test:agent-task-runtime-package-staging": "tsx tests/agent-task-runtime-package-staging.test.ts", "test:external-native-package-materialization": "tsx tests/external-native-package-materialization.test.ts", @@ -130,6 +130,8 @@ "test:runner-workspace-apply": "tsx tests/runner-workspace-apply.test.ts", "test:runner-workspace-publisher": "node tests/runner-workspace-publisher.test.mjs", "test:native-agent-task-lifecycle": "node tests/execute-native-agent-task-lifecycle.test.mjs", + "test:native-agent-task-interruption": "node tests/execute-native-agent-task-interruption.test.mjs", + "test:native-agent-task-playground-e2e": "tsx tests/execute-native-agent-task-playground-e2e.test.ts", "test:bench-command-step-behavior": "tsx tests/bench-command-step-behavior.test.ts", "test:generic-primitives": "npm run test:artifact-path-primitives && npm run test:browser-callback-materialization-contracts && npm run test:source-package-compiler-primitives && npm run test:bench-command-step-behavior && npm run test:generic-ability-runtime-run", "test:browser-artifact-session": "tsx tests/browser-artifact-session.test.ts", diff --git a/packages/cli/src/commands/agent-task-run.ts b/packages/cli/src/commands/agent-task-run.ts index 1fafac248..094e74903 100644 --- a/packages/cli/src/commands/agent-task-run.ts +++ b/packages/cli/src/commands/agent-task-run.ts @@ -1,7 +1,7 @@ -import { lstat, mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises" +import { lstat, mkdir, mkdtemp, readFile, realpath, rename, rm, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" -import { dirname, join } from "node:path" -import { AGENT_TASK_RUN_REQUEST_SCHEMA, HEADLESS_AGENT_TASK_REQUEST_SCHEMA, artifactResultEnvelope, buildAgentTaskRecipe, DEFAULT_WORDPRESS_VERSION, headlessAgentTaskRequestToRunInput, normalizeAgentRuntimeWorkload, normalizeAgentTaskRunResult, normalizeAgentTerminalResult, normalizeArtifactResultTypedArtifacts, normalizeHeadlessAgentTaskRequest, normalizeHeadlessAgentTaskResult, normalizeTaskInput, parseCommandJson, parseCommandOptions, resolveEffectiveRuntimeToolPolicy, type AgentTaskRunInput, type AgentTaskRunResultSummary, type AgentTerminalResult, type ArtifactResultEnvelope, type HeadlessAgentTaskResult, type SandboxToolPolicySnapshot, type TypedArtifactDTO } from "@automattic/wp-codebox-core" +import { dirname, isAbsolute, join, relative, resolve } from "node:path" +import { AGENT_TASK_RUN_REQUEST_SCHEMA, HEADLESS_AGENT_TASK_REQUEST_SCHEMA, artifactResultEnvelope, buildAgentTaskRecipe, DEFAULT_WORDPRESS_VERSION, headlessAgentTaskRequestToRunInput, normalizeAgentRuntimeExecutionChanges, normalizeAgentRuntimeWorkload, normalizeAgentTaskRunResult, normalizeAgentTerminalResult, normalizeArtifactResultTypedArtifacts, normalizeHeadlessAgentTaskRequest, normalizeHeadlessAgentTaskResult, normalizeTaskInput, parseCommandJson, parseCommandOptions, publicArtifactRefGroups, resolveEffectiveRuntimeToolPolicy, type AgentTaskRunInput, type AgentTaskRunResultSummary, type AgentTerminalResult, type ArtifactResultEnvelope, type HeadlessAgentTaskResult, type SandboxToolPolicySnapshot, type TypedArtifactDTO } from "@automattic/wp-codebox-core" import { stripUndefined } from "@automattic/wp-codebox-core/internals" import { runRecipeRunCommand } from "./recipe-run.js" @@ -167,17 +167,19 @@ export async function runAgentTask(input: AgentTaskRunInput, options: AgentTaskR const runtimeRecord = objectValue(run.runtime) || {} const agentBundle = objectValue(input.agent_bundle) || {} const metadataRuntime = objectValue(objectValue(run.metadata)?.agent_runtime) + const agentResult = objectValue(run.agentResult) || objectValue(runRecord.agentResult) || objectValue(artifactsRecord.agentResult) || {} + const executionChanges = await validatedExecutionChanges(agentResult, artifacts) const workload = normalizeAgentRuntimeWorkload(metadataRuntime?.workload ?? run, { requiredOutputs: stringRecord(agentBundle.engine_data_outputs), toolRecorders: agentBundle.tool_recorders, workloadId: stringValue(agentBundle.workload_id) || stringValue(agentBundle.agent_slug) || stringValue(agentBundle.flow_slug) || undefined, + executionChanges, }) const hasAgentBundle = Object.keys(agentBundle).length > 0 const recipeSuccess = Boolean(run.success) && (!hasAgentBundle || workload.success) const agentTaskResult = agentTaskResultFromRun(run, runRecord, artifactsRecord) const terminalResult = terminalResultFromRun(run, agentTaskResult) const completionOutcome = objectValue(run.completionOutcome) || objectValue(run.completion_outcome) || objectValue(artifactsRecord.completionOutcome) || objectValue(artifactsRecord.completion_outcome) || {} - const agentResult = objectValue(run.agentResult) || objectValue(runRecord.agentResult) || objectValue(artifactsRecord.agentResult) || {} const normalizedRunResult = normalizeAgentTaskRunResult({ ...run, success: recipeSuccess, @@ -713,6 +715,70 @@ function artifactResultDiagnostics(diagnostics: Array>): })) } +const MAX_CANONICAL_EXECUTION_ARTIFACT_BYTES = 5 * 1024 * 1024 + +/** + * Counters are runner diagnostics, not proof of a change. Only captured canonical + * artifacts under the recipe's trusted root may satisfy semantic output policy. + */ +export async function validatedExecutionChanges(agentResult: Record, artifactRoot: string) { + const counters = normalizeAgentRuntimeExecutionChanges(agentResult) + if (!counters || counters.changed_files_count <= 0 || counters.patch_bytes <= 0) return undefined + + const groups = publicArtifactRefGroups(agentResult) + // Recipe capture exposes the canonical descriptors as changedFiles/patch; some + // runners additionally project them into public refs. Treat either shape as a + // single reference, never a counter-only substitute. + const directChanged = objectValue(agentResult.changedFiles) + const directPatch = objectValue(agentResult.patch) + const changedRefs = groups.changed_files.length > 0 ? groups.changed_files : (stringValue(directChanged?.artifact) ? [{ path: stringValue(directChanged?.artifact), sha256: stringValue(directChanged?.sha256) }] : []) + const patchRefs = groups.patches.length > 0 ? groups.patches : (stringValue(directPatch?.artifact) ? [{ path: stringValue(directPatch?.artifact), sha256: stringValue(directPatch?.sha256) }] : []) + if (changedRefs.length !== 1 || patchRefs.length !== 1) return undefined + const [changedRef] = changedRefs + const [patchRef] = patchRefs + if (!changedRef?.path || !patchRef?.path) return undefined + + try { + const root = await realpath(resolve(artifactRoot)) + const bundleDirectory = stringValue(objectValue(agentResult.artifacts)?.directory) + const base = bundleDirectory ? await trustedArtifactPath(root, bundleDirectory, true) : root + const [changedPath, patchPath] = await Promise.all([trustedArtifactPath(base, changedRef.path), trustedArtifactPath(base, patchRef.path)]) + const [changedText, patch] = await Promise.all([readBoundedArtifact(changedPath), readBoundedArtifact(patchPath)]) + const changed = objectValue(JSON.parse(changedText)) + const files = Array.isArray(changed?.files) ? changed.files : [] + const paths = files.map((file) => stringValue(objectValue(file)?.relativePath)).filter(Boolean) + if (changed?.schema !== "wp-codebox/changed-files/v1" || paths.length === 0 || paths.length !== counters.changed_files_count || !patch.trim() || Buffer.byteLength(patch) !== counters.patch_bytes) return undefined + const patchPaths = patch.split("\n").flatMap((line) => { + if (!line.startsWith("--- ") && !line.startsWith("+++ ")) return [] + const path = line.slice(4).split("\t", 1)[0].trim().replace(/^[ab]\//, "") + return path === "/dev/null" ? [] : [path] + }) + if (patchPaths.length === 0 || paths.some((path) => !patchPaths.includes(path)) || patchPaths.some((path) => !paths.includes(path))) return undefined + return normalizeAgentRuntimeExecutionChanges({ + changedFiles: { count: paths.length, artifact: changedRef.path, bytes: Buffer.byteLength(changedText), sha256: changedRef.sha256 }, + patch: { bytes: Buffer.byteLength(patch), artifact: patchRef.path, sha256: patchRef.sha256 }, + }) + } catch { + return undefined + } +} + +async function trustedArtifactPath(root: string, artifactPath: string, directory = false): Promise { + const candidate = isAbsolute(artifactPath) ? resolve(artifactPath) : resolve(root, artifactPath) + const stat = await lstat(candidate) + if ((!directory && !stat.isFile()) || (directory && !stat.isDirectory()) || stat.isSymbolicLink() || (!directory && stat.size > MAX_CANONICAL_EXECUTION_ARTIFACT_BYTES)) throw new Error("Canonical execution artifact must be a bounded regular file.") + const resolved = await realpath(candidate) + const contained = relative(root, resolved) + if (resolved !== root && (contained === ".." || contained.startsWith(`..${String.fromCharCode(47)}`) || isAbsolute(contained))) throw new Error("Canonical execution artifact escapes the trusted artifact root.") + return resolved +} + +async function readBoundedArtifact(path: string): Promise { + const bytes = await readFile(path) + if (bytes.length > MAX_CANONICAL_EXECUTION_ARTIFACT_BYTES || bytes.includes(0)) throw new Error("Canonical execution artifact must be bounded text.") + return bytes.toString("utf8") +} + async function readJsonRecord(path: string): Promise | undefined> { if (!path) return undefined try { diff --git a/packages/runtime-core/src/agent-runtime-workload.ts b/packages/runtime-core/src/agent-runtime-workload.ts index b463f2f13..3695895b1 100644 --- a/packages/runtime-core/src/agent-runtime-workload.ts +++ b/packages/runtime-core/src/agent-runtime-workload.ts @@ -97,11 +97,18 @@ export interface AgentRuntimeWorkload { metadata: Record } +export interface AgentRuntimeExecutionChanges { + changed_files_count: number + patch_bytes: number + refs: AgentRuntimeWorkloadArtifact[] +} + export interface AgentRuntimeWorkloadOptions { requiredOutputs?: Record | string[] toolRecorders?: unknown workloadId?: string normalizerAdapters?: AgentRuntimeWorkloadNormalizerAdapter[] + executionChanges?: AgentRuntimeExecutionChanges } export type AgentRuntimeWorkloadDraft = Omit & { success?: boolean } @@ -123,7 +130,12 @@ export function normalizeAgentRuntimeWorkload(raw: unknown, options: AgentRuntim const adapters = workloadNormalizerAdapters(options) const workload = canonicalWorkload ?? workloadFromNormalizerAdapters(raw, options, adapters) ?? emptyWorkload(options) const toolRecorderOutputs = outputsFromToolRecorders(workload, options.toolRecorders) - const outputs = { ...workload.outputs, ...toolRecorderOutputs } + const executionChanges = options.executionChanges + const outputs = { + ...workload.outputs, + ...toolRecorderOutputs, + ...(executionChanges ? { execution_changes: executionChangeSummary(executionChanges) } : {}), + } const diagnostics = [...workload.diagnostics, ...diagnosticsFromWorkload({ ...workload, outputs }, options)] const success = workload.success !== false && diagnostics.every((diagnostic) => !isFailureDiagnostic(diagnostic)) @@ -204,7 +216,7 @@ function diagnosticsFromWorkload(workload: WorkloadDraft, options: AgentRuntimeW }) } - if (workload.scenarios.length === 0 && Object.keys(workload.outputs).length === 0) { + if (workload.scenarios.length === 0 && Object.keys(workload.outputs).length === 0 && !options.executionChanges) { diagnostics.push({ class: "agent_runtime.workload.incomplete", message: "Agent runtime workload did not produce scenarios or semantic outputs.", @@ -215,6 +227,43 @@ function diagnosticsFromWorkload(workload: WorkloadDraft, options: AgentRuntimeW return diagnostics } +export function normalizeAgentRuntimeExecutionChanges(raw: unknown): AgentRuntimeExecutionChanges | undefined { + const agentResult = objectValue(raw) + const changedFiles = objectValue(agentResult?.changedFiles) + const patch = objectValue(agentResult?.patch) + const changedFilesCount = numberValue(changedFiles?.count) + const patchBytes = numberValue(patch?.bytes) + if (!changedFilesCount || !patchBytes) return undefined + + const refs = [ + executionChangeRef("codebox-changed-files", changedFiles), + executionChangeRef("codebox-patch", patch), + ].filter((ref): ref is AgentRuntimeWorkloadArtifact => Boolean(ref)) + + return { changed_files_count: changedFilesCount, patch_bytes: patchBytes, refs } +} + +function executionChangeSummary(changes: AgentRuntimeExecutionChanges): Record { + return { + summary: `Agent execution changed ${changes.changed_files_count} file${changes.changed_files_count === 1 ? "" : "s"} with a ${changes.patch_bytes}-byte patch.`, + changed_files_count: changes.changed_files_count, + patch_bytes: changes.patch_bytes, + refs: changes.refs, + } +} + +function executionChangeRef(kind: string, evidence: Record | undefined): AgentRuntimeWorkloadArtifact | undefined { + const path = stringValue(evidence?.artifact) + if (!path) return undefined + return stripUndefined({ + id: path, + kind, + path, + sha256: stringValue(evidence?.sha256) || undefined, + size_bytes: numberValue(evidence?.bytes), + }) +} + function missingRequiredOutputs(workload: WorkloadDraft, requiredOutputs: AgentRuntimeWorkloadOptions["requiredOutputs"]): Array<{ name: string; path?: string }> { const required = Array.isArray(requiredOutputs) ? Object.fromEntries(requiredOutputs.map((name) => [name, name])) diff --git a/packages/runtime-playground/src/artifacts.ts b/packages/runtime-playground/src/artifacts.ts index 7013caf65..45f77bd7a 100644 --- a/packages/runtime-playground/src/artifacts.ts +++ b/packages/runtime-playground/src/artifacts.ts @@ -739,7 +739,9 @@ export async function directoryDiff(baselineDirectory: string, currentDirectory: } const path = `${targetPrefix}/${relativePath}` - patches.push(fileDiff(path, before ?? "", after ?? "", before === undefined, after === undefined)) + // Patches are promoted into the caller workspace, so their headers are + // repository-relative while the manifest retains the sandbox mount target. + patches.push(fileDiff(relativePath, before ?? "", after ?? "", before === undefined, after === undefined)) files.push({ path, relativePath, @@ -880,8 +882,8 @@ function isRecord(value: unknown): value is Record { function fileDiff(path: string, before: string, after: string, isAdded: boolean, isDeleted: boolean): string { const beforeLines = splitLines(before) const afterLines = splitLines(after) - const gitOldPath = `a${path}` - const gitNewPath = `b${path}` + const gitOldPath = `a/${path}` + const gitNewPath = `b/${path}` const oldPath = isAdded ? "/dev/null" : gitOldPath const newPath = isDeleted ? "/dev/null" : gitNewPath const lines = [ diff --git a/tests/agent-task-canonical-evidence.test.ts b/tests/agent-task-canonical-evidence.test.ts new file mode 100644 index 000000000..b04250ae5 --- /dev/null +++ b/tests/agent-task-canonical-evidence.test.ts @@ -0,0 +1,41 @@ +import assert from "node:assert/strict" +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { validatedExecutionChanges } from "../packages/cli/src/commands/agent-task-run.js" + +const root = await mkdtemp(join(tmpdir(), "wp-codebox-canonical-evidence-")) +const artifacts = join(root, "artifacts") +const changedPath = join(artifacts, "files", "changed-files.json") +const patchPath = join(artifacts, "files", "patch.diff") +const patch = "--- a/README.md\n+++ b/README.md\n@@ -1 +1 @@\n-before\n+after\n" +const changed = { schema: "wp-codebox/changed-files/v1", files: [{ relativePath: "README.md", status: "modified" }] } + +const agentResult = (overrides: Record = {}) => ({ + changedFiles: { count: 1, bytes: Buffer.byteLength(JSON.stringify(changed)), artifact: "files/changed-files.json" }, + patch: { bytes: Buffer.byteLength(patch), artifact: "files/patch.diff" }, + ...overrides, +}) + +try { + await mkdir(join(artifacts, "files"), { recursive: true }) + assert.equal(await validatedExecutionChanges(agentResult(), artifacts), undefined, "counters alone are not evidence") + + await writeFile(changedPath, JSON.stringify(changed)) + await writeFile(patchPath, patch) + const valid = await validatedExecutionChanges(agentResult(), artifacts) + assert.deepEqual(valid?.refs.map((ref) => ref.kind), ["codebox-changed-files", "codebox-patch"], "valid captured canonical refs are semantic evidence") + + await rm(patchPath) + assert.equal(await validatedExecutionChanges(agentResult(), artifacts), undefined, "missing canonical files cannot satisfy semantic outputs") + await writeFile(patchPath, "") + assert.equal(await validatedExecutionChanges(agentResult(), artifacts), undefined, "empty patches cannot satisfy semantic outputs") + await writeFile(patchPath, patch.replaceAll("README.md", "other.md")) + assert.equal(await validatedExecutionChanges(agentResult(), artifacts), undefined, "mismatched paths cannot satisfy semantic outputs") + await writeFile(patchPath, patch) + assert.equal(await validatedExecutionChanges(agentResult({ patch: { bytes: 1, artifact: "files/patch.diff" } }), artifacts), undefined, "counter and capture mismatches cannot satisfy semantic outputs") +} finally { + await rm(root, { recursive: true, force: true }) +} + +console.log("agent task canonical evidence ok") diff --git a/tests/agent-task-contracts.test.ts b/tests/agent-task-contracts.test.ts index 9b5ccda18..253922146 100644 --- a/tests/agent-task-contracts.test.ts +++ b/tests/agent-task-contracts.test.ts @@ -3,7 +3,7 @@ import { mkdirSync, mkdtempSync, readdirSync, rmSync, statSync, writeFileSync } import { tmpdir } from "node:os" import { join } from "node:path" import { chdir, cwd } from "node:process" -import { AGENT_TASK_RUN_REQUEST_SCHEMA, AGENT_TASK_RUN_RESULT_JSON_SCHEMA, AGENT_TASK_RUN_RESULT_SCHEMA, ARTIFACT_RESULT_ENVELOPE_SCHEMA, HEADLESS_AGENT_TASK_REQUEST_JSON_SCHEMA, HEADLESS_AGENT_TASK_REQUEST_SCHEMA, HEADLESS_AGENT_TASK_RESULT_JSON_SCHEMA, HEADLESS_AGENT_TASK_RESULT_SCHEMA, PREVIEW_LEASE_SCHEMA, TYPED_ARTIFACT_SCHEMA, buildAgentTaskRecipe, headlessAgentTaskRequestToRunInput, normalizeAgentRuntimeWorkload, normalizeAgentTaskRunResult, normalizeAgentTerminalResult, normalizeArtifactResultTypedArtifacts, normalizeHeadlessAgentTaskRequest, normalizeHeadlessAgentTaskResult, normalizeRecipeRunSummary, normalizeTaskInput } from "../packages/runtime-core/src/index.js" +import { AGENT_TASK_RUN_REQUEST_SCHEMA, AGENT_TASK_RUN_RESULT_JSON_SCHEMA, AGENT_TASK_RUN_RESULT_SCHEMA, ARTIFACT_RESULT_ENVELOPE_SCHEMA, HEADLESS_AGENT_TASK_REQUEST_JSON_SCHEMA, HEADLESS_AGENT_TASK_REQUEST_SCHEMA, HEADLESS_AGENT_TASK_RESULT_JSON_SCHEMA, HEADLESS_AGENT_TASK_RESULT_SCHEMA, PREVIEW_LEASE_SCHEMA, TYPED_ARTIFACT_SCHEMA, buildAgentTaskRecipe, headlessAgentTaskRequestToRunInput, normalizeAgentRuntimeExecutionChanges, normalizeAgentRuntimeWorkload, normalizeAgentTaskRunResult, normalizeAgentTerminalResult, normalizeArtifactResultTypedArtifacts, normalizeHeadlessAgentTaskRequest, normalizeHeadlessAgentTaskResult, normalizeRecipeRunSummary, normalizeTaskInput } from "../packages/runtime-core/src/index.js" import { effectivePolicyCommands } from "../packages/runtime-core/src/contracts.js" import { commandCatalogOutput } from "../packages/cli/src/commands/discovery.js" import { agentTaskResultFromRun, agentTaskRunExitCode, agentTaskRunJsonOutput, normalizeAgentTaskRunCliInput, writeAgentTaskRunResultFile } from "../packages/cli/src/commands/agent-task-run.js" @@ -273,6 +273,28 @@ assert.equal(strictNestedTerminal, undefined) const strictRuntimeWorkload = normalizeAgentRuntimeWorkload({ outputs: { answer: "legacy" } }) assert.deepEqual(strictRuntimeWorkload.outputs, {}) +const executionChanges = normalizeAgentRuntimeExecutionChanges({ + changedFiles: { count: 1, artifact: "changed-files.json", bytes: 128, sha256: "changed" }, + patch: { artifact: "patch.diff", bytes: 256, sha256: "patch" }, +}) +const changedRuntimeWorkload = normalizeAgentRuntimeWorkload({}, { executionChanges }) +assert.equal(changedRuntimeWorkload.success, true) +assert.equal(changedRuntimeWorkload.diagnostics.some((diagnostic) => diagnostic.data?.reason === "missing_semantic_outputs"), false) +assert.deepEqual(changedRuntimeWorkload.outputs.execution_changes, { + summary: "Agent execution changed 1 file with a 256-byte patch.", + changed_files_count: 1, + patch_bytes: 256, + refs: [ + { id: "changed-files.json", kind: "codebox-changed-files", path: "changed-files.json", sha256: "changed", size_bytes: 128 }, + { id: "patch.diff", kind: "codebox-patch", path: "patch.diff", sha256: "patch", size_bytes: 256 }, + ], +}) +assert.deepEqual(normalizeAgentRuntimeExecutionChanges({ changedFiles: { count: 0 }, patch: { bytes: 0 } }), undefined) +assert.deepEqual(normalizeAgentRuntimeExecutionChanges({ changedFiles: { count: 1 }, patch: { bytes: 0 } }), undefined) +const emptyChangeRuntimeWorkload = normalizeAgentRuntimeWorkload({}, { executionChanges: normalizeAgentRuntimeExecutionChanges({ changedFiles: { count: 0 }, patch: { bytes: 0 } }) }) +assert.equal(emptyChangeRuntimeWorkload.success, false) +assert.equal(emptyChangeRuntimeWorkload.diagnostics.some((diagnostic) => diagnostic.data?.reason === "missing_semantic_outputs"), true) + const typedArtifactRefs = normalizeArtifactResultTypedArtifacts({ typed_artifacts: [{ schema: TYPED_ARTIFACT_SCHEMA, diff --git a/tests/agent-task-reusable-workflow.test.ts b/tests/agent-task-reusable-workflow.test.ts index 020d34057..3df9c5e9b 100644 --- a/tests/agent-task-reusable-workflow.test.ts +++ b/tests/agent-task-reusable-workflow.test.ts @@ -225,16 +225,27 @@ assert.deepEqual(nativeTaskInput.task_input.sandbox_tool_policy.tools, hostedDoc allowed: true, runtime: { environment: "runtime_local", capability_scope: "runtime_local" }, })), "Hosted Docs Agent run 29298164272 must emit canonical sandbox-local workspace metadata") -assert.deepEqual(nativeTaskInput.task_input.workspaces, [{ - target: "/workspace", - mode: "readwrite", - sourceMode: "repo-backed", - seed: { - type: "directory", - source: tmp, - excludePaths: [".git/**", ".codebox/**", "node_modules/**", "vendor/**", "dist/**", "build/**", "coverage/**", ".cache/**"], - }, -}], "Hosted failures 29324157852 and 29324563665 placed the runner seed outside task_input, so agent-task-run discarded it") + assert.deepEqual(nativeTaskInput.task_input.workspaces.map((entry: Record) => ({ + target: entry.target, + mode: entry.mode, + sourceMode: entry.sourceMode, + seed: { ...(entry.seed as Record), source: "[external snapshot]" }, + })), [{ + target: "/workspace", + mode: "readwrite", + sourceMode: "repo-backed", + seed: { + type: "directory", + source: "[external snapshot]", + excludePaths: [".git/**", ".codebox/**", "node_modules/**", "vendor/**", "dist/**", "build/**", "coverage/**", ".cache/**", ".env", ".env.*", ".npmrc", ".yarnrc.yml", ".pypirc", ".netrc", "auth.json", "id_rsa", "id_ed25519", "*.pem", "*.key", "credential files"], + }, + }], "Hosted failures 29324157852 and 29324563665 placed the runner seed outside task_input, so agent-task-run discarded it") + const seed = nativeTaskInput.task_input.workspaces[0] + assert.notEqual(seed.seed.source, tmp) + const seedProvenance = nativeTaskInput.task_input.runtime_task.input.metadata.runner_workspace_seed + assert.match(seedProvenance.digest.sha256, /^[a-f0-9]{64}$/) + assert.deepEqual(seedProvenance.excludes, [".git/**", ".codebox/**", "node_modules/**", "vendor/**", "dist/**", "build/**", "coverage/**", ".cache/**", ".env", ".env.*", ".npmrc", ".yarnrc.yml", ".pypirc", ".netrc", "auth.json", "id_rsa", "id_ed25519", "*.pem", "*.key", "credential files"]) + assert.doesNotMatch(JSON.stringify(seedProvenance), new RegExp(tmp.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))) assert.equal(nativeTaskInput.workspaces, undefined, "Runner workspace configuration must only use the canonical task_input.workspaces field") const executeNativeAgentTask = new URL("../.github/scripts/run-agent-task/execute-native-agent-task.mjs", import.meta.url).pathname diff --git a/tests/execute-native-agent-task-interruption.test.mjs b/tests/execute-native-agent-task-interruption.test.mjs new file mode 100644 index 000000000..03965b533 --- /dev/null +++ b/tests/execute-native-agent-task-interruption.test.mjs @@ -0,0 +1,111 @@ +import assert from "node:assert/strict" +import { existsSync } from "node:fs" +import { mkdtemp, mkdir, readdir, readFile, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join, resolve } from "node:path" +import { spawn } from "node:child_process" +import { setTimeout as delay } from "node:timers/promises" + +const root = resolve(".") +const executor = join(root, ".github/scripts/run-agent-task/execute-native-agent-task.mjs") +const SIGNAL_EXIT_CODES = { SIGINT: 130, SIGTERM: 143, SIGHUP: 129 } + +async function prepare() { + const temp = await mkdtemp(join(tmpdir(), "wp-codebox-native-interruption-")) + // A dedicated TMPDIR makes seed-snapshot residue deterministic to observe. + const seedTemp = join(temp, "seed-tmp") + const workspace = join(temp, "workspace") + await mkdir(seedTemp, { recursive: true }) + await mkdir(join(workspace, ".codebox"), { recursive: true }) + await writeFile(join(workspace, "README.md"), "before\n") + const request = { + schema: "wp-codebox/agent-task-workflow-request/v1", + model: { provider: "openai", name: "gpt-5" }, + external_package_source: { + repository: "owner/agents", + revision: "a".repeat(40), + path: "agent.agent.json", + digest: `sha256-bytes-v1:${"b".repeat(64)}`, + }, + runtime_sources: [], + workload: { id: "interruption-1", label: "Interruption" }, + target_repo: "owner/repo", + prompt: "Interruption lifecycle fixture.", + writable_paths: "README.md", + runner_workspace: { enabled: true, repo: "owner/repo", base: "main", branch_prefix: "wp-codebox/agent-task/" }, + verification_commands: [], + drift_checks: [], + success: { requires_pr: false }, + access: { caller_repo: "owner/repo", allowed_repos: ["owner/repo"], access_token_repos: ["owner/repo"] }, + limits: { max_turns: 1, time_budget_ms: 1000 }, + artifacts: { expected: [], declarations: [] }, + outputs: { projections: {} }, + callback_data: {}, + run_agent: true, + dry_run: true, + } + await writeFile(join(workspace, ".codebox", "agent-task-request.json"), JSON.stringify(request)) + return { temp, seedTemp, workspace } +} + +function spawnExecutor({ seedTemp, workspace }, extraEnv = {}) { + const child = spawn(process.execPath, [executor], { + cwd: workspace, + env: { + ...process.env, + NODE_ENV: "test", + TMPDIR: seedTemp, + AGENT_TASK_REQUEST_PATH: join(workspace, ".codebox", "agent-task-request.json"), + AGENT_TASK_WORKSPACE: workspace, + WP_CODEBOX_WORKFLOW_ROOT: root, + GITHUB_TOKEN: "token", + EXTERNAL_PACKAGE_SOURCE_POLICY: JSON.stringify({ version: 1, repositories: { "owner/agents": ["agent.agent.json"] } }), + ...extraEnv, + }, + }) + let stderr = "" + child.stderr.on("data", (chunk) => { stderr += chunk }) + const closed = new Promise((done) => { child.on("close", (code, signal) => done({ code, signal, stderr })) }) + return { child, closed } +} + +async function seedSnapshotEntries(seedTemp) { + return (await readdir(seedTemp)).filter((entry) => entry.startsWith("wp-codebox-runner-workspace-seed-")) +} + +for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"]) { + const fixture = await prepare() + const pauseFile = join(fixture.temp, "pause.json") + const { child, closed } = spawnExecutor(fixture, { WP_CODEBOX_TEST_SEED_SNAPSHOT_PAUSE_FILE: pauseFile }) + + const deadline = Date.now() + 30_000 + while (!existsSync(pauseFile)) { + assert.ok(Date.now() < deadline, `executor did not reach the seed snapshot pause hook for ${signal}`) + await delay(50) + } + const marker = JSON.parse(await readFile(pauseFile, "utf8")) + assert.equal(marker.schema, "wp-codebox/test-seed-snapshot-pause/v1") + assert.ok(marker.seed_snapshot_source.startsWith(fixture.seedTemp), "seed snapshot is created under the controlled TMPDIR") + assert.ok(existsSync(marker.seed_snapshot_source), "seed snapshot exists while the executor is paused") + + child.kill(signal) + const exit = await closed + assert.equal(exit.signal, null, `the executor handles ${signal} itself instead of dying to the default signal action\n${exit.stderr}`) + assert.equal(exit.code, SIGNAL_EXIT_CODES[signal], `conventional exit code for ${signal}\n${exit.stderr}`) + assert.equal(existsSync(marker.seed_snapshot_source), false, `the temp seed snapshot directory is removed on ${signal}`) + assert.deepEqual(await seedSnapshotEntries(fixture.seedTemp), [], `no runner workspace seed content survives ${signal}`) + assert.equal(await readFile(join(fixture.workspace, "README.md"), "utf8"), "before\n", "the host workspace is untouched") + assert.deepEqual((await readdir(join(fixture.workspace, ".codebox"))).sort(), ["agent-task-artifacts", "agent-task-request.json"], "no runner workspace materials survive in the workspace") + assert.deepEqual(await readdir(join(fixture.workspace, ".codebox", "agent-task-artifacts")), [], "no artifacts are staged before interruption") +} + +// Normal completion continues to clean up through the same coordinator. +const fixture = await prepare() +const { closed } = spawnExecutor(fixture) +const exit = await closed +assert.equal(exit.code, 0, exit.stderr) +assert.deepEqual(await seedSnapshotEntries(fixture.seedTemp), [], "no runner workspace seed content survives a normal run") +const result = JSON.parse(await readFile(join(fixture.workspace, ".codebox", "agent-task-workflow-result.json"), "utf8")) +assert.equal(result.status, "skipped") + +console.log("native agent task interruption ok") diff --git a/tests/execute-native-agent-task-playground-e2e.test.ts b/tests/execute-native-agent-task-playground-e2e.test.ts new file mode 100644 index 000000000..76fa74400 --- /dev/null +++ b/tests/execute-native-agent-task-playground-e2e.test.ts @@ -0,0 +1,99 @@ +import assert from "node:assert/strict" +import { execFile } from "node:child_process" +import { createHash } from "node:crypto" +import { access, chmod, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join, resolve } from "node:path" +import { promisify } from "node:util" +import { materializeRuntimeSources, parseExternalPackageSourcePolicy, sha256BytesV1 } from "../.github/scripts/run-agent-task/materialize-external-native-package.mjs" +import { createRunnerWorkspaceSeedSnapshot } from "../.github/scripts/run-agent-task/runner-workspace-seed-snapshot.mjs" + +const execFileAsync = promisify(execFile) +const root = resolve(".") +const executor = join(root, ".github/scripts/run-agent-task/execute-native-agent-task.mjs") +const uploader = join(root, ".github/scripts/run-agent-task/prepare-agent-task-upload.mjs") +const fixture = JSON.parse(await readFile(new URL("../fixtures/agent-task-runtime-sources-run-29299109269.json", import.meta.url), "utf8")) +const temp = await mkdtemp(join(tmpdir(), "wp-codebox-native-agent-task-e2e-")) +const workspace = join(temp, "workspace") +const packagePath = join(temp, "fixture-agent.agent.json") +const interceptor = join(temp, "openai-interceptor") +const publisher = join(temp, "publisher.mjs") +const gh = join(temp, "gh") + +try { + await mkdir(join(workspace, ".codebox"), { recursive: true }) + await mkdir(join(workspace, "node_modules"), { recursive: true }) + await writeFile(join(workspace, "README.md"), "before\n") + await writeFile(join(workspace, ".env"), "PRIVATE_WORKSPACE_SENTINEL\n") + await writeFile(join(workspace, ".env.example"), "PUBLIC_TEMPLATE_VALUE=example\n") + await writeFile(join(workspace, ".npmrc"), "//registry.example/:_authToken=PRIVATE_NPM_SENTINEL\n") + await writeFile(join(workspace, ".netrc"), "machine example login private password PRIVATE_NETRC_SENTINEL\n") + await writeFile(join(workspace, "id_ed25519"), "PRIVATE_KEY_SENTINEL\n") + await writeFile(join(workspace, ".codebox", "private.txt"), "PRIVATE_CODEBOX_SENTINEL\n") + await writeFile(join(workspace, "node_modules", "private.txt"), "PRIVATE_NODE_MODULES_SENTINEL\n") + const packageBytes = Buffer.from(`${JSON.stringify({ schema_version: 1, bundle_slug: "fixture-agent", agent: { agent_slug: "fixture-agent", agent_name: "Fixture Agent", description: "Playground imported-agent fixture.", agent_config: { instructions: "Read and update README.", enabled_tools: ["workspace_read", "workspace_edit"], modes: ["chat"] } } })}\n`) + await writeFile(packagePath, packageBytes) + await mkdir(interceptor, { recursive: true }) + await writeFile(join(interceptor, "interceptor.php"), ` $url, 'body' => $args['body'] ?? '' ); file_put_contents( $file, wp_json_encode( $requests ) ); + if ( str_ends_with( $url, '/models' ) ) $body = array( 'object' => 'list', 'data' => array( array( 'id' => 'gpt-5.5', 'object' => 'model' ) ) ); + else { $turn = count( array_filter( $requests, static fn( $request ) => str_ends_with( $request['url'], '/responses' ) ) ); $output = 1 === $turn ? array( array( 'id' => 'fc-read', 'type' => 'function_call', 'call_id' => 'read', 'name' => 'workspace_read', 'arguments' => wp_json_encode( array( 'path' => 'README.md' ) ) ) ) : ( 2 === $turn ? array( array( 'id' => 'fc-edit', 'type' => 'function_call', 'call_id' => 'edit', 'name' => 'workspace_edit', 'arguments' => wp_json_encode( array( 'path' => 'README.md', 'old_string' => 'before', 'new_string' => 'after' ) ) ) ) : array( array( 'id' => 'msg-fixture', 'type' => 'message', 'role' => 'assistant', 'status' => 'completed', 'content' => array( array( 'type' => 'output_text', 'text' => 'Updated README.', 'annotations' => array() ) ) ) ) ); $body = array( 'id' => 'fixture-' . $turn, 'object' => 'response', 'status' => 'completed', 'output' => $output, 'usage' => array( 'input_tokens' => 1, 'output_tokens' => 1, 'total_tokens' => 2 ) ); } + return array( 'headers' => array( 'content-type' => 'application/json' ), 'body' => wp_json_encode( $body ), 'response' => array( 'code' => 200, 'message' => 'OK' ), 'cookies' => array(), 'filename' => null ); +}, 1000, 3 );`) + const provider = fixture.runtime_sources[1] + const policyRaw = JSON.stringify({ version: 1, repositories: { "fixture/agent": ["fixture-agent.agent.json"] }, runtime_sources: { "automattic/agents-api": ["."], "wordpress/php-ai-client": ["."] }, runtime_artifacts: [{ url: provider.source.url, sha256: provider.source.sha256 }] }) + const policy = parseExternalPackageSourcePolicy(policyRaw) + const sources = await materializeRuntimeSources(fixture.runtime_sources, { policy, forbiddenRoots: [workspace] }) + const lowered = sources.lowered.reduce((result: Record, entry: Record) => { + for (const [key, values] of Object.entries(entry)) result[key] = [...(result[key] ?? []), ...values] + return result + }, {}) + lowered.provider_plugin_paths = [...(lowered.provider_plugin_paths ?? []), interceptor] + lowered.provider_plugins = [...(lowered.provider_plugins ?? []), { source: interceptor, slug: "native-agent-task-openai-interceptor", pluginFile: "native-agent-task-openai-interceptor/interceptor.php", activate: true, loadAs: "plugin" }] + const preflightSnapshot = await createRunnerWorkspaceSeedSnapshot(workspace) + for (const secret of [".env", ".npmrc", ".netrc", "id_ed25519"]) await assert.rejects(access(join(preflightSnapshot.source, secret)), /ENOENT/, `workspace_read cannot access ${secret} because the seed excludes it`) + await rm(preflightSnapshot.source, { recursive: true, force: true }) + const request = { schema: "wp-codebox/agent-task-workflow-request/v1", model: { provider: "openai", name: "gpt-5.5" }, external_package_source: { repository: "fixture/agent", revision: "a".repeat(40), path: "fixture-agent.agent.json", digest: sha256BytesV1(packageBytes) }, runtime_sources: [provider], workload: { id: "native-e2e", label: "Native E2E" }, target_repo: "owner/repo", prompt: "Read README.md, then replace before with after.", writable_paths: "README.md", runner_workspace: { enabled: true, repo: "owner/repo", base: "main", branch_prefix: "fixture/" }, verification_commands: [{ command: "test \"$(cat README.md)\" = after", description: "Verify README" }], drift_checks: [], validation_dependencies: "", success: { requires_pr: true }, access: { caller_repo: "owner/repo", allowed_repos: ["owner/repo"], access_token_repos: ["owner/repo"] }, limits: { max_turns: 3, time_budget_ms: 180000 }, artifacts: { expected: [], declarations: [] }, outputs: { projections: { pr_url: "metadata.runner_workspace_publication.pull_request.url" } }, callback_data: {}, run_agent: true, dry_run: false } + await writeFile(join(workspace, ".codebox", "agent-task-request.json"), JSON.stringify(request)) + await writeFile(publisher, `export async function publishRunnerWorkspace() { return { schema: "wp-codebox/runner-workspace-publication-result/v1", success: true, status: "published", backend: "fixture", pull_request: { url: "https://github.com/owner/repo/pull/1", opened: true, reused: false } } }`) + await writeFile(gh, `#!/usr/bin/env node\nprocess.stdout.write(JSON.stringify({ html_url: "https://github.com/owner/repo/pull/1", base: { repo: { full_name: "owner/repo" } } }))\n`) + await chmod(gh, 0o755) + const execution = await execFileAsync(process.execPath, [executor], { cwd: workspace, timeout: 300_000, maxBuffer: 2 * 1024 * 1024, env: { ...process.env, NODE_ENV: "test", AGENT_TASK_REQUEST_PATH: join(workspace, ".codebox", "agent-task-request.json"), AGENT_TASK_WORKSPACE: workspace, WP_CODEBOX_WORKFLOW_ROOT: root, WP_CODEBOX_TEST_EXTERNAL_PACKAGE_PATH: packagePath, WP_CODEBOX_TEST_RUNTIME_SOURCE_INPUTS: JSON.stringify(lowered), WP_CODEBOX_TEST_PUBLISHER_MODULE: publisher, GITHUB_TOKEN: "e2e-github-secret", OPENAI_API_KEY: "e2e-openai-secret", EXPLICIT_ACCESS_TOKEN_CONFIGURED: "true", EXTERNAL_PACKAGE_SOURCE_POLICY: policyRaw, PATH: `${temp}:${process.env.PATH ?? ""}` } }).then(() => ({ code: 0 }), (error: any) => ({ code: error.code, stderr: error.stderr })) + const result = JSON.parse(await readFile(join(workspace, ".codebox", "agent-task-workflow-result.json"), "utf8")) + const input = JSON.parse(await readFile(join(workspace, ".codebox", "native-agent-task-input.json"), "utf8")) + const seededWorkspace = input.task_input.workspaces[0] + assert.deepEqual(input.task_input.workspaces.map((entry: { target: string, mode: string }) => ({ target: entry.target, mode: entry.mode })), [{ target: "/workspace", mode: "readwrite" }]) + assert.notEqual(seededWorkspace.seed.source, workspace) + const seedProvenance = input.task_input.runtime_task.input.metadata.runner_workspace_seed + assert.match(seedProvenance.digest.sha256, /^[a-f0-9]{64}$/) + assert.equal(seedProvenance.files, 2, "only README and the explicit .env.example template are copied") + assert.equal(seedProvenance.excluded.files, 6) + assert.deepEqual(seedProvenance.excluded.categories, [{ category: "credentials", count: 2 }, { category: "environment", count: 1 }, { category: "generated-tree", count: 2 }, { category: "private-key", count: 1 }]) + assert.equal(execution.code, 0, `${execution.stderr ?? ""}\n${JSON.stringify(result)}`) + assert.equal(result.success, true) + assert.equal(await readFile(join(workspace, "README.md"), "utf8"), "after\n") + assert.equal(result.runtime_result.agent_result?.changedFiles?.count, 1) + assert.ok(result.runtime_result.agent_result?.patch?.bytes > 0) + assert.equal(result.runtime_result.metadata.agent_runtime.workload.outputs.execution_changes.changed_files_count, 1) + assert.ok(result.runtime_result.metadata.agent_runtime.workload.outputs.execution_changes.patch_bytes > 0) + assert.deepEqual(result.runtime_result.typed_artifacts, []) + assert.equal(result.runtime_result.agent_task_run_result.refs.changed_files.length, 1) + assert.equal(result.runtime_result.agent_task_run_result.refs.patches.length, 1) + assert.equal(result.runtime_result.metadata.runner_workspace_publication.pull_request.url, "https://github.com/owner/repo/pull/1") + const serialized = JSON.stringify(result) + for (const privateValue of ["e2e-github-secret", "e2e-openai-secret", "PRIVATE_WORKSPACE_SENTINEL", "PRIVATE_NPM_SENTINEL", "PRIVATE_NETRC_SENTINEL", "PRIVATE_KEY_SENTINEL", "PRIVATE_CODEBOX_SENTINEL", "PRIVATE_NODE_MODULES_SENTINEL", sources.root, workspace]) assert.doesNotMatch(serialized, new RegExp(privateValue.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))) + await execFileAsync(process.execPath, [uploader], { cwd: workspace, env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_REQUEST_PATH: join(workspace, ".codebox", "agent-task-request.json"), AGENT_TASK_UPLOAD_PATH: join(workspace, ".codebox", "agent-task-upload"), GITHUB_TOKEN: "e2e-github-secret", OPENAI_API_KEY: "e2e-openai-secret" } }) + const staged = JSON.stringify(await Promise.all((await readdir(join(workspace, ".codebox", "agent-task-upload"), { recursive: true })).map((path) => readFile(join(workspace, ".codebox", "agent-task-upload", path), "utf8").catch(() => "")))) + for (const privateValue of ["wp-codebox-runner-workspace-seed-", workspace, "PRIVATE_WORKSPACE_SENTINEL", "PRIVATE_NPM_SENTINEL", "PRIVATE_NETRC_SENTINEL", "PRIVATE_KEY_SENTINEL"]) assert.doesNotMatch(staged, new RegExp(privateValue.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))) + const stagedInput = JSON.parse(await readFile(join(workspace, ".codebox", "agent-task-upload", ".codebox", "native-agent-task-input.json"), "utf8")) + assert.deepEqual(stagedInput.task_input.workspaces[0].seed, { kind: "runner-workspace-seed", digest: seedProvenance.digest.sha256, files: 2, bytes: seedProvenance.bytes, excludes: seedProvenance.excludes, excluded: seedProvenance.excluded }) + const digest = createHash("sha256").update(await readFile(join(workspace, "README.md"))).digest("hex") + assert.equal(digest.length, 64) + await rm(sources.root, { recursive: true, force: true }) + console.log("native agent task Playground e2e ok") +} finally { + await rm(temp, { recursive: true, force: true }) +} diff --git a/tests/runtime-sources-materialization.test.ts b/tests/runtime-sources-materialization.test.ts index e3b119a2c..9f6aaab76 100644 --- a/tests/runtime-sources-materialization.test.ts +++ b/tests/runtime-sources-materialization.test.ts @@ -287,7 +287,11 @@ await withTempDir("wp-codebox-runtime-sources-workflow-", async (directory) => { const executor = await readFile(new URL("../.github/scripts/run-agent-task/execute-native-agent-task.mjs", import.meta.url), "utf8") assert.match(executor, /for \(const signal of \["SIGINT", "SIGTERM", "SIGHUP"\]\)/) -assert.match(executor, /cleanupPrivateRuntimeSources\(\)\.finally\(\(\) => process\.exit\(128\)\)/) +assert.match(executor, /const SIGNAL_EXIT_CODES = \{ SIGINT: 130, SIGTERM: 143, SIGHUP: 129 \}/) +assert.match(executor, /cleanupMaterializedSources\(\)\.finally\(\(\) => process\.exit\(SIGNAL_EXIT_CODES\[signal\]\)\)/) +assert.match(executor, /\} finally \{\n await cleanupMaterializedSources\(\)\n\}/, "every top-level completion path routes through the single cleanup coordinator") +assert.equal(executor.match(/function cleanupMaterializedSources/g)?.length, 1, "cleanup stays centralized in one coordinator") +assert.doesNotMatch(executor, /cleanupPrivateRuntimeSources|cleanupRunnerWorkspaceSeedSnapshot/, "no duplicate independent cleanup logic") assert.match(executor, /const executionInputPath = privateRuntimeSourceRoot \? join\(privateRuntimeSourceRoot, "native-agent-task-input\.json"\) : runtimeInputPath/) assert.match(executor, /const privatePreparationRoot = privateRuntimeSourceRoot \? join\(privateRuntimeSourceRoot, "prepared-runtime-sources"\) : ""/) assert.match(executor, /sanitizeRuntimeSourceValue\(nativeRuntimeResult, privateRuntimeSourceRootForSanitization\)/)