Harden runner workspace evidence and uploads

Author: chubes4

.github/scripts/run-agent-task/prepare-agent-task-upload.mjs

@@ -13,6 +13,7 @@ const secretValues = ["OPENAI_API_KEY", "MODEL_PROVIDER_SECRET_1", "MODEL_PROVID
 const runtimeSourceRoot = process.env.WP_CODEBOX_RUNTIME_SOURCE_ROOT ? resolve(process.env.WP_CODEBOX_RUNTIME_SOURCE_ROOT) : ""
 const runtimeSourcePrefix = process.env.WP_CODEBOX_RUNTIME_SOURCE_PREFIX ? resolve(process.env.WP_CODEBOX_RUNTIME_SOURCE_PREFIX) : ""
 const runtimeSourceRoots = [runtimeSourceRoot, runtimeSourcePrefix].filter(Boolean)
+const privateUploadRoots = [...runtimeSourceRoots, workspace]
 const SOURCE_TREE = /(^|\/)(prepared-plugins|prepared-source-packages|source-package[^/]*)(\/|$)/i
 const SOURCE_FILE = /\.(?:php|phtml|js|mjs|cjs|jsx|ts|tsx)$/i
 const PHP_OPENING_TAG = /<\?(?:php|=)(?:\s|$)/i
@@ -32,16 +33,32 @@ function redact(value) {
 }
 
 function sanitizeText(text) {
-  return sanitizeRuntimeSourceJson(text, runtimeSourceRoots)
+  return sanitizeRuntimeSourceJson(text, privateUploadRoots)
 }
 
 function compactNativeInput(text) {
   const privateFields = new Set(["source_package_root", "component_contracts", "extra_plugins", "provider_plugins", "runtime_overlays", "prepared_sources"])
-  const compact = (value) => {
+  let seedProvenance = {}
+  try {
+    const parsed = JSON.parse(sanitizeText(text))
+    seedProvenance = record(record(record(parsed).task_input).runtime_task?.input?.metadata).runner_workspace_seed
+  } catch {}
+  const compact = (value, key = "") => {
     if (Array.isArray(value)) return value.map(compact)
     const entry = record(value)
     if (!Object.keys(entry).length) return value
-    return Object.fromEntries(Object.entries(entry).flatMap(([key, item]) => privateFields.has(key) ? [] : [[key, compact(item)]]))
+    if (key === "seed" && typeof entry.source === "string") {
+      const provenance = record(seedProvenance)
+      return {
+        kind: "runner-workspace-seed",
+        digest: record(provenance.digest).sha256,
+        files: provenance.files,
+        bytes: provenance.bytes,
+        excludes: provenance.excludes,
+        excluded: provenance.excluded,
+      }
+    }
+    return Object.fromEntries(Object.entries(entry).flatMap(([childKey, item]) => privateFields.has(childKey) ? [] : [[childKey, compact(item, childKey)]]))
   }
   try {
     return `${JSON.stringify(compact(JSON.parse(sanitizeText(text))), null, 2)}\n`
@@ -50,6 +67,38 @@ function compactNativeInput(text) {
   }
 }
 
+function assertNoSeedSnapshotPaths(text) {
+  if (/wp-codebox-runner-workspace-seed-/i.test(text)) throw new Error("Temporary runner workspace seed paths must never be persisted in artifact uploads.")
+  try {
+    const visit = (value) => {
+      if (Array.isArray(value)) return value.forEach(visit)
+      const entry = record(value)
+      if (!Object.keys(entry).length) return
+      if (record(entry.seed).source && isAbsolute(record(entry.seed).source)) throw new Error("Absolute runner workspace seed paths must never be persisted in artifact uploads.")
+      Object.values(entry).forEach(visit)
+    }
+    visit(JSON.parse(text))
+  } catch (error) {
+    if (error instanceof Error && /seed paths/.test(error.message)) throw error
+  }
+}
+
+function sanitizeSeedSnapshotJson(text) {
+  try {
+    const compact = (value, key = "") => {
+      if (typeof value === "string") return value.replace(/\/?[^\s"']*wp-codebox-runner-workspace-seed-[^\s"']*/gi, "[runner-workspace-seed]")
+      if (Array.isArray(value)) return value.map((entry) => compact(entry))
+      const entry = record(value)
+      if (!Object.keys(entry).length) return value
+      if (key === "seed" && typeof entry.source === "string") return { kind: "runner-workspace-seed" }
+      return Object.fromEntries(Object.entries(entry).map(([childKey, item]) => [childKey, compact(item, childKey)]))
+    }
+    return `${JSON.stringify(compact(JSON.parse(text)), null, 2)}\n`
+  } catch {
+    return text
+  }
+}
+
 function isPrivateRuntimePath(value) {
   if (!runtimeSourceRoots.length || typeof value !== "string") return false
   const path = resolve(value)
@@ -82,8 +131,10 @@ async function stageTextFile(source, destination, options = {}) {
   const contents = openedMetadata.isFile() && openedMetadata.size <= MAX_UPLOAD_FILE_BYTES ? await handle.readFile() : null
   await handle.close()
   if (!contents || contents.includes(0) || !isUtf8(contents)) return false
-  const text = redact(options.compactNativeInput ? compactNativeInput(contents.toString("utf8")) : sanitizeText(contents.toString("utf8")))
-  assertNoRuntimeSourcePaths(text, runtimeSourceRoots, "Runtime source paths must never be persisted in artifact uploads.")
+  const sanitized = options.compactNativeInput ? compactNativeInput(contents.toString("utf8")) : sanitizeText(contents.toString("utf8"))
+  const text = redact(sanitizeSeedSnapshotJson(sanitized))
+  assertNoRuntimeSourcePaths(text, privateUploadRoots, "Runtime source or workspace paths must never be persisted in artifact uploads.")
+  assertNoSeedSnapshotPaths(text)
   if (containsRuntimeSourceContent(text)) throw new Error("Prepared runtime plugin source contents must never be staged for artifact upload.")
   await mkdir(resolve(destination, ".."), { recursive: true })
   await writeFile(destination, text)
@@ -163,7 +214,8 @@ async function finalScan(directory) {
     else if (entry.isFile()) {
       const bytes = await readFile(path)
       const text = isUtf8(bytes) ? bytes.toString("utf8") : ""
-      assertNoRuntimeSourcePaths(text, runtimeSourceRoots, "Runtime source paths must never be persisted in artifact uploads.")
+      assertNoRuntimeSourcePaths(text, privateUploadRoots, "Runtime source or workspace paths must never be persisted in artifact uploads.")
+      assertNoSeedSnapshotPaths(text)
       if (containsRuntimeSourceContent(text)) throw new Error("Prepared runtime plugin source contents must never be persisted in artifact uploads.")
     } else throw new Error("Only regular files may be persisted in artifact uploads.")
   }

.github/scripts/run-agent-task/runner-workspace-seed-snapshot.mjs

@@ -3,8 +3,11 @@ import { chmod, copyFile, lstat, mkdir, mkdtemp, readdir, readFile, rm } from "n
 import { tmpdir } from "node:os"
 import { join, relative, resolve } from "node:path"
 
-export const RUNNER_WORKSPACE_SEED_EXCLUDES = [".git/**", ".codebox/**", "node_modules/**", "vendor/**", "dist/**", "build/**", "coverage/**", ".cache/**"]
-const EXCLUDED_ROOT_NAMES = new Set(RUNNER_WORKSPACE_SEED_EXCLUDES.map((pattern) => pattern.slice(0, -3)))
+// Snapshot policy is intentionally allow-by-exception: disposable build trees and
+// credential-bearing files never enter the agent-readable workspace. `.env.example`
+// is the one documented environment-template exception.
+export const RUNNER_WORKSPACE_SEED_EXCLUDES = [".git/**", ".codebox/**", "node_modules/**", "vendor/**", "dist/**", "build/**", "coverage/**", ".cache/**", ".env", ".env.*", ".npmrc", ".yarnrc.yml", ".pypirc", ".netrc", "auth.json", "id_rsa", "id_ed25519", "*.pem", "*.key", "credential files"]
+const EXCLUDED_ROOT_NAMES = new Set(RUNNER_WORKSPACE_SEED_EXCLUDES.filter((pattern) => pattern.endsWith("/**")).map((pattern) => pattern.slice(0, -3)))
 const MAX_FILES = 10_000
 const MAX_BYTES = 256 * 1024 * 1024
 
@@ -18,22 +21,40 @@ function fileMode(stat) {
   return stat.mode & 0o111 ? 0o755 : 0o644
 }
 
+function secretCategory(path) {
+  const name = path.split("/").at(-1)?.toLowerCase() || ""
+  if ((name === ".env" || name.startsWith(".env.")) && name !== ".env.example") return "environment"
+  if ([".npmrc", ".yarnrc.yml", ".pypirc", ".netrc", "auth.json", "credentials", "credentials.json", "secrets.json", "token.json"].includes(name)) return "credentials"
+  if (["id_rsa", "id_ed25519", "id_ecdsa", "id_dsa"].includes(name) || /\.(?:pem|key|p12|pfx)$/i.test(name)) return "private-key"
+  return ""
+}
+
 export async function createRunnerWorkspaceSeedSnapshot(source) {
   const sourceRoot = resolve(source)
   const root = await mkdtemp(join(tmpdir(), "wp-codebox-runner-workspace-seed-"))
   const digest = createHash("sha256")
   let fileCount = 0
   let byteCount = 0
+  const excluded = new Map()
+  const exclude = (category) => excluded.set(category, (excluded.get(category) || 0) + 1)
 
   try {
     async function copyTree(currentSource, currentTarget) {
       const entries = await readdir(currentSource, { withFileTypes: true })
       for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
-        if (EXCLUDED_ROOT_NAMES.has(entry.name)) continue
+        if (EXCLUDED_ROOT_NAMES.has(entry.name)) {
+          exclude("generated-tree")
+          continue
+        }
         const input = join(currentSource, entry.name)
         const output = join(currentTarget, entry.name)
         const stat = await lstat(input)
         const path = relative(sourceRoot, input).replaceAll("\\", "/")
+        const category = secretCategory(path)
+        if (category) {
+          exclude(category)
+          continue
+        }
         if (stat.isSymbolicLink() || (!stat.isDirectory() && !stat.isFile())) {
           throw snapshotError(`Runner workspace seed contains unsupported filesystem entry: ${path}`)
         }
@@ -67,6 +88,10 @@ export async function createRunnerWorkspaceSeedSnapshot(source) {
         files: fileCount,
         bytes: byteCount,
         excludes: RUNNER_WORKSPACE_SEED_EXCLUDES,
+        excluded: {
+          files: [...excluded.values()].reduce((total, count) => total + count, 0),
+          categories: [...excluded.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([category, count]) => ({ category, count })),
+        },
       },
     }
   } catch (error) {

docs/runner-workspace-seed-policy.md

@@ -0,0 +1,5 @@
+# Runner Workspace Seed Policy
+
+Runner workspaces are copied from a bounded, disposable snapshot. The snapshot excludes generated trees and credential material before an agent can read it: `.env` and `.env.*` except `.env.example`, `.npmrc`, `.yarnrc.yml` (it can carry registry auth), `.pypirc`, `.netrc`, `auth.json`, common credential files, SSH private-key names, and private-key extensions.
+
+Snapshot provenance records only a digest, copied file and byte counts, the policy patterns, and aggregate exclusion categories/counts. It never records excluded path names or the temporary snapshot location.

package.json

@@ -85,7 +85,7 @@
     "generate:fanout-aggregation-contract": "tsx scripts/generate-fanout-aggregation-contract-fixture.ts",
     "smoke": "tsx scripts/run-smoke.ts",
     "test:redaction": "tsx tests/redaction.test.ts",
-    "test:agent-task-contracts": "tsx tests/agent-task-contracts.test.ts && npm run test:agent-task-workflow-interface && npm run test:runtime-sources-materialization && tsx tests/agent-task-reusable-workflow.test.ts",
+    "test:agent-task-contracts": "tsx tests/agent-task-contracts.test.ts && tsx tests/agent-task-canonical-evidence.test.ts && npm run test:agent-task-workflow-interface && npm run test:runtime-sources-materialization && tsx tests/agent-task-reusable-workflow.test.ts",
     "test:agent-task-workflow-interface": "tsx tests/run-agent-task-reusable-workflow-interface.test.ts",
     "test:agent-task-runtime-package-staging": "tsx tests/agent-task-runtime-package-staging.test.ts",
     "test:external-native-package-materialization": "tsx tests/external-native-package-materialization.test.ts",

packages/cli/src/commands/agent-task-run.ts

@@ -1,7 +1,7 @@
-import { lstat, mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises"
+import { lstat, mkdir, mkdtemp, readFile, realpath, rename, rm, writeFile } from "node:fs/promises"
 import { tmpdir } from "node:os"
-import { dirname, join } from "node:path"
-import { AGENT_TASK_RUN_REQUEST_SCHEMA, HEADLESS_AGENT_TASK_REQUEST_SCHEMA, artifactResultEnvelope, buildAgentTaskRecipe, DEFAULT_WORDPRESS_VERSION, headlessAgentTaskRequestToRunInput, normalizeAgentRuntimeExecutionChanges, normalizeAgentRuntimeWorkload, normalizeAgentTaskRunResult, normalizeAgentTerminalResult, normalizeArtifactResultTypedArtifacts, normalizeHeadlessAgentTaskRequest, normalizeHeadlessAgentTaskResult, normalizeTaskInput, parseCommandJson, parseCommandOptions, resolveEffectiveRuntimeToolPolicy, type AgentTaskRunInput, type AgentTaskRunResultSummary, type AgentTerminalResult, type ArtifactResultEnvelope, type HeadlessAgentTaskResult, type SandboxToolPolicySnapshot, type TypedArtifactDTO } from "@automattic/wp-codebox-core"
+import { dirname, isAbsolute, join, relative, resolve } from "node:path"
+import { AGENT_TASK_RUN_REQUEST_SCHEMA, HEADLESS_AGENT_TASK_REQUEST_SCHEMA, artifactResultEnvelope, buildAgentTaskRecipe, DEFAULT_WORDPRESS_VERSION, headlessAgentTaskRequestToRunInput, normalizeAgentRuntimeExecutionChanges, normalizeAgentRuntimeWorkload, normalizeAgentTaskRunResult, normalizeAgentTerminalResult, normalizeArtifactResultTypedArtifacts, normalizeHeadlessAgentTaskRequest, normalizeHeadlessAgentTaskResult, normalizeTaskInput, parseCommandJson, parseCommandOptions, publicArtifactRefGroups, resolveEffectiveRuntimeToolPolicy, type AgentTaskRunInput, type AgentTaskRunResultSummary, type AgentTerminalResult, type ArtifactResultEnvelope, type HeadlessAgentTaskResult, type SandboxToolPolicySnapshot, type TypedArtifactDTO } from "@automattic/wp-codebox-core"
 import { stripUndefined } from "@automattic/wp-codebox-core/internals"
 import { runRecipeRunCommand } from "./recipe-run.js"
 
@@ -168,7 +168,7 @@ export async function runAgentTask(input: AgentTaskRunInput, options: AgentTaskR
     const agentBundle = objectValue(input.agent_bundle) || {}
     const metadataRuntime = objectValue(objectValue(run.metadata)?.agent_runtime)
     const agentResult = objectValue(run.agentResult) || objectValue(runRecord.agentResult) || objectValue(artifactsRecord.agentResult) || {}
-    const executionChanges = normalizeAgentRuntimeExecutionChanges(agentResult)
+    const executionChanges = await validatedExecutionChanges(agentResult, artifacts)
     const workload = normalizeAgentRuntimeWorkload(metadataRuntime?.workload ?? run, {
       requiredOutputs: stringRecord(agentBundle.engine_data_outputs),
       toolRecorders: agentBundle.tool_recorders,
@@ -715,6 +715,70 @@ function artifactResultDiagnostics(diagnostics: Array<Record<string, unknown>>):
   }))
 }
 
+const MAX_CANONICAL_EXECUTION_ARTIFACT_BYTES = 5 * 1024 * 1024
+
+/**
+ * Counters are runner diagnostics, not proof of a change. Only captured canonical
+ * artifacts under the recipe's trusted root may satisfy semantic output policy.
+ */
+export async function validatedExecutionChanges(agentResult: Record<string, unknown>, artifactRoot: string) {
+  const counters = normalizeAgentRuntimeExecutionChanges(agentResult)
+  if (!counters || counters.changed_files_count <= 0 || counters.patch_bytes <= 0) return undefined
+
+  const groups = publicArtifactRefGroups(agentResult)
+  // Recipe capture exposes the canonical descriptors as changedFiles/patch; some
+  // runners additionally project them into public refs. Treat either shape as a
+  // single reference, never a counter-only substitute.
+  const directChanged = objectValue(agentResult.changedFiles)
+  const directPatch = objectValue(agentResult.patch)
+  const changedRefs = groups.changed_files.length > 0 ? groups.changed_files : (stringValue(directChanged?.artifact) ? [{ path: stringValue(directChanged?.artifact), sha256: stringValue(directChanged?.sha256) }] : [])
+  const patchRefs = groups.patches.length > 0 ? groups.patches : (stringValue(directPatch?.artifact) ? [{ path: stringValue(directPatch?.artifact), sha256: stringValue(directPatch?.sha256) }] : [])
+  if (changedRefs.length !== 1 || patchRefs.length !== 1) return undefined
+  const [changedRef] = changedRefs
+  const [patchRef] = patchRefs
+  if (!changedRef?.path || !patchRef?.path) return undefined
+
+  try {
+    const root = await realpath(resolve(artifactRoot))
+    const bundleDirectory = stringValue(objectValue(agentResult.artifacts)?.directory)
+    const base = bundleDirectory ? await trustedArtifactPath(root, bundleDirectory, true) : root
+    const [changedPath, patchPath] = await Promise.all([trustedArtifactPath(base, changedRef.path), trustedArtifactPath(base, patchRef.path)])
+    const [changedText, patch] = await Promise.all([readBoundedArtifact(changedPath), readBoundedArtifact(patchPath)])
+    const changed = objectValue(JSON.parse(changedText))
+    const files = Array.isArray(changed?.files) ? changed.files : []
+    const paths = files.map((file) => stringValue(objectValue(file)?.relativePath)).filter(Boolean)
+    if (changed?.schema !== "wp-codebox/changed-files/v1" || paths.length === 0 || paths.length !== counters.changed_files_count || !patch.trim() || Buffer.byteLength(patch) !== counters.patch_bytes) return undefined
+    const patchPaths = patch.split("\n").flatMap((line) => {
+      if (!line.startsWith("--- ") && !line.startsWith("+++ ")) return []
+      const path = line.slice(4).split("\t", 1)[0].trim().replace(/^[ab]\//, "")
+      return path === "/dev/null" ? [] : [path]
+    })
+    if (patchPaths.length === 0 || paths.some((path) => !patchPaths.includes(path)) || patchPaths.some((path) => !paths.includes(path))) return undefined
+    return normalizeAgentRuntimeExecutionChanges({
+      changedFiles: { count: paths.length, artifact: changedRef.path, bytes: Buffer.byteLength(changedText), sha256: changedRef.sha256 },
+      patch: { bytes: Buffer.byteLength(patch), artifact: patchRef.path, sha256: patchRef.sha256 },
+    })
+  } catch {
+    return undefined
+  }
+}
+
+async function trustedArtifactPath(root: string, artifactPath: string, directory = false): Promise<string> {
+  const candidate = isAbsolute(artifactPath) ? resolve(artifactPath) : resolve(root, artifactPath)
+  const stat = await lstat(candidate)
+  if ((!directory && !stat.isFile()) || (directory && !stat.isDirectory()) || stat.isSymbolicLink() || (!directory && stat.size > MAX_CANONICAL_EXECUTION_ARTIFACT_BYTES)) throw new Error("Canonical execution artifact must be a bounded regular file.")
+  const resolved = await realpath(candidate)
+  const contained = relative(root, resolved)
+  if (resolved !== root && (contained === ".." || contained.startsWith(`..${String.fromCharCode(47)}`) || isAbsolute(contained))) throw new Error("Canonical execution artifact escapes the trusted artifact root.")
+  return resolved
+}
+
+async function readBoundedArtifact(path: string): Promise<string> {
+  const bytes = await readFile(path)
+  if (bytes.length > MAX_CANONICAL_EXECUTION_ARTIFACT_BYTES || bytes.includes(0)) throw new Error("Canonical execution artifact must be bounded text.")
+  return bytes.toString("utf8")
+}
+
 async function readJsonRecord(path: string): Promise<Record<string, unknown> | undefined> {
   if (!path) return undefined
   try {