Materialize immutable native runtime sources

Author: chubes4

.github/scripts/run-agent-task/build-codebox-task-request.mjs

@@ -1,5 +1,5 @@
 import { mkdirSync, writeFileSync, appendFileSync } from "node:fs"
-import { normalizeExternalPackageSource, parseExternalPackageSourcePolicy } from "./materialize-external-native-package.mjs"
+import { normalizeExternalPackageSource, normalizeRuntimeSources, parseExternalPackageSourcePolicy } from "./materialize-external-native-package.mjs"
 
 function parseJson(name, fallback, expected) {
   const raw = process.env[name]
@@ -64,9 +64,11 @@ function commandList(name) {
 }
 
 const artifactDeclarations = parseJson("ARTIFACT_DECLARATIONS", [], "array")
+const externalPackagePolicy = parseExternalPackageSourcePolicy(requiredString("EXTERNAL_PACKAGE_SOURCE_POLICY"))
 const request = {
   schema: "wp-codebox/agent-task-workflow-request/v1",
-  external_package_source: normalizeExternalPackageSource(parseJson("EXTERNAL_PACKAGE_SOURCE", {}, "object"), parseExternalPackageSourcePolicy(requiredString("EXTERNAL_PACKAGE_SOURCE_POLICY"))),
+  external_package_source: normalizeExternalPackageSource(parseJson("EXTERNAL_PACKAGE_SOURCE", {}, "object"), externalPackagePolicy),
+  runtime_sources: normalizeRuntimeSources(parseJson("RUNTIME_SOURCES", [], "array"), externalPackagePolicy),
   workload: {
     id: process.env.WORKLOAD_ID || "agent-task",
     label: process.env.WORKLOAD_LABEL || "Run Agent Task",

.github/scripts/run-agent-task/execute-native-agent-task.mjs

@@ -1,7 +1,8 @@
+import { rmSync } from "node:fs"
 import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises"
 import { join, resolve } from "node:path"
 import { spawn } from "node:child_process"
-import { materializeExternalNativePackage, normalizeExternalPackageSource, parseExternalPackageSourcePolicy } from "./materialize-external-native-package.mjs"
+import { materializeExternalNativePackage, materializeRuntimeSources, normalizeExternalPackageSource, normalizeRuntimeSources, parseExternalPackageSourcePolicy } from "./materialize-external-native-package.mjs"
 import { readNativeResult } from "./native-result-file.mjs"
 
 const requestPath = process.env.AGENT_TASK_REQUEST_PATH || ".codebox/agent-task-request.json"
@@ -189,6 +190,7 @@ const driftChecks = commandEntries(request.drift_checks, "drift_checks")
 const runId = `${request.workload?.id || "agent-task"}-${process.env.GITHUB_RUN_ID || "local"}`.replace(/[^A-Za-z0-9._-]+/g, "-")
 const externalPackagePolicy = parseExternalPackageSourcePolicy(string(process.env.EXTERNAL_PACKAGE_SOURCE_POLICY))
 const externalPackageSource = normalizeExternalPackageSource(request.external_package_source, externalPackagePolicy)
+const runtimeSources = normalizeRuntimeSources(request.runtime_sources ?? [], externalPackagePolicy)
 const artifactsPath = join(workspace, ".codebox", "agent-task-artifacts")
 const runtimeInputPath = join(workspace, ".codebox", "native-agent-task-input.json")
 const resultPath = join(workspace, ".codebox", "agent-task-workflow-result.json")
@@ -220,6 +222,14 @@ if (accessError) {
 const materializedPackage = request.run_agent && !request.dry_run
   ? await materializeExternalNativePackage(externalPackageSource, { policy: externalPackagePolicy })
   : undefined
+const materializedRuntimeSources = request.run_agent && !request.dry_run
+  ? await materializeRuntimeSources(runtimeSources, { policy: externalPackagePolicy })
+  : undefined
+process.once("exit", () => { if (materializedRuntimeSources?.root) rmSync(materializedRuntimeSources.root, { recursive: true, force: true }) })
+const runtimeSourceInputs = (materializedRuntimeSources?.lowered ?? []).reduce((input, lowered) => {
+  for (const [key, entries] of Object.entries(lowered)) input[key] = [...(input[key] ?? []), ...entries]
+  return input
+}, {})
 
 const taskInput = {
   schema: "wp-codebox/agent-task-run-request/v1",
@@ -235,6 +245,7 @@ const taskInput = {
     provider: request.model?.provider,
     model: request.model?.name,
     secret_env: ["OPENAI_API_KEY", "MODEL_PROVIDER_SECRET_1", "MODEL_PROVIDER_SECRET_2", "MODEL_PROVIDER_SECRET_3", "MODEL_PROVIDER_SECRET_4", "MODEL_PROVIDER_SECRET_5"].filter((name) => process.env[name]),
+    ...runtimeSourceInputs,
     allowed_tools: runnerWorkspaceTools,
     sandbox_tool_policy: {
       schema: "wp-codebox/sandbox-tool-policy/v1",
@@ -265,7 +276,7 @@ const taskInput = {
         artifact_declarations: request.artifacts?.declarations || [],
         required_artifacts: request.artifacts?.expected || [],
         output_projections: [],
-        metadata: { workload: request.workload, ...(materializedPackage ? { imported_agent: materializedPackage.identity } : {}) },
+        metadata: { workload: request.workload, ...(materializedPackage ? { imported_agent: materializedPackage.identity } : {}), runtime_sources: materializedRuntimeSources?.descriptors.map(({ metadata, ...provenance }) => ({ ...provenance, metadata })) ?? [] },
       },
     },
   },
@@ -285,6 +296,7 @@ const runtimeResult = request.run_agent && !request.dry_run
   ? await readNativeResult(nativeResultPath, controlledCodeboxPath, secretValues, redact)
   : {}
 await rm(nativeResultPath, { force: true })
+await rm(materializedRuntimeSources?.root ?? "", { recursive: true, force: true })
 
 await redactArtifactFiles(artifactsPath)
 

.github/scripts/run-agent-task/materialize-external-native-package.mjs

@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto"
-import { mkdtemp, rm } from "node:fs/promises"
+import { mkdtemp, rm, mkdir, writeFile } from "node:fs/promises"
 import { tmpdir } from "node:os"
 import { join } from "node:path"
 import { spawn } from "node:child_process"
@@ -10,6 +10,7 @@ const DIGEST = /^[0-9a-f]{64}$/i
 const DIGEST_SCHEME = "sha256-bytes-v1"
 const MAX_PACKAGE_BYTES = 1024 * 1024
 const SLUG = /^[a-z0-9]+(?:-[a-z0-9]+)*$/
+const ROLE = /^(component|provider_plugin|bundled_library)$/
 const string = (value) => typeof value === "string" ? value.trim() : ""
 
 export function canonicalExternalNativeAgentIdentity(bytes) {
@@ -38,6 +39,11 @@ function normalizePath(value) {
   return path
 }
 
+function normalizeRuntimePath(value) {
+  if (string(value) === ".") return "."
+  return normalizePath(value)
+}
+
 export function sha256BytesV1(bytes) {
   return `${DIGEST_SCHEME}:${createHash("sha256").update(bytes).digest("hex")}`
 }
@@ -96,8 +102,115 @@ export function parseExternalPackageSourcePolicy(raw) {
       return agentPath
     })
   }
-  if (Object.keys(repositories).length === 0) throw new Error("EXTERNAL_PACKAGE_SOURCE_POLICY must authorize at least one repository.")
-  return { version: 1, repositories }
+  const runtime_sources = normalizeRuntimeSourcePolicy(policy.runtime_sources)
+  if (Object.keys(repositories).length === 0 && Object.keys(runtime_sources).length === 0) throw new Error("EXTERNAL_PACKAGE_SOURCE_POLICY must authorize at least one repository.")
+  return { version: 1, repositories, runtime_sources }
+}
+
+function normalizeRuntimeSourcePolicy(value) {
+  if (value === undefined) return {}
+  if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("EXTERNAL_PACKAGE_SOURCE_POLICY runtime_sources must be a repositories mapping.")
+  const repositories = {}
+  for (const [repository, paths] of Object.entries(value)) {
+    const normalizedRepository = string(repository).toLowerCase()
+    if (!REPOSITORY.test(normalizedRepository) || !Array.isArray(paths) || paths.length === 0) throw new Error("EXTERNAL_PACKAGE_SOURCE_POLICY contains an invalid runtime source repository entry.")
+    repositories[normalizedRepository] = paths.map((path) => normalizeRuntimePath(string(path)))
+  }
+  return repositories
+}
+
+export function normalizeRuntimeSources(value, policy = {}) {
+  if (!Array.isArray(value)) throw new Error("runtime_sources must be a JSON array.")
+  return value.map((entry, index) => normalizeRuntimeSource(entry, policy, `runtime_sources[${index}]`))
+}
+
+export function normalizeRuntimeSource(value, policy = {}, label = "runtime_source") {
+  const source = value && typeof value === "object" && !Array.isArray(value) ? value : {}
+  if (source.version !== 1) throw new Error(`${label}.version must be 1.`)
+  const role = string(source.role)
+  const repository = string(source.repository).toLowerCase()
+  const revision = string(source.revision).toLowerCase()
+  const path = normalizeRuntimePath(string(source.path))
+  if (!ROLE.test(role)) throw new Error(`${label}.role must be component, provider_plugin, or bundled_library.`)
+  if (!REPOSITORY.test(repository)) throw new Error(`${label}.repository must be an OWNER/REPO identifier.`)
+  if (!COMMIT.test(revision)) throw new Error(`${label}.revision must be a full immutable 40-character commit SHA.`)
+  const allowedPaths = policy.runtime_sources?.[repository]
+  if (!Array.isArray(allowedPaths) || !allowedPaths.includes(path)) throw new Error("Runtime source repository or path is not authorized.")
+  const digest = source.digest === undefined || source.digest === "" ? undefined : normalizeRuntimeDigest(source.digest, label)
+  const metadata = source.metadata && typeof source.metadata === "object" && !Array.isArray(source.metadata) ? source.metadata : {}
+  const normalized = { version: 1, role, repository, revision, path, ...(digest ? { digest } : {}), metadata: normalizeRuntimeMetadata(role, metadata, label) }
+  return normalized
+}
+
+function normalizeRuntimeDigest(value, label) {
+  const [scheme, digest] = string(value).split(":", 2)
+  if (scheme !== "sha256-git-archive-v1" || !DIGEST.test(digest)) throw new Error(`${label}.digest must use sha256-git-archive-v1:<64 lowercase hexadecimal characters>.`)
+  return `${scheme}:${digest.toLowerCase()}`
+}
+
+function normalizeRuntimeMetadata(role, metadata, label) {
+  const slug = string(metadata.slug)
+  const pluginFile = string(metadata.pluginFile)
+  const activate = metadata.activate
+  if (role === "component") {
+    if (!SLUG.test(slug) || !["plugin", "mu-plugin"].includes(metadata.loadAs)) throw new Error(`${label}.metadata must declare a slug and loadAs plugin or mu-plugin for a component.`)
+    if (pluginFile && !normalizePath(pluginFile)) throw new Error(`${label}.metadata.pluginFile must be a safe relative path.`)
+    if (activate !== undefined && typeof activate !== "boolean") throw new Error(`${label}.metadata.activate must be boolean when provided.`)
+    return { slug, loadAs: metadata.loadAs, ...(pluginFile ? { pluginFile } : {}), ...(activate === undefined ? {} : { activate }) }
+  }
+  if (role === "provider_plugin") {
+    if (slug && !SLUG.test(slug)) throw new Error(`${label}.metadata.slug must be a stable plugin slug.`)
+    if (pluginFile && !normalizePath(pluginFile)) throw new Error(`${label}.metadata.pluginFile must be a safe relative path.`)
+    if (activate !== undefined && typeof activate !== "boolean") throw new Error(`${label}.metadata.activate must be boolean when provided.`)
+    return { ...(slug ? { slug } : {}), ...(pluginFile ? { pluginFile } : {}), ...(activate === undefined ? { activate: true } : { activate }) }
+  }
+  if (!SLUG.test(string(metadata.library)) || !SLUG.test(string(metadata.strategy))) throw new Error(`${label}.metadata must declare library and strategy for a bundled library.`)
+  const target = string(metadata.target)
+  if (target && (!target.startsWith("/") || target.split("/").includes(".."))) throw new Error(`${label}.metadata.target must be an absolute path without traversal.`)
+  return { library: string(metadata.library), strategy: string(metadata.strategy), ...(target ? { target } : {}) }
+}
+
+export function sha256GitArchiveV1(bytes) {
+  return `sha256-git-archive-v1:${createHash("sha256").update(bytes).digest("hex")}`
+}
+
+export async function materializeRuntimeSources(sources, options = {}) {
+  const descriptors = normalizeRuntimeSources(sources, options.policy)
+  const root = await mkdtemp(join(options.tempRoot ?? tmpdir(), "wp-codebox-runtime-sources-"))
+  try {
+    const lowered = []
+    for (const [index, descriptor] of descriptors.entries()) {
+      const checkout = join(root, `source-${index}`)
+      const source = join(checkout, "source")
+      const environment = publicGitEnvironment(root)
+      const remote = options.remotes?.[descriptor.repository] ?? canonicalPublicGithubRepositorySource(descriptor.repository)
+      await run("git", ["init", "--quiet", checkout], { env: environment })
+      await run("git", ["remote", "add", "origin", remote], { cwd: checkout, env: environment })
+      await run("git", ["-c", "credential.helper=", "-c", "http.extraHeader=", "fetch", "--depth=1", "origin", descriptor.revision], { cwd: checkout, env: environment })
+      const commit = (await run("git", ["rev-parse", "FETCH_HEAD^{commit}"], { cwd: checkout, env: environment })).toString("utf8").trim().toLowerCase()
+      if (commit !== descriptor.revision) throw new Error("Runtime source revision did not resolve to the requested immutable commit.")
+      const objectType = (await run("git", ["cat-file", "-t", `${descriptor.revision}:${descriptor.path}`], { cwd: checkout, env: environment })).toString("utf8").trim()
+      if (objectType !== "tree") throw new Error("Runtime source path must identify a directory.")
+      const entries = (await run("git", ["ls-tree", "-r", "-z", descriptor.revision, "--", descriptor.path], { cwd: checkout, env: environment })).toString("utf8").split("\0").filter(Boolean)
+      if (entries.length === 0 || entries.some((entry) => !/^100(?:644|755)\s+blob\s+[0-9a-f]{40}\t/.test(entry))) throw new Error("Runtime source must contain only regular files; symlinks and special files are rejected.")
+      const archive = await run("git", ["archive", "--format=tar", descriptor.revision, descriptor.path], { cwd: checkout, env: environment })
+      if (descriptor.digest && sha256GitArchiveV1(archive) !== descriptor.digest) throw new Error("Runtime source archive digest does not match the trusted descriptor.")
+      await mkdir(source, { recursive: true })
+      const archivePath = join(checkout, "source.tar")
+      await writeFile(archivePath, archive)
+      await run("tar", ["-xf", archivePath, "-C", source], { cwd: checkout, env: environment })
+      const materializedPath = join(source, descriptor.path)
+      lowered.push(lowerRuntimeSource(descriptor, materializedPath))
+    }
+    return { root, descriptors, lowered }
+  } catch (error) { await rm(root, { recursive: true, force: true }); throw error }
+}
+
+export function lowerRuntimeSource(descriptor, source) {
+  const provenance = { role: descriptor.role, repository: descriptor.repository, revision: descriptor.revision, path: descriptor.path, ...(descriptor.digest ? { digest: descriptor.digest } : {}) }
+  if (descriptor.role === "component") return { component_contracts: [{ path: source, ...descriptor.metadata, metadata: { runtime_source: provenance } }] }
+  if (descriptor.role === "provider_plugin") return { provider_plugin_paths: [source], provider_plugins: [{ source, ...descriptor.metadata, metadata: { runtime_source: provenance } }] }
+  return { runtime_overlays: [{ kind: "bundled-library", source, ...descriptor.metadata, metadata: { runtime_source: provenance } }] }
 }
 
 function run(command, args, options = {}) {

.github/workflows/agent-task-contracts.yml

@@ -13,6 +13,7 @@ on:
       - "package-lock.json"
       - "package.json"
       - "tests/agent-task-*.test.ts"
+      - "tests/runtime-sources-materialization.test.ts"
       - "tests/redaction.test.ts"
       - "tests/production-boundary-enforcement.test.ts"
       - "tests/runtime-tool-policy.test.ts"
@@ -28,6 +29,7 @@ on:
       - "package-lock.json"
       - "package.json"
       - "tests/agent-task-*.test.ts"
+      - "tests/runtime-sources-materialization.test.ts"
       - "tests/redaction.test.ts"
       - "tests/production-boundary-enforcement.test.ts"
       - "tests/runtime-tool-policy.test.ts"

.github/workflows/run-agent-task.yml

@@ -11,6 +11,10 @@ name: Run Agent Task (reusable)
         description: "JSON descriptor for exactly one standalone .agent.json file: repository, immutable revision, file path, and sha256-bytes-v1 digest."
         type: string
         required: true
+      runtime_sources:
+        description: Versioned JSON array of trusted immutable public runtime source descriptors.
+        type: string
+        default: '[]'
       workload_id:
         description: Stable workload identifier for this run.
         type: string
@@ -219,6 +223,7 @@ jobs:
         id: plan
         env:
           EXTERNAL_PACKAGE_SOURCE: ${{ inputs.external_package_source }}
+          RUNTIME_SOURCES: ${{ inputs.runtime_sources }}
           EXTERNAL_PACKAGE_SOURCE_POLICY: ${{ secrets.EXTERNAL_PACKAGE_SOURCE_POLICY }}
           WORKLOAD_ID: ${{ inputs.workload_id }}
           WORKLOAD_LABEL: ${{ inputs.workload_label }}

contracts/agent-task-workflow-request.fixture.json

@@ -6,6 +6,7 @@
     "path": "packages/example-agent.agent.json",
     "digest": "sha256-bytes-v1:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
   },
+  "runtime_sources": [],
   "workload": {
     "id": "example-maintenance",
     "label": "Run example maintenance",

contracts/run-agent-task-reusable-workflow-interface.v1.json

@@ -4,6 +4,7 @@
   "inputs": {
     "wp_codebox_release_ref": { "required": true, "type": "string" },
     "external_package_source": { "required": true, "type": "string" },
+    "runtime_sources": { "required": false, "type": "string", "default": "[]" },
     "workload_id": { "required": false, "type": "string", "default": "agent-task" },
     "workload_label": { "required": false, "type": "string", "default": "Run Agent Task" },
     "component_id": { "required": false, "type": "string", "default": "agent-task" },

docs/agent-task-reusable-workflow.md

@@ -55,6 +55,8 @@ This release-coherence contract fixes [#1759](https://github.com/Automattic/wp-c
 - `wp_codebox_release_ref`: required exact immutable WP Codebox release tag in `vX.Y.Z` form.
 - `external_package_source`: immutable descriptor with `repository`, full commit `revision`, one package-relative `.agent.json` `path`, and `digest`. Packages are supported only from publicly accessible GitHub repositories, fetched from canonical `https://github.com/OWNER/REPOSITORY.git` without credentials. `digest` is exactly `sha256-bytes-v1:<lowercase-sha256>` over the raw file bytes; filenames and JSON content are UTF-8-safe and are not normalized before hashing.
 - `EXTERNAL_PACKAGE_SOURCE_POLICY`: required reusable-workflow secret, supplied by the caller's operator-controlled secret configuration. Its strict version 1 JSON shape is `{"version":1,"repositories":{"owner/repository":["agents/example.agent.json"]}}`. Every entry is an exact standalone `.agent.json` path. The policy is validated in runner memory, is never part of task input, and is not uploaded.
+
+`runtime_sources` is an optional versioned JSON array for immutable public runtime dependencies. Each descriptor declares `version: 1`, `role` (`component`, `provider_plugin`, or `bundled_library`), `repository`, a full 40-character `revision`, exact directory `path`, optional `sha256-git-archive-v1` digest, and role metadata. The operator-controlled policy extends the same secret with `runtime_sources`, for example `{"version":1,"repositories":{},"runtime_sources":{"owner/repository":["plugins/example"]}}`. The runner uses unauthenticated Git transport, verifies the commit, tree type, regular-file-only contents, and optional digest, stages outside the target workspace, and removes the staging root after execution. Runtime source descriptors lower into existing component, provider plugin, and runtime overlay inputs; artifacts retain provenance only.
 - `target_repo`: `OWNER/REPO` target repository.
 - `prompt`, `writable_paths`, provider/model, `max_turns`, `time_budget_ms`, callback data, and artifact declarations: native task inputs.
 - `runner_workspace`: JSON runner-workspace publication request owned by the package.