Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/scripts/run-agent-task/execute-native-agent-task.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -590,9 +590,9 @@ if (execution.code === 0 && runtimeResult.success === true && request.runner_wor
const publicCore = await import(pathToFileURL(join(codeboxRoot, "packages/runtime-core/dist/public.js")).href)
const refs = publicCore.normalizePublicArtifactRefDTOs(runtimeResult)
.filter((ref) => ref.kind === "codebox-patch" || ref.kind === "codebox-changed-files")
workspaceApply = await runnerWorkspaceCore.applyRunnerWorkspacePatch({ artifactRoot: artifactsPath, artifactRefs: refs, workspaceRoot: workspace, writablePaths })
workspaceApply = await runnerWorkspaceCore.applyRunnerWorkspacePatch({ artifactRoot: artifactsPath, artifactRefs: refs, workspaceRoot: workspace, writablePaths, seedIdentity: runnerWorkspaceSeedSnapshot?.provenance.identity })
} catch (error) {
downstreamFailure = { stage: "apply", message: bounded(error instanceof Error ? error.message : String(error), MAX_OUTPUT_CHARS) }
downstreamFailure = { stage: "apply", message: bounded(error instanceof Error ? error.message : String(error), MAX_OUTPUT_CHARS), ...(error?.evidence ? { evidence: error.evidence } : {}) }
}
}
runtimeResult = sanitizeRuntimeSourceValue(runtimeResult, runtimeSourceOutputRoots)
Expand Down
17 changes: 17 additions & 0 deletions .github/scripts/run-agent-task/prepare-agent-task-upload.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,22 @@ function projectWorkflowResult(value) {
}).filter(([, entry]) => entry !== undefined)))
}

async function stageApplyFailureEvidence(result) {
const evidence = record(record(result.failure).evidence)
const files = [
["patch", record(evidence.patch)],
["changed-files", record(evidence.changed_files)],
]
for (const [name, descriptor] of files) {
const artifactPath = safeRelativeArtifactPath(descriptor.artifact_path)
if (!artifactPath) continue
const source = resolve(artifactsPath, artifactPath)
if (relative(artifactsPath, source).startsWith("..")) continue
const destination = join(uploadPath, ".codebox", "agent-task-artifacts", "apply-failure", name === "patch" ? "rejected.patch" : "changed-files.json")
await stageTextFile(source, destination, { allowTargetCode: true })
}
}

function safeTargetPath(value) {
const path = safeRelativeArtifactPath(value)
return path ? `workspace/${path}` : undefined
Expand Down Expand Up @@ -469,6 +485,7 @@ await mkdir(uploadPath, { recursive: true })
await stageTextFile(requestPath, join(uploadPath, ".codebox", "agent-task-request.json"))
await stageTextFile(resultSource, join(uploadPath, ".codebox", "agent-task-workflow-result.json"), { projectWorkflowResult: true })
await stageTextFile(join(workspace, ".codebox", "native-agent-task-input.json"), join(uploadPath, ".codebox", "native-agent-task-input.json"), { compactNativeInput: true })
await stageApplyFailureEvidence(result)
for (const path of declaredPaths) {
const source = resolve(artifactsPath, path)
if (relative(artifactsPath, source).startsWith("..") || sourceCategory(path, source)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ import { createHash } from "node:crypto"
import { chmod, copyFile, lstat, mkdir, mkdtemp, readdir, readFile, rm } from "node:fs/promises"
import { tmpdir } from "node:os"
import { join, relative, resolve } from "node:path"
import { execFile } from "node:child_process"
import { promisify } from "node:util"

// Snapshot policy is intentionally allow-by-exception: disposable build trees and
// credential-bearing files never enter the agent-readable workspace. `.env.example`
Expand All @@ -10,6 +12,7 @@ export const RUNNER_WORKSPACE_SEED_EXCLUDES = [".git/**", ".codebox/**", "node_m
const EXCLUDED_ROOT_NAMES = new Set(RUNNER_WORKSPACE_SEED_EXCLUDES.filter((pattern) => pattern.endsWith("/**")).map((pattern) => pattern.slice(0, -3)))
const MAX_FILES = 10_000
const MAX_BYTES = 256 * 1024 * 1024
const execFileAsync = promisify(execFile)

function snapshotError(message) {
const error = new Error(message)
Expand Down Expand Up @@ -80,11 +83,17 @@ export async function createRunnerWorkspaceSeedSnapshot(source) {
const sourceStat = await lstat(sourceRoot)
if (!sourceStat.isDirectory() || sourceStat.isSymbolicLink()) throw snapshotError("Runner workspace seed source must be a real directory.")
await copyTree(sourceRoot, root)
const contentDigest = digest.digest("hex")
const head = await gitHead(sourceRoot)
return {
source: root,
provenance: {
schema: "wp-codebox/runner-workspace-seed-snapshot/v1",
digest: { sha256: digest.digest("hex") },
digest: { sha256: contentDigest },
identity: {
content_digest: { algorithm: "sha256", value: contentDigest },
...(head ? { git: { head } } : {}),
},
files: fileCount,
bytes: byteCount,
excludes: RUNNER_WORKSPACE_SEED_EXCLUDES,
Expand All @@ -99,3 +108,13 @@ export async function createRunnerWorkspaceSeedSnapshot(source) {
throw error
}
}

async function gitHead(source) {
try {
const { stdout } = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: source })
const head = stdout.trim()
return /^[a-f0-9]{40}$/i.test(head) ? head : ""
} catch {
return ""
}
}
87 changes: 85 additions & 2 deletions packages/runtime-core/src/runner-workspace-apply.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,22 @@ export interface RunnerWorkspaceApplyRequest {
artifactRefs: RunnerWorkspaceArtifactRef[]
workspaceRoot: string
writablePaths: string[]
seedIdentity?: RunnerWorkspaceSeedIdentity
verify?: () => Promise<void>
}

export interface RunnerWorkspaceSeedIdentity {
content_digest: { algorithm: "sha256"; value: string }
git?: { head: string }
}

export interface RunnerWorkspaceApplyFailureEvidence {
expected_identity?: RunnerWorkspaceSeedIdentity
actual_identity: RunnerWorkspaceSeedIdentity
patch: { artifact_path: string; sha256: string }
changed_files: { artifact_path: string; sha256: string }
}

export interface RunnerWorkspaceApplyResult {
schema: "wp-codebox/runner-workspace-apply-result/v1"
status: "applied" | "no-op"
Expand Down Expand Up @@ -72,15 +85,27 @@ export async function applyRunnerWorkspacePatch(request: RunnerWorkspaceApplyReq

const changed = parseChangedFiles(changedRaw)
validateChangedFiles(changed, request.writablePaths)
const failureEvidence = runnerWorkspaceFailureEvidence(request.seedIdentity, await runnerWorkspaceIdentity(workspaceRoot), artifactRoot, patchPath, patch, changedPath, changedRaw)
if (request.seedIdentity) {
if (!sameIdentity(request.seedIdentity, failureEvidence.actual_identity)) {
throw applyFailure("Runner workspace seed identity does not match the host workspace; refusing to apply patch.", {
...failureEvidence,
})
}
}
if (changed.length === 0) {
if (patch.trim()) throw new Error("Canonical patch is non-empty but changed-files declares no changes.")
return { schema: "wp-codebox/runner-workspace-apply-result/v1", status: "no-op", changedFiles: [] }
}
if (!patch.trim()) throw new Error("Canonical changed-files declares changes but patch is empty.")
validatePatchPaths(patch, changed)

await execGit(workspaceRoot, ["apply", "--check", "--whitespace=error", "--", patchPath])
await execGit(workspaceRoot, ["apply", "--whitespace=error", "--", patchPath])
try {
await execGit(workspaceRoot, ["apply", "--check", "--whitespace=error", "--", patchPath])
await execGit(workspaceRoot, ["apply", "--whitespace=error", "--", patchPath])
} catch (error) {
throw applyFailure(error instanceof Error ? error.message : String(error), failureEvidence)
}
const files = await snapshotWorkspace(workspaceRoot)
validateAppliedWorkspace(baseline, files, changed)
if (request.verify) await request.verify()
Expand All @@ -98,6 +123,36 @@ export async function applyRunnerWorkspacePatch(request: RunnerWorkspaceApplyReq
}
}

export async function runnerWorkspaceIdentity(root: string): Promise<RunnerWorkspaceSeedIdentity> {
const digest = createHash("sha256")
async function visit(directory: string): Promise<void> {
for (const entry of (await readdir(directory, { withFileTypes: true })).sort((left, right) => left.name.localeCompare(right.name))) {
if ([".git", ".codebox", "node_modules", "vendor", "dist", "build", "coverage", ".cache"].includes(entry.name)) continue
const absolute = resolve(directory, entry.name)
const path = relative(root, absolute).replaceAll("\\", "/")
const stat = await lstat(absolute)
if (excludedSeedFile(path)) continue
if (stat.isSymbolicLink() || (!stat.isDirectory() && !stat.isFile())) throw new Error(`Runner workspace contains an unsupported path type: ${path}`)
if (stat.isDirectory()) {
digest.update(`directory\0${path}\n`)
await visit(absolute)
continue
}
const bytes = await readFile(absolute)
digest.update(`file\0${path}\0${(stat.mode & 0o111 ? 0o755 : 0o644).toString(8)}\0${bytes.length}\n`)
digest.update(bytes)
}
}
await visit(root)
const identity: RunnerWorkspaceSeedIdentity = { content_digest: { algorithm: "sha256", value: digest.digest("hex") } }
try {
const { stdout } = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root, maxBuffer: 1024 })
const head = stdout.trim()
if (/^[a-f0-9]{40}$/i.test(head)) identity.git = { head }
} catch { /* A non-git workspace still has a content identity. */ }
return identity
}

/** Ensures checks did not mutate approved output or introduce unrelated files. */
export async function verifyRunnerWorkspaceIntegrity(snapshot: RunnerWorkspaceIntegritySnapshot): Promise<void> {
const current = await snapshotWorkspace(snapshot.workspaceRoot)
Expand Down Expand Up @@ -217,6 +272,34 @@ function validateAppliedWorkspace(baseline: RunnerWorkspacePublicationFile[], cu
}
}

function sameIdentity(expected: RunnerWorkspaceSeedIdentity, actual: RunnerWorkspaceSeedIdentity): boolean {
return expected.content_digest.algorithm === "sha256"
&& expected.content_digest.value === actual.content_digest.value
&& (!expected.git?.head || expected.git.head === actual.git?.head)
}

function runnerWorkspaceFailureEvidence(expected: RunnerWorkspaceSeedIdentity | undefined, actual: RunnerWorkspaceSeedIdentity, artifactRoot: string, patchPath: string, patch: string, changedPath: string, changedRaw: string): RunnerWorkspaceApplyFailureEvidence {
return {
...(expected ? { expected_identity: expected } : {}),
actual_identity: actual,
patch: { artifact_path: relative(artifactRoot, patchPath).replaceAll("\\", "/"), sha256: createHash("sha256").update(patch).digest("hex") },
changed_files: { artifact_path: relative(artifactRoot, changedPath).replaceAll("\\", "/"), sha256: createHash("sha256").update(changedRaw).digest("hex") },
}
}

function excludedSeedFile(path: string): boolean {
const name = path.split("/").at(-1)?.toLowerCase() ?? ""
return (name === ".env" || (name.startsWith(".env.") && name !== ".env.example"))
|| [".npmrc", ".yarnrc.yml", ".pypirc", ".netrc", "auth.json", "credentials", "credentials.json", "secrets.json", "token.json", "id_rsa", "id_ed25519", "id_ecdsa", "id_dsa"].includes(name)
|| /\.(?:pem|key|p12|pfx)$/i.test(name)
}

function applyFailure(message: string, evidence: RunnerWorkspaceApplyFailureEvidence): Error {
const error = new Error(message) as Error & { evidence?: RunnerWorkspaceApplyFailureEvidence }
error.evidence = evidence
return error
}

async function execGit(cwd: string, args: string[]): Promise<void> {
try {
await execFileAsync("git", args, { cwd, maxBuffer: MAX_PATCH_BYTES })
Expand Down
49 changes: 48 additions & 1 deletion tests/execute-native-agent-task-lifecycle.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@ import { chmod, mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises"
import { tmpdir } from "node:os"
import { join, resolve } from "node:path"
import { spawn } from "node:child_process"
import { execFile } from "node:child_process"
import { promisify } from "node:util"

const root = resolve(".")
const executor = join(root, ".github/scripts/run-agent-task/execute-native-agent-task.mjs")
const uploader = join(root, ".github/scripts/run-agent-task/prepare-agent-task-upload.mjs")
const execFileAsync = promisify(execFile)

async function run(mode = "success") {
const temp = await mkdtemp(join(tmpdir(), "wp-codebox-native-lifecycle-"))
Expand Down Expand Up @@ -61,19 +65,22 @@ async function run(mode = "success") {

const fakeCli = join(temp, "fake-cli.mjs")
const noOp = mode.startsWith("no-op")
const mismatch = mode === "mismatch"
const applyReject = mode === "apply-reject"
await writeFile(fakeCli, `
import { mkdir, readFile, writeFile } from "node:fs/promises"
import { join } from "node:path"
const output = process.argv[process.argv.indexOf("--result-file") + 1]
const input = JSON.parse(await readFile(process.argv[process.argv.indexOf("--input-file") + 1], "utf8"))
const root = join(process.cwd(), ".codebox", "agent-task-artifacts", "files")
await mkdir(root, { recursive: true })
const patch = ${noOp} ? "" : "--- a/README.md\\n+++ b/README.md\\n@@ -1 +1 @@\\n-before\\n+after\\n"
const patch = ${JSON.stringify(noOp ? "" : applyReject ? "--- a/README.md\n+++ b/README.md\n@@ -1 +1 @@\n-not-before\n+after\n" : "--- a/README.md\n+++ b/README.md\n@@ -1 +1 @@\n-before\n+after\n")}
await writeFile(join(root, "patch.diff"), patch)
await writeFile(join(root, "changed-files.json"), JSON.stringify({
schema: "wp-codebox/changed-files/v1",
files: ${noOp} ? [] : [{ path: "/workspace/README.md", relativePath: "README.md", status: "modified", beforeMode: "100644", afterMode: "100644" }],
}))
${mismatch ? "await writeFile(join(process.cwd(), \"README.md\"), \"diverged\\n\")" : ""}
await writeFile(join(process.cwd(), ".codebox", "order"), "runtime\\n")
const summary = {
schema: "wp-codebox/agent-task-run-result/v1",
Expand Down Expand Up @@ -190,4 +197,44 @@ const noOpMaintenance = await run("no-op-maintenance")
assert.equal(noOpMaintenance.code, 0)
assert(!noOpMaintenance.order.includes("publish"))

const mismatch = await run("mismatch")
assert.equal(mismatch.code, 1)
assert.equal(await readFile(join(mismatch.workspace, "README.md"), "utf8"), "diverged\n", "base mismatch fails before the rejected patch can mutate the host workspace")
assert.equal(mismatch.result.failure.stage, "apply")
assert.match(mismatch.result.failure.message, /seed identity does not match/)
assert.notEqual(mismatch.result.failure.evidence.expected_identity.content_digest.value, mismatch.result.failure.evidence.actual_identity.content_digest.value)
assert.equal(mismatch.result.failure.evidence.patch.artifact_path, "files/patch.diff")
assert.equal(mismatch.result.failure.evidence.changed_files.artifact_path, "files/changed-files.json")
await execFileAsync(process.execPath, [uploader], {
cwd: mismatch.workspace,
env: {
...process.env,
AGENT_TASK_WORKSPACE: mismatch.workspace,
AGENT_TASK_REQUEST_PATH: join(mismatch.workspace, ".codebox", "agent-task-request.json"),
AGENT_TASK_UPLOAD_PATH: join(mismatch.workspace, ".codebox", "agent-task-upload"),
},
})
assert.equal(await readFile(join(mismatch.workspace, ".codebox", "agent-task-upload", ".codebox", "agent-task-artifacts", "apply-failure", "rejected.patch"), "utf8"), "--- a/README.md\n+++ b/README.md\n@@ -1 +1 @@\n-before\n+after\n")
assert.match(await readFile(join(mismatch.workspace, ".codebox", "agent-task-upload", ".codebox", "agent-task-artifacts", "apply-failure", "changed-files.json"), "utf8"), /README\.md/)

const applyReject = await run("apply-reject")
assert.equal(applyReject.code, 1)
assert.equal(await readFile(join(applyReject.workspace, "README.md"), "utf8"), "before\n", "a rejected patch does not partially mutate the matching host workspace")
assert.equal(applyReject.result.failure.stage, "apply")
assert.match(applyReject.result.failure.message, /Host git apply failed/)
assert.deepEqual(applyReject.result.failure.evidence.expected_identity, applyReject.result.failure.evidence.actual_identity, "the failure records the matching seed and host identities")
assert.equal(applyReject.result.failure.evidence.patch.artifact_path, "files/patch.diff")
assert.equal(applyReject.result.failure.evidence.changed_files.artifact_path, "files/changed-files.json")
await execFileAsync(process.execPath, [uploader], {
cwd: applyReject.workspace,
env: {
...process.env,
AGENT_TASK_WORKSPACE: applyReject.workspace,
AGENT_TASK_REQUEST_PATH: join(applyReject.workspace, ".codebox", "agent-task-request.json"),
AGENT_TASK_UPLOAD_PATH: join(applyReject.workspace, ".codebox", "agent-task-upload"),
},
})
assert.match(await readFile(join(applyReject.workspace, ".codebox", "agent-task-upload", ".codebox", "agent-task-artifacts", "apply-failure", "rejected.patch"), "utf8"), /-not-before/)
assert.match(await readFile(join(applyReject.workspace, ".codebox", "agent-task-upload", ".codebox", "agent-task-artifacts", "apply-failure", "changed-files.json"), "utf8"), /README\.md/)

console.log("native agent task lifecycle ok")
13 changes: 10 additions & 3 deletions tests/execute-native-agent-task-playground-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ try {
await writeFile(join(workspace, "id_ed25519"), "PRIVATE_KEY_SENTINEL\n")
await writeFile(join(workspace, ".codebox", "private.txt"), "PRIVATE_CODEBOX_SENTINEL\n")
await writeFile(join(workspace, "node_modules", "private.txt"), "PRIVATE_NODE_MODULES_SENTINEL\n")
await execFileAsync("git", ["init", "--quiet"], { cwd: workspace })
await execFileAsync("git", ["config", "user.email", "tests@example.invalid"], { cwd: workspace })
await execFileAsync("git", ["config", "user.name", "Tests"], { cwd: workspace })
await execFileAsync("git", ["add", "."], { cwd: workspace })
await execFileAsync("git", ["commit", "--quiet", "-m", "seed"], { cwd: workspace })
const packageBytes = Buffer.from(`${JSON.stringify({ schema_version: 1, bundle_slug: "fixture-agent", agent: { agent_slug: "fixture-agent", agent_name: "Fixture Agent", description: "Playground imported-agent fixture.", agent_config: { instructions: "Read and update README.", enabled_tools: ["workspace_read", "workspace_edit"], modes: ["chat"] } } })}\n`)
await writeFile(packagePath, packageBytes)
await mkdir(interceptor, { recursive: true })
Expand Down Expand Up @@ -68,10 +73,12 @@ add_filter( 'pre_http_request', static function( $preempt, $args, $url ) {
assert.deepEqual(input.task_input.workspaces.map((entry: { target: string, mode: string }) => ({ target: entry.target, mode: entry.mode })), [{ target: "/workspace", mode: "readwrite" }])
assert.notEqual(seededWorkspace.seed.source, workspace)
const seedProvenance = input.task_input.runtime_task.input.metadata.runner_workspace_seed
assert.match(seedProvenance.digest.sha256, /^[a-f0-9]{64}$/)
assert.match(seedProvenance.digest.sha256, /^[a-f0-9]{64}$/)
assert.match(seedProvenance.identity.content_digest.value, /^[a-f0-9]{64}$/)
assert.match(seedProvenance.identity.git.head, /^[a-f0-9]{40}$/)
assert.equal(seedProvenance.files, 2, "only README and the explicit .env.example template are copied")
assert.equal(seedProvenance.excluded.files, 6)
assert.deepEqual(seedProvenance.excluded.categories, [{ category: "credentials", count: 2 }, { category: "environment", count: 1 }, { category: "generated-tree", count: 2 }, { category: "private-key", count: 1 }])
assert.equal(seedProvenance.excluded.files, 7)
assert.deepEqual(seedProvenance.excluded.categories, [{ category: "credentials", count: 2 }, { category: "environment", count: 1 }, { category: "generated-tree", count: 3 }, { category: "private-key", count: 1 }])
assert.equal(execution.code, 0, `${execution.stderr ?? ""}\n${JSON.stringify(result)}`)
assert.equal(result.success, true)
assert.equal(await readFile(join(workspace, "README.md"), "utf8"), "after\n")
Expand Down
Loading
Loading