Skip to content

Commit bf9fb92

Browse files
committed
Bound retained recipe command evidence
1 parent 95a3778 commit bf9fb92

10 files changed

Lines changed: 418 additions & 14 deletions

File tree

.github/workflows/agent-task-contracts.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ jobs:
9696
- run: npm run test:native-agent-task-playground-e2e
9797
- run: npm run test:native-agent-task-interruption
9898
- run: npm run test:trusted-apply-artifact-channel
99+
- run: npm run test:runtime-command-artifact-bounds
99100
- run: npm run test:redaction
100101
- run: npm run test:production-boundary-enforcement
101102
- run: npm run test:runtime-tool-policy

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,7 @@
154154
"test:workspace-preload-artifacts": "tsx tests/workspace-preload-artifacts.test.ts",
155155
"test:runner-workspace-apply": "tsx tests/runner-workspace-apply.test.ts",
156156
"test:trusted-apply-artifact-channel": "tsx tests/trusted-apply-artifact-channel.integration.test.ts",
157+
"test:runtime-command-artifact-bounds": "tsx tests/runtime-command-artifact-bounds.test.ts",
157158
"test:runner-workspace-publisher": "node tests/runner-workspace-publisher.test.mjs",
158159
"test:native-agent-task-lifecycle": "node tests/execute-native-agent-task-lifecycle.test.mjs",
159160
"test:native-agent-task-interruption": "node tests/execute-native-agent-task-interruption.test.mjs",

packages/cli/src/commands/recipe-run-output.ts

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { setTimeout as delay } from "node:timers/promises"
2-
import type { RecipeRunSummary, RuntimeRunRecord } from "@automattic/wp-codebox-core"
2+
import type { ExecutionResult, RecipeRunSummary, RuntimeRunRecord } from "@automattic/wp-codebox-core"
3+
import { boundedExecutionResultsForArtifacts } from "@automattic/wp-codebox-core/internals"
34
import { serializeError } from "../output.js"
45
import type { RunOutput } from "../runtime-command-wrappers.js"
56
import { RecipePhaseError } from "./recipe-run-phases.js"
@@ -95,7 +96,14 @@ function hasTerminalRecipePhaseFailure(output: RecipeRunOutput): boolean {
9596
}
9697

9798
export async function writeRecipeJsonOutput(output: unknown): Promise<void> {
98-
await writeStdout(`${JSON.stringify(output, null, 2)}\n`)
99+
await writeStdout(`${JSON.stringify(boundedRecipeJsonOutput(output), null, 2)}\n`)
100+
}
101+
102+
export function boundedRecipeJsonOutput(output: unknown): unknown {
103+
if (!output || typeof output !== "object" || Array.isArray(output)) return output
104+
const record = output as Record<string, unknown>
105+
if (!Array.isArray(record.executions)) return output
106+
return { ...record, executions: boundedExecutionResultsForArtifacts(record.executions as ExecutionResult[]) }
99107
}
100108

101109
export async function writeRecipeSummaryHumanOutput(summary: RecipeRunSummary): Promise<void> {

packages/cli/src/commands/recipe-run.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
327327
await awaitRecipe("runtime.observe:runtime-info", runtime!.observe({ type: "runtime-info" }))
328328
await awaitRecipe("runtime.observe:mounts", runtime!.observe({ type: "mounts" }))
329329
runRecord = await runRegistry.update(runRecord.runId, { status: "collecting_artifacts", runtime: await runtime!.info() })
330-
artifacts = await awaitRecipe("runtime.collect-artifacts", collectRecipeRuntimeArtifacts(runtime!, { includeLogs: true, includeObservations: true }, { snapshotTimeoutMs: SUCCESSFUL_RECIPE_RUNTIME_SNAPSHOT_TIMEOUT_MS }))
330+
artifacts = await awaitRecipe("runtime.collect-artifacts", collectRecipeRuntimeArtifacts(runtime!, { includeLogs: true, includeObservations: true }, { snapshotTimeoutMs: SUCCESSFUL_RECIPE_RUNTIME_SNAPSHOT_TIMEOUT_MS, activeExecution: executions.at(-1) }))
331331
browserEvidence = await recipeBrowserEvidence(artifacts, executions, recipe)
332332
await artifactPointer.update({ runtime: await runtime!.info(), artifacts, phases: phaseTracker.list(), browserEvidence })
333333
await materializeTypedRecipeDeclaredArtifacts(artifacts, declaredArtifacts)
@@ -350,7 +350,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
350350
const recipeFailure = strictFailure ?? agentFailure ?? verifyFailure
351351
const successfulRecipe = !recipeFailure
352352
if (successfulRecipe && options.previewHoldSeconds) {
353-
artifacts = await awaitRecipe("runtime.collect-artifacts.preview-hold", collectRecipeRuntimeArtifacts(runtime, { includeLogs: true, includeObservations: true, previewHoldSeconds: options.previewHoldSeconds }, { snapshotTimeoutMs: SUCCESSFUL_RECIPE_RUNTIME_SNAPSHOT_TIMEOUT_MS }))
353+
artifacts = await awaitRecipe("runtime.collect-artifacts.preview-hold", collectRecipeRuntimeArtifacts(runtime, { includeLogs: true, includeObservations: true, previewHoldSeconds: options.previewHoldSeconds }, { snapshotTimeoutMs: SUCCESSFUL_RECIPE_RUNTIME_SNAPSHOT_TIMEOUT_MS, activeExecution: executions.at(-1) }))
354354
browserEvidence = await recipeBrowserEvidence(artifacts, executions, recipe)
355355
await artifactPointer.update({ runtime: await runtime.info(), artifacts, phases: phaseTracker.list(), browserEvidence })
356356
declaredArtifacts = await collectRecipeDeclaredArtifacts(recipe, runtime)

packages/cli/src/recipe-evidence.ts

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,31 @@ export interface RecipeArtifactsFinalizationController {
301301
interface RecipeRuntimeArtifactCollectionOptions {
302302
timeoutMs?: number
303303
snapshotTimeoutMs?: number
304+
activeExecution?: RecipeEvidenceExecutionResult
305+
}
306+
307+
export class RecipeRuntimeArtifactCollectionError extends Error {
308+
readonly code = "recipe-artifact-collection-failed"
309+
readonly operation = "runtime.collect-artifacts"
310+
readonly subsystem = "artifact-bundle"
311+
readonly workflowStepIndex?: number
312+
readonly command?: string
313+
readonly artifactPath?: string
314+
readonly payload?: unknown
315+
readonly limits?: unknown
316+
readonly causeStack?: string
317+
318+
constructor(activeExecution: RecipeEvidenceExecutionResult | undefined, cause: Error) {
319+
const causeRecord = cause as Error & Record<string, unknown>
320+
super(`Recipe artifact collection failed${activeExecution?.recipeStepIndex === undefined ? "" : ` at workflow step ${activeExecution.recipeStepIndex}`}: ${cause.message}`, { cause })
321+
this.name = "RecipeRuntimeArtifactCollectionError"
322+
this.workflowStepIndex = activeExecution?.recipeStepIndex
323+
this.command = activeExecution?.recipeCommand ?? activeExecution?.command
324+
this.artifactPath = typeof causeRecord.artifactPath === "string" ? causeRecord.artifactPath : undefined
325+
this.payload = causeRecord.payload
326+
this.limits = causeRecord.limits
327+
this.causeStack = typeof causeRecord.causeStack === "string" ? causeRecord.causeStack : cause.stack?.slice(0, 16 * 1024)
328+
}
304329
}
305330

306331
const execFileAsync = promisify(execFile)
@@ -322,7 +347,7 @@ export async function collectAndFinalizeFailedRecipeArtifacts(args: {
322347

323348
if (!artifacts) {
324349
try {
325-
artifacts = await collectRecipeRuntimeArtifacts(args.runtime, { includeLogs: true, includeObservations: true }, { snapshotTimeoutMs: 20_000, timeoutMs: 30_000 })
350+
artifacts = await collectRecipeRuntimeArtifacts(args.runtime, { includeLogs: true, includeObservations: true }, { snapshotTimeoutMs: 20_000, timeoutMs: 30_000, activeExecution: args.executions.at(-1) })
326351
} catch {
327352
return undefined
328353
}
@@ -354,11 +379,19 @@ export async function collectRecipeRuntimeArtifacts(runtime: Runtime, spec: Arti
354379
}
355380
}
356381

357-
const artifacts = runtime.collectArtifacts(spec)
358-
if (options.timeoutMs && options.timeoutMs > 0) {
359-
return timeoutOrReject(artifacts, options.timeoutMs, `Runtime artifact collection exceeded ${options.timeoutMs}ms`)
382+
try {
383+
const artifacts = runtime.collectArtifacts(spec)
384+
if (options.timeoutMs && options.timeoutMs > 0) {
385+
return await timeoutOrReject(artifacts, options.timeoutMs, `Runtime artifact collection exceeded ${options.timeoutMs}ms`)
386+
}
387+
return await artifacts
388+
} catch (error) {
389+
const candidate = error instanceof Error ? error as Error & Record<string, unknown> : undefined
390+
if (candidate?.code === "command-artifact-collection-failed" || error instanceof RangeError) {
391+
throw new RecipeRuntimeArtifactCollectionError(options.activeExecution, candidate ?? Object.assign(new Error(String(error)), { code: "runtime-allocation-failed" }))
392+
}
393+
throw error
360394
}
361-
return artifacts
362395
}
363396

364397
async function timeoutOrUndefined<T>(promise: Promise<T>, timeoutMs: number): Promise<T | undefined> {
Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
import type { ExecutionResult } from "./runtime-contracts.js"
2+
import { normalizeJsonValue } from "./object-utils.js"
3+
4+
export const COMMAND_ARTIFACT_STRING_MAX_BYTES = 1024 * 1024
5+
export const COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES = 16 * 1024 * 1024
6+
export const COMMAND_ARTIFACT_MAX_NODES = 25_000
7+
export const COMMAND_ARTIFACT_MAX_RECORDS = 1_000
8+
const COMMAND_ARTIFACT_MAX_TRUNCATIONS = 1_000
9+
const COMMAND_ARTIFACT_KEY_MAX_BYTES = 256
10+
const COMMAND_ARTIFACT_IDENTITY_MAX_BYTES = 1024
11+
const COMMAND_ARTIFACT_PATH_MAX_BYTES = 4096
12+
const EXECUTION_RESULT_KEYS = new Set(["id", "command", "args", "exitCode", "stdout", "stderr", "result", "diagnostics", "artifactRefs", "startedAt", "finishedAt", "artifactCapture"])
13+
14+
export interface CommandArtifactTruncation {
15+
path: string
16+
reason: "string-byte-limit" | "total-string-byte-limit" | "node-limit"
17+
observedBytes?: number
18+
capturedBytes?: number
19+
configuredLimitBytes?: number
20+
}
21+
22+
export interface CommandArtifactCapture {
23+
schema: "wp-codebox/command-artifact-capture/v1"
24+
truncated: true
25+
limits: {
26+
capturedStringBytesPerValue: number
27+
capturedStringBytesTotal: number
28+
nodes: number
29+
records: number
30+
}
31+
fields: CommandArtifactTruncation[]
32+
omittedFieldCount?: number
33+
omittedCommandCount?: number
34+
}
35+
36+
export type BoundedExecutionResult<T extends ExecutionResult = ExecutionResult> = T & { artifactCapture?: CommandArtifactCapture }
37+
38+
export function boundedExecutionResultsForArtifacts<T extends ExecutionResult>(commands: T[]): Array<BoundedExecutionResult<T>> {
39+
const budget = {
40+
remainingBytes: COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES,
41+
remainingNodes: COMMAND_ARTIFACT_MAX_NODES,
42+
remainingTruncations: COMMAND_ARTIFACT_MAX_TRUNCATIONS,
43+
}
44+
const selectedCommands = commands.length <= COMMAND_ARTIFACT_MAX_RECORDS
45+
? commands
46+
: [...commands.slice(0, COMMAND_ARTIFACT_MAX_RECORDS - 1), commands.at(-1)!]
47+
48+
return selectedCommands.map((command, selectedIndex) => {
49+
const fields: CommandArtifactTruncation[] = []
50+
let omittedFieldCount = 0
51+
const capture = (value: unknown, path: string): unknown => boundedArtifactValue(value, path, budget, (field) => {
52+
if (fields.length < 100 && budget.remainingTruncations > 0) {
53+
fields.push(field)
54+
budget.remainingTruncations -= 1
55+
} else {
56+
omittedFieldCount += 1
57+
}
58+
})
59+
const extensions = Object.fromEntries(
60+
Object.entries(command)
61+
.filter(([key]) => !EXECUTION_RESULT_KEYS.has(key))
62+
.map(([key, value]) => [key, capture(normalizeJsonValue(value), key)]),
63+
)
64+
const projected = {
65+
...extensions,
66+
id: boundedIdentityString(command.id),
67+
command: boundedIdentityString(command.command),
68+
args: capture(normalizeJsonValue(command.args), "args") as string[],
69+
exitCode: command.exitCode,
70+
stdout: capture(command.stdout, "stdout") as string,
71+
stderr: capture(command.stderr, "stderr") as string,
72+
...(command.result === undefined ? {} : { result: capture(normalizeJsonValue(command.result), "result") as ExecutionResult["result"] }),
73+
...(command.diagnostics === undefined ? {} : { diagnostics: capture(normalizeJsonValue(command.diagnostics), "diagnostics") }),
74+
...(command.artifactRefs === undefined ? {} : { artifactRefs: capture(normalizeJsonValue(command.artifactRefs), "artifactRefs") as ExecutionResult["artifactRefs"] }),
75+
startedAt: boundedIdentityString(command.startedAt),
76+
finishedAt: boundedIdentityString(command.finishedAt),
77+
} as BoundedExecutionResult<T>
78+
const omittedCommandCount = commands.length - selectedCommands.length
79+
if (fields.length > 0 || omittedFieldCount > 0 || (omittedCommandCount > 0 && selectedIndex === selectedCommands.length - 1)) {
80+
projected.artifactCapture = {
81+
schema: "wp-codebox/command-artifact-capture/v1",
82+
truncated: true,
83+
limits: {
84+
capturedStringBytesPerValue: COMMAND_ARTIFACT_STRING_MAX_BYTES,
85+
capturedStringBytesTotal: COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES,
86+
nodes: COMMAND_ARTIFACT_MAX_NODES,
87+
records: COMMAND_ARTIFACT_MAX_RECORDS,
88+
},
89+
fields,
90+
...(omittedFieldCount > 0 ? { omittedFieldCount } : {}),
91+
...(omittedCommandCount > 0 && selectedIndex === selectedCommands.length - 1 ? { omittedCommandCount } : {}),
92+
}
93+
}
94+
return projected
95+
})
96+
}
97+
98+
function boundedArtifactValue(
99+
value: unknown,
100+
path: string,
101+
budget: { remainingBytes: number; remainingNodes: number; remainingTruncations: number },
102+
recordTruncation: (field: CommandArtifactTruncation) => void,
103+
): unknown {
104+
if (value === null || value === undefined || (typeof value !== "string" && typeof value !== "object")) {
105+
return value
106+
}
107+
if (budget.remainingNodes <= 0) {
108+
recordTruncation({ path: truncateUtf8(path, COMMAND_ARTIFACT_PATH_MAX_BYTES), reason: "node-limit" })
109+
if (typeof value === "string") return ""
110+
return Array.isArray(value) ? [] : {}
111+
}
112+
budget.remainingNodes -= 1
113+
if (typeof value === "string") {
114+
const observedBytes = Buffer.byteLength(value, "utf8")
115+
const totalBudgetLimited = budget.remainingBytes < Math.min(observedBytes, COMMAND_ARTIFACT_STRING_MAX_BYTES)
116+
const configuredLimitBytes = Math.min(COMMAND_ARTIFACT_STRING_MAX_BYTES, budget.remainingBytes)
117+
const captured = truncateUtf8(value, configuredLimitBytes)
118+
const capturedBytes = Buffer.byteLength(captured, "utf8")
119+
budget.remainingBytes -= capturedBytes
120+
if (capturedBytes < observedBytes) {
121+
recordTruncation({
122+
path: truncateUtf8(path, COMMAND_ARTIFACT_PATH_MAX_BYTES),
123+
reason: totalBudgetLimited ? "total-string-byte-limit" : "string-byte-limit",
124+
observedBytes,
125+
capturedBytes,
126+
configuredLimitBytes,
127+
})
128+
}
129+
return captured
130+
}
131+
if (Array.isArray(value)) {
132+
return value.map((item, index) => boundedArtifactValue(item, `${path}[${index}]`, budget, recordTruncation))
133+
}
134+
if (value && typeof value === "object") {
135+
return Object.fromEntries(Object.entries(value).map(([key, item]) => {
136+
const capturedKey = truncateUtf8(key, COMMAND_ARTIFACT_KEY_MAX_BYTES)
137+
return [capturedKey, boundedArtifactValue(item, `${path}.${capturedKey}`, budget, recordTruncation)]
138+
}))
139+
}
140+
return value
141+
}
142+
143+
function boundedIdentityString(value: string): string {
144+
return truncateUtf8(value, COMMAND_ARTIFACT_IDENTITY_MAX_BYTES)
145+
}
146+
147+
export function truncateUtf8(value: string, maxBytes: number): string {
148+
if (maxBytes <= 0) return ""
149+
if (Buffer.byteLength(value, "utf8") <= maxBytes) return value
150+
151+
let low = 0
152+
let high = Math.min(value.length, maxBytes)
153+
while (low < high) {
154+
const middle = Math.ceil((low + high) / 2)
155+
if (Buffer.byteLength(value.slice(0, middle), "utf8") <= maxBytes) low = middle
156+
else high = middle - 1
157+
}
158+
const captured = value.slice(0, low)
159+
return /[\uD800-\uDBFF]$/.test(captured) ? captured.slice(0, -1) : captured
160+
}

packages/runtime-core/src/internals.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
* subpath instead.
55
*/
66
export * from "./benchmark-substrate.js"
7+
export * from "./bounded-execution-results.js"
78
export * from "./fanout-aggregation.js"
89
export * from "./fanout-execution.js"
910
export * from "./object-utils.js"

packages/runtime-playground/src/artifact-bundle-builder.ts

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ import {
4242
previewLeaseSummary,
4343
type Snapshot,
4444
} from "@automattic/wp-codebox-core"
45-
import { normalizeJsonValue, stripUndefined } from "@automattic/wp-codebox-core/internals"
45+
import { COMMAND_ARTIFACT_MAX_NODES, COMMAND_ARTIFACT_MAX_RECORDS, COMMAND_ARTIFACT_STRING_MAX_BYTES, COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES, boundedExecutionResultsForArtifacts, normalizeJsonValue, stripUndefined, truncateUtf8, type BoundedExecutionResult } from "@automattic/wp-codebox-core/internals"
4646
import type { BrowserArtifact } from "./browser-artifacts.js"
4747
import { firstCommandWordPressAdminAuthRequirement } from "./command-auth-requirements.js"
4848
import { writeTrustedApplyArtifacts } from "./trusted-apply-artifact-channel.js"
@@ -92,7 +92,7 @@ export interface ArtifactBundleBuilderSource {
9292
pluginCheckManifestFiles(): ArtifactManifestFile[]
9393
themeCheckManifestFiles(): ArtifactManifestFile[]
9494
formatRuntimeLog(): string
95-
formatCommandsLog(): string
95+
formatCommandsLog(commands: ExecutionResult[]): string
9696
recordArtifactsCollected(bundleId: string, createdAt: string, spec: ArtifactSpec): void
9797
}
9898

@@ -366,11 +366,25 @@ export class ArtifactBundleBuilder {
366366
if (replaySnapshot) {
367367
await writeRedactedArtifact(redactor, partialBlueprintAfterPath, source.artifactRoot, artifactJson(partialBlueprintAfter))
368368
}
369+
let artifactCommands: BoundedExecutionResult[]
370+
try {
371+
artifactCommands = boundedExecutionResultsForArtifacts(source.commands)
372+
} catch (error) {
373+
throw commandArtifactCollectionError(source.commands, "commands.jsonl", error)
374+
}
369375
await writeJsonLines(eventsPath, source.events, redactor, source.artifactRoot)
370-
await writeJsonLines(commandsPath, source.commands, redactor, source.artifactRoot)
376+
try {
377+
await writeJsonLines(commandsPath, artifactCommands, redactor, source.artifactRoot)
378+
} catch (error) {
379+
throw commandArtifactCollectionError(source.commands, "commands.jsonl", error)
380+
}
371381
await writeJsonLines(observationsPath, source.observations, redactor, source.artifactRoot)
372382
await writeRedactedArtifact(redactor, runtimeLogPath, source.artifactRoot, source.formatRuntimeLog())
373-
await writeRedactedArtifact(redactor, commandsLogPath, source.artifactRoot, source.formatCommandsLog())
383+
try {
384+
await writeRedactedArtifact(redactor, commandsLogPath, source.artifactRoot, source.formatCommandsLog(artifactCommands))
385+
} catch (error) {
386+
throw commandArtifactCollectionError(source.commands, "logs/commands.log", error)
387+
}
374388
await writeRedactedArtifact(redactor, mountsPath, source.artifactRoot, artifactJson(source.mounts))
375389
await writeRedactedArtifact(redactor, capturedMountsPath, source.artifactRoot, artifactJson(serializeCapturedMountFiles(capturedMounts)))
376390
await writeRedactedArtifact(redactor, diffsPath, source.artifactRoot, artifactJson(mountDiffs))
@@ -758,6 +772,30 @@ async function writeJsonLines(path: string, records: unknown[], redactor: Artifa
758772
await writeRedactedArtifact(redactor, path, artifactRoot, artifactJsonLines(records.map((record) => normalizeJsonValue(record))))
759773
}
760774

775+
function commandArtifactCollectionError(commands: ExecutionResult[], artifactPath: string, cause: unknown): Error {
776+
const activeCommand = commands.at(-1)
777+
return Object.assign(new Error(`Command artifact collection failed while writing ${artifactPath}`, { cause }), {
778+
name: "CommandArtifactCollectionError",
779+
code: "command-artifact-collection-failed",
780+
operation: "serialize-command-evidence",
781+
subsystem: "artifact-bundle",
782+
artifactPath,
783+
payload: {
784+
commandCount: commands.length,
785+
stdoutBytes: commands.reduce((total, command) => total + Buffer.byteLength(command.stdout, "utf8"), 0),
786+
stderrBytes: commands.reduce((total, command) => total + Buffer.byteLength(command.stderr, "utf8"), 0),
787+
},
788+
limits: {
789+
capturedStringBytesPerValue: COMMAND_ARTIFACT_STRING_MAX_BYTES,
790+
capturedStringBytesTotal: COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES,
791+
nodes: COMMAND_ARTIFACT_MAX_NODES,
792+
records: COMMAND_ARTIFACT_MAX_RECORDS,
793+
},
794+
...(activeCommand ? { commandIndex: commands.length - 1, command: activeCommand.command, commandId: activeCommand.id } : {}),
795+
causeStack: cause instanceof Error ? truncateUtf8(cause.stack ?? `${cause.name}: ${cause.message}`, 16 * 1024) : undefined,
796+
})
797+
}
798+
761799
function artifactJson(value: unknown): string {
762800
return `${JSON.stringify(normalizeJsonValue(value), null, 2)}\n`
763801
}

0 commit comments

Comments
 (0)