Bound retained recipe command evidence

.github/workflows/agent-task-contracts.yml

@@ -96,6 +96,7 @@ jobs:
       - run: npm run test:native-agent-task-playground-e2e
       - run: npm run test:native-agent-task-interruption
       - run: npm run test:trusted-apply-artifact-channel
+      - run: npm run test:runtime-command-artifact-bounds
       - run: npm run test:redaction
       - run: npm run test:production-boundary-enforcement
       - run: npm run test:runtime-tool-policy

package.json

@@ -154,6 +154,7 @@
     "test:workspace-preload-artifacts": "tsx tests/workspace-preload-artifacts.test.ts",
     "test:runner-workspace-apply": "tsx tests/runner-workspace-apply.test.ts",
     "test:trusted-apply-artifact-channel": "tsx tests/trusted-apply-artifact-channel.integration.test.ts",
+    "test:runtime-command-artifact-bounds": "tsx tests/runtime-command-artifact-bounds.test.ts",
     "test:runner-workspace-publisher": "node tests/runner-workspace-publisher.test.mjs",
     "test:native-agent-task-lifecycle": "node tests/execute-native-agent-task-lifecycle.test.mjs",
     "test:native-agent-task-interruption": "node tests/execute-native-agent-task-interruption.test.mjs",

packages/cli/src/commands/recipe-run-output.ts

@@ -1,5 +1,6 @@
 import { setTimeout as delay } from "node:timers/promises"
-import type { RecipeRunSummary, RuntimeRunRecord } from "@automattic/wp-codebox-core"
+import type { ExecutionResult, RecipeRunSummary, RuntimeRunRecord } from "@automattic/wp-codebox-core"
+import { boundedExecutionResultsForArtifacts } from "@automattic/wp-codebox-core/internals"
 import { serializeError } from "../output.js"
 import type { RunOutput } from "../runtime-command-wrappers.js"
 import { RecipePhaseError } from "./recipe-run-phases.js"
@@ -95,7 +96,14 @@ function hasTerminalRecipePhaseFailure(output: RecipeRunOutput): boolean {
 }
 
 export async function writeRecipeJsonOutput(output: unknown): Promise<void> {
-  await writeStdout(`${JSON.stringify(output, null, 2)}\n`)
+  await writeStdout(`${JSON.stringify(boundedRecipeJsonOutput(output), null, 2)}\n`)
+}
+
+export function boundedRecipeJsonOutput(output: unknown): unknown {
+  if (!output || typeof output !== "object" || Array.isArray(output)) return output
+  const record = output as Record<string, unknown>
+  if (!Array.isArray(record.executions)) return output
+  return { ...record, executions: boundedExecutionResultsForArtifacts(record.executions as ExecutionResult[]) }
 }
 
 export async function writeRecipeSummaryHumanOutput(summary: RecipeRunSummary): Promise<void> {

packages/cli/src/commands/recipe-run.ts

@@ -327,7 +327,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
       await awaitRecipe("runtime.observe:runtime-info", runtime!.observe({ type: "runtime-info" }))
       await awaitRecipe("runtime.observe:mounts", runtime!.observe({ type: "mounts" }))
       runRecord = await runRegistry.update(runRecord.runId, { status: "collecting_artifacts", runtime: await runtime!.info() })
-      artifacts = await awaitRecipe("runtime.collect-artifacts", collectRecipeRuntimeArtifacts(runtime!, { includeLogs: true, includeObservations: true }, { snapshotTimeoutMs: SUCCESSFUL_RECIPE_RUNTIME_SNAPSHOT_TIMEOUT_MS }))
+      artifacts = await awaitRecipe("runtime.collect-artifacts", collectRecipeRuntimeArtifacts(runtime!, { includeLogs: true, includeObservations: true }, { snapshotTimeoutMs: SUCCESSFUL_RECIPE_RUNTIME_SNAPSHOT_TIMEOUT_MS, activeExecution: executions.at(-1) }))
       browserEvidence = await recipeBrowserEvidence(artifacts, executions, recipe)
       await artifactPointer.update({ runtime: await runtime!.info(), artifacts, phases: phaseTracker.list(), browserEvidence })
       await materializeTypedRecipeDeclaredArtifacts(artifacts, declaredArtifacts)
@@ -350,7 +350,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
     const recipeFailure = strictFailure ?? agentFailure ?? verifyFailure
     const successfulRecipe = !recipeFailure
     if (successfulRecipe && options.previewHoldSeconds) {
-      artifacts = await awaitRecipe("runtime.collect-artifacts.preview-hold", collectRecipeRuntimeArtifacts(runtime, { includeLogs: true, includeObservations: true, previewHoldSeconds: options.previewHoldSeconds }, { snapshotTimeoutMs: SUCCESSFUL_RECIPE_RUNTIME_SNAPSHOT_TIMEOUT_MS }))
+      artifacts = await awaitRecipe("runtime.collect-artifacts.preview-hold", collectRecipeRuntimeArtifacts(runtime, { includeLogs: true, includeObservations: true, previewHoldSeconds: options.previewHoldSeconds }, { snapshotTimeoutMs: SUCCESSFUL_RECIPE_RUNTIME_SNAPSHOT_TIMEOUT_MS, activeExecution: executions.at(-1) }))
       browserEvidence = await recipeBrowserEvidence(artifacts, executions, recipe)
       await artifactPointer.update({ runtime: await runtime.info(), artifacts, phases: phaseTracker.list(), browserEvidence })
       declaredArtifacts = await collectRecipeDeclaredArtifacts(recipe, runtime)

packages/cli/src/recipe-evidence.ts

@@ -301,6 +301,31 @@ export interface RecipeArtifactsFinalizationController {
 interface RecipeRuntimeArtifactCollectionOptions {
   timeoutMs?: number
   snapshotTimeoutMs?: number
+  activeExecution?: RecipeEvidenceExecutionResult
+}
+
+export class RecipeRuntimeArtifactCollectionError extends Error {
+  readonly code = "recipe-artifact-collection-failed"
+  readonly operation = "runtime.collect-artifacts"
+  readonly subsystem = "artifact-bundle"
+  readonly workflowStepIndex?: number
+  readonly command?: string
+  readonly artifactPath?: string
+  readonly payload?: unknown
+  readonly limits?: unknown
+  readonly causeStack?: string
+
+  constructor(activeExecution: RecipeEvidenceExecutionResult | undefined, cause: Error) {
+    const causeRecord = cause as Error & Record<string, unknown>
+    super(`Recipe artifact collection failed${activeExecution?.recipeStepIndex === undefined ? "" : ` at workflow step ${activeExecution.recipeStepIndex}`}: ${cause.message}`, { cause })
+    this.name = "RecipeRuntimeArtifactCollectionError"
+    this.workflowStepIndex = activeExecution?.recipeStepIndex
+    this.command = activeExecution?.recipeCommand ?? activeExecution?.command
+    this.artifactPath = typeof causeRecord.artifactPath === "string" ? causeRecord.artifactPath : undefined
+    this.payload = causeRecord.payload
+    this.limits = causeRecord.limits
+    this.causeStack = typeof causeRecord.causeStack === "string" ? causeRecord.causeStack : cause.stack?.slice(0, 16 * 1024)
+  }
 }
 
 const execFileAsync = promisify(execFile)
@@ -322,7 +347,7 @@ export async function collectAndFinalizeFailedRecipeArtifacts(args: {
 
   if (!artifacts) {
     try {
-      artifacts = await collectRecipeRuntimeArtifacts(args.runtime, { includeLogs: true, includeObservations: true }, { snapshotTimeoutMs: 20_000, timeoutMs: 30_000 })
+      artifacts = await collectRecipeRuntimeArtifacts(args.runtime, { includeLogs: true, includeObservations: true }, { snapshotTimeoutMs: 20_000, timeoutMs: 30_000, activeExecution: args.executions.at(-1) })
     } catch {
       return undefined
     }
@@ -354,11 +379,19 @@ export async function collectRecipeRuntimeArtifacts(runtime: Runtime, spec: Arti
     }
   }
 
-  const artifacts = runtime.collectArtifacts(spec)
-  if (options.timeoutMs && options.timeoutMs > 0) {
-    return timeoutOrReject(artifacts, options.timeoutMs, `Runtime artifact collection exceeded ${options.timeoutMs}ms`)
+  try {
+    const artifacts = runtime.collectArtifacts(spec)
+    if (options.timeoutMs && options.timeoutMs > 0) {
+      return await timeoutOrReject(artifacts, options.timeoutMs, `Runtime artifact collection exceeded ${options.timeoutMs}ms`)
+    }
+    return await artifacts
+  } catch (error) {
+    const candidate = error instanceof Error ? error as Error & Record<string, unknown> : undefined
+    if (candidate?.code === "command-artifact-collection-failed" || error instanceof RangeError) {
+      throw new RecipeRuntimeArtifactCollectionError(options.activeExecution, candidate ?? Object.assign(new Error(String(error)), { code: "runtime-allocation-failed" }))
+    }
+    throw error
   }
-  return artifacts
 }
 
 async function timeoutOrUndefined<T>(promise: Promise<T>, timeoutMs: number): Promise<T | undefined> {

packages/runtime-core/src/bounded-execution-results.ts

@@ -0,0 +1,168 @@
+import type { ExecutionResult } from "./runtime-contracts.js"
+import { normalizeJsonValue } from "./object-utils.js"
+
+export const COMMAND_ARTIFACT_STRING_MAX_BYTES = 1024 * 1024
+export const COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES = 2 * 1024 * 1024
+export const COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES = 16 * 1024 * 1024
+export const COMMAND_ARTIFACT_MAX_NODES = 25_000
+export const COMMAND_ARTIFACT_MAX_RECORDS = 1_000
+const COMMAND_ARTIFACT_MAX_TRUNCATIONS = 1_000
+const COMMAND_ARTIFACT_KEY_MAX_BYTES = 256
+const COMMAND_ARTIFACT_IDENTITY_MAX_BYTES = 1024
+const COMMAND_ARTIFACT_PATH_MAX_BYTES = 4096
+const EXECUTION_RESULT_KEYS = new Set(["id", "command", "args", "exitCode", "stdout", "stderr", "result", "diagnostics", "artifactRefs", "startedAt", "finishedAt", "artifactCapture"])
+const EXECUTION_IDENTITY_EXTENSION_KEYS = new Set(["recipePhase", "recipeCommand", "fuzzCaseId", "fuzzPhase"])
+
+export interface CommandArtifactTruncation {
+  path: string
+  reason: "string-byte-limit" | "command-string-byte-limit" | "total-string-byte-limit" | "node-limit"
+  observedBytes?: number
+  capturedBytes?: number
+  configuredLimitBytes?: number
+}
+
+export interface CommandArtifactCapture {
+  schema: "wp-codebox/command-artifact-capture/v1"
+  truncated: true
+  limits: {
+    capturedStringBytesPerValue: number
+    capturedStringBytesPerCommand: number
+    capturedStringBytesTotal: number
+    nodes: number
+    records: number
+  }
+  fields: CommandArtifactTruncation[]
+  omittedFieldCount?: number
+  omittedCommandCount?: number
+}
+
+export type BoundedExecutionResult<T extends ExecutionResult = ExecutionResult> = T & { artifactCapture?: CommandArtifactCapture }
+
+export function boundedExecutionResultsForArtifacts<T extends ExecutionResult>(commands: T[]): Array<BoundedExecutionResult<T>> {
+  const budget = {
+    remainingBytes: COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES,
+    remainingCommandBytes: COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES,
+    remainingNodes: COMMAND_ARTIFACT_MAX_NODES,
+    remainingTruncations: COMMAND_ARTIFACT_MAX_TRUNCATIONS,
+  }
+  const selectedCommands = commands.length <= COMMAND_ARTIFACT_MAX_RECORDS
+    ? commands
+    : [...commands.slice(0, COMMAND_ARTIFACT_MAX_RECORDS - 1), commands.at(-1)!]
+
+  return selectedCommands.map((command, selectedIndex) => {
+    budget.remainingCommandBytes = COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES
+    const fields: CommandArtifactTruncation[] = []
+    let omittedFieldCount = 0
+    const capture = (value: unknown, path: string): unknown => boundedArtifactValue(value, path, budget, (field) => {
+      if (fields.length < 100 && budget.remainingTruncations > 0) {
+        fields.push(field)
+        budget.remainingTruncations -= 1
+      } else {
+        omittedFieldCount += 1
+      }
+    })
+    const extensions = Object.fromEntries(
+      Object.entries(command)
+        .filter(([key]) => !EXECUTION_RESULT_KEYS.has(key))
+        .map(([key, value]) => [key, typeof value === "string" && EXECUTION_IDENTITY_EXTENSION_KEYS.has(key) ? boundedIdentityString(value) : capture(normalizeJsonValue(value), key)]),
+    )
+    const projected = {
+      ...extensions,
+      id: boundedIdentityString(command.id),
+      command: boundedIdentityString(command.command),
+      args: capture(normalizeJsonValue(command.args), "args") as string[],
+      exitCode: command.exitCode,
+      stdout: capture(command.stdout, "stdout") as string,
+      stderr: capture(command.stderr, "stderr") as string,
+      ...(command.result === undefined ? {} : { result: capture(normalizeJsonValue(command.result), "result") as ExecutionResult["result"] }),
+      ...(command.diagnostics === undefined ? {} : { diagnostics: capture(normalizeJsonValue(command.diagnostics), "diagnostics") }),
+      ...(command.artifactRefs === undefined ? {} : { artifactRefs: capture(normalizeJsonValue(command.artifactRefs), "artifactRefs") as ExecutionResult["artifactRefs"] }),
+      startedAt: boundedIdentityString(command.startedAt),
+      finishedAt: boundedIdentityString(command.finishedAt),
+    } as BoundedExecutionResult<T>
+    const omittedCommandCount = commands.length - selectedCommands.length
+    if (fields.length > 0 || omittedFieldCount > 0 || (omittedCommandCount > 0 && selectedIndex === selectedCommands.length - 1)) {
+      projected.artifactCapture = {
+        schema: "wp-codebox/command-artifact-capture/v1",
+        truncated: true,
+        limits: {
+          capturedStringBytesPerValue: COMMAND_ARTIFACT_STRING_MAX_BYTES,
+          capturedStringBytesPerCommand: COMMAND_ARTIFACT_COMMAND_STRING_MAX_BYTES,
+          capturedStringBytesTotal: COMMAND_ARTIFACT_TOTAL_STRING_MAX_BYTES,
+          nodes: COMMAND_ARTIFACT_MAX_NODES,
+          records: COMMAND_ARTIFACT_MAX_RECORDS,
+        },
+        fields,
+        ...(omittedFieldCount > 0 ? { omittedFieldCount } : {}),
+        ...(omittedCommandCount > 0 && selectedIndex === selectedCommands.length - 1 ? { omittedCommandCount } : {}),
+      }
+    }
+    return projected
+  })
+}
+
+function boundedArtifactValue(
+  value: unknown,
+  path: string,
+  budget: { remainingBytes: number; remainingCommandBytes: number; remainingNodes: number; remainingTruncations: number },
+  recordTruncation: (field: CommandArtifactTruncation) => void,
+): unknown {
+  if (value === null || value === undefined || (typeof value !== "string" && typeof value !== "object")) {
+    return value
+  }
+  if (budget.remainingNodes <= 0) {
+    recordTruncation({ path: truncateUtf8(path, COMMAND_ARTIFACT_PATH_MAX_BYTES), reason: "node-limit" })
+    if (typeof value === "string") return ""
+    return Array.isArray(value) ? [] : {}
+  }
+  budget.remainingNodes -= 1
+  if (typeof value === "string") {
+    const observedBytes = Buffer.byteLength(value, "utf8")
+    const totalBudgetLimited = budget.remainingBytes < Math.min(observedBytes, COMMAND_ARTIFACT_STRING_MAX_BYTES)
+    const commandBudgetLimited = budget.remainingCommandBytes < Math.min(observedBytes, COMMAND_ARTIFACT_STRING_MAX_BYTES)
+    const configuredLimitBytes = Math.min(COMMAND_ARTIFACT_STRING_MAX_BYTES, budget.remainingCommandBytes, budget.remainingBytes)
+    const captured = truncateUtf8(value, configuredLimitBytes)
+    const capturedBytes = Buffer.byteLength(captured, "utf8")
+    budget.remainingBytes -= capturedBytes
+    budget.remainingCommandBytes -= capturedBytes
+    if (capturedBytes < observedBytes) {
+      recordTruncation({
+        path: truncateUtf8(path, COMMAND_ARTIFACT_PATH_MAX_BYTES),
+        reason: totalBudgetLimited ? "total-string-byte-limit" : commandBudgetLimited ? "command-string-byte-limit" : "string-byte-limit",
+        observedBytes,
+        capturedBytes,
+        configuredLimitBytes,
+      })
+    }
+    return captured
+  }
+  if (Array.isArray(value)) {
+    return value.map((item, index) => boundedArtifactValue(item, `${path}[${index}]`, budget, recordTruncation))
+  }
+  if (value && typeof value === "object") {
+    return Object.fromEntries(Object.entries(value).map(([key, item]) => {
+      const capturedKey = truncateUtf8(key, COMMAND_ARTIFACT_KEY_MAX_BYTES)
+      return [capturedKey, boundedArtifactValue(item, `${path}.${capturedKey}`, budget, recordTruncation)]
+    }))
+  }
+  return value
+}
+
+function boundedIdentityString(value: string): string {
+  return truncateUtf8(value, COMMAND_ARTIFACT_IDENTITY_MAX_BYTES)
+}
+
+export function truncateUtf8(value: string, maxBytes: number): string {
+  if (maxBytes <= 0) return ""
+  if (Buffer.byteLength(value, "utf8") <= maxBytes) return value
+
+  let low = 0
+  let high = Math.min(value.length, maxBytes)
+  while (low < high) {
+    const middle = Math.ceil((low + high) / 2)
+    if (Buffer.byteLength(value.slice(0, middle), "utf8") <= maxBytes) low = middle
+    else high = middle - 1
+  }
+  const captured = value.slice(0, low)
+  return /[\uD800-\uDBFF]$/.test(captured) ? captured.slice(0, -1) : captured
+}

packages/runtime-core/src/internals.ts

@@ -4,6 +4,7 @@
  * subpath instead.
  */
 export * from "./benchmark-substrate.js"
+export * from "./bounded-execution-results.js"
 export * from "./fanout-aggregation.js"
 export * from "./fanout-execution.js"
 export * from "./object-utils.js"