Tighten runtime episode trace envelopes

* Tighten runtime episode trace envelopes

* Fix runtime episode JSON schema

Author: chubes4

README.md

@@ -305,16 +305,33 @@ scenario, benchmark, task-set, and model-eval belong in consumers that project
 the trace into their own schemas.
 
 Episode steps include stable ids and refs for the step, requested action,
-runtime execution, optional observation, and collected artifact bundle. Action
-and execution refs include SHA-256 digests over the canonical trace v1 JSON
-payload so callers can compare command/result records without depending on
+runtime execution, optional observation, and collected artifact bundle. Actions
+use the generic `wp-codebox/runtime-episode-action/v1` envelope with
+`kind: "command"`, a command name, string args, optional `cwd`, optional
+`timeoutMs`, and a SHA-256 digest over that replay payload. Observations use
+the generic `wp-codebox/runtime-episode-observation/v1` envelope with id, type,
+data, timestamp, and digest. Refs carry matching digests so callers can compare
+action args, observations, executions, snapshots, and artifacts without parsing
 presentation logs.
 
+Replay is bounded to the generic runtime contract. A consumer can replay a step
+by creating a compatible backend runtime, applying the same mounts/artifact
+inputs, and executing the action envelope in order. WP Codebox intentionally
+does not record rewards, scenario ids, benchmark metadata, model-eval fields, or
+product-specific success semantics; those belong to the caller's projection of
+the trace. Backend behavior still matters: command availability, WordPress
+version, installed packages, mounted files, network policy, and secrets policy
+must be reproduced by the replay harness.
+
 Runtime snapshots are explicit about their semantics. The current Playground
 backend returns `semantics: "metadata-only"`, which records runtime and mount
 metadata for audit context but is not a full replayable WordPress state image.
-Call `collectArtifacts()` when the caller needs replay inputs, changed files,
-patches, logs, observations, and artifact bundle refs.
+These snapshots are trace markers, not database or filesystem checkpoints. Call
+`collectArtifacts()` when the caller needs replay inputs, changed files, patches,
+logs, observations, and artifact bundle refs. Future backends may emit stronger
+snapshot semantics such as runtime-state artifacts, but replay consumers should
+branch on the snapshot `semantics` field instead of assuming a portable restore
+point.
 
 ## CLI Commands
 

packages/runtime-core/src/index.ts

@@ -7,6 +7,8 @@ export * from "./workspace-policy.js"
 export type RuntimeBackendKind = "wordpress-playground" | (string & {})
 
 export const RUNTIME_EPISODE_TRACE_SCHEMA = "wp-codebox/runtime-episode-trace/v1" as const
+export const RUNTIME_EPISODE_ACTION_SCHEMA = "wp-codebox/runtime-episode-action/v1" as const
+export const RUNTIME_EPISODE_OBSERVATION_SCHEMA = "wp-codebox/runtime-episode-observation/v1" as const
 
 export const RUNTIME_EPISODE_TRACE_JSON_SCHEMA = {
   $id: RUNTIME_EPISODE_TRACE_SCHEMA,
@@ -21,7 +23,39 @@ export const RUNTIME_EPISODE_TRACE_JSON_SCHEMA = {
     reset: { type: "object", required: ["id", "runtime", "observations", "observationRefs"] },
     steps: {
       type: "array",
-      items: { type: "object", required: ["id", "index", "action", "actionRef", "execution", "executionRef"] },
+      items: {
+        type: "object",
+        required: ["id", "index", "action", "actionRef", "execution", "executionRef"],
+        properties: {
+          action: {
+            type: "object",
+            required: ["schema", "id", "kind", "command", "args", "digest"],
+            properties: {
+              schema: { const: RUNTIME_EPISODE_ACTION_SCHEMA },
+              id: { type: "string", minLength: 1 },
+              kind: { const: "command" },
+              command: { type: "string", minLength: 1 },
+              args: { type: "array", items: { type: "string" } },
+              cwd: { type: "string" },
+              timeoutMs: { type: "number", minimum: 0 },
+              digest: {
+                type: "object",
+                required: ["algorithm", "value"],
+                properties: {
+                  algorithm: { const: "sha256" },
+                  value: { type: "string", pattern: "^[a-f0-9]{64}$" },
+                },
+                additionalProperties: false,
+              },
+            },
+            additionalProperties: false,
+          },
+          observation: {
+            type: "object",
+            required: ["schema", "id", "type", "data", "observedAt", "digest"],
+          },
+        },
+      },
     },
     snapshots: {
       type: "array",
@@ -400,8 +434,14 @@ export interface RuntimeEpisodeTraceRef {
   path?: string
 }
 
-export interface RuntimeEpisodeActionRecord extends ExecutionSpec {
+export interface RuntimeEpisodeActionRecord {
+  schema: typeof RUNTIME_EPISODE_ACTION_SCHEMA
   id: string
+  kind: "command"
+  command: string
+  args: string[]
+  cwd?: string
+  timeoutMs?: number
   digest: RuntimeEpisodeContentDigest
 }
 
@@ -422,10 +462,12 @@ export interface ObservationSpec {
 }
 
 export interface ObservationResult {
+  schema?: typeof RUNTIME_EPISODE_OBSERVATION_SCHEMA
   id?: string
   type: string
   data: unknown
   observedAt: string
+  digest?: RuntimeEpisodeContentDigest
 }
 
 export interface LifecycleEvent {
@@ -1202,6 +1244,33 @@ export function runtimeEpisodeDigest(value: unknown): RuntimeEpisodeContentDiges
   }
 }
 
+function runtimeEpisodeActionDigestPayload(action: RuntimeEpisodeActionRecord | ExecutionSpec): Record<string, unknown> {
+  const payload: Record<string, unknown> = {
+    schema: RUNTIME_EPISODE_ACTION_SCHEMA,
+    kind: "command",
+    command: action.command,
+    args: Array.isArray(action.args) ? action.args : [],
+  }
+
+  if (typeof action.cwd === "string") {
+    payload.cwd = action.cwd
+  }
+  if (typeof action.timeoutMs === "number") {
+    payload.timeoutMs = action.timeoutMs
+  }
+
+  return payload
+}
+
+function runtimeEpisodeObservationDigestPayload(observation: ObservationResult): Record<string, unknown> {
+  return {
+    schema: RUNTIME_EPISODE_OBSERVATION_SCHEMA,
+    type: observation.type,
+    data: observation.data,
+    observedAt: observation.observedAt,
+  }
+}
+
 export function validateRuntimeEpisodeTrace(trace: unknown): RuntimeEpisodeTraceValidationResult {
   const issues: RuntimeEpisodeTraceValidationIssue[] = []
   const candidate = trace as Partial<RuntimeEpisodeTrace> | null
@@ -1230,9 +1299,21 @@ export function validateRuntimeEpisodeTrace(trace: unknown): RuntimeEpisodeTrace
   }
   if (!Array.isArray(candidate.reset?.observations)) {
     issues.push({ path: "$.reset.observations", message: "reset observations must be an array" })
+  } else {
+    candidate.reset.observations.forEach((observation, index) => {
+      validateRuntimeEpisodeObservation(observation, `$.reset.observations[${index}]`, issues)
+    })
   }
   if (!Array.isArray(candidate.reset?.observationRefs)) {
     issues.push({ path: "$.reset.observationRefs", message: "reset observationRefs must be an array" })
+  } else {
+    candidate.reset.observationRefs.forEach((ref, index) => {
+      validateRuntimeEpisodeTraceRef(ref, `$.reset.observationRefs[${index}]`, "observation", issues)
+      const observation = candidate.reset?.observations?.[index]
+      if (observation) {
+        validateRuntimeEpisodeRefDigest(ref, observation.digest, `$.reset.observationRefs[${index}]`, issues)
+      }
+    })
   }
   if (!Array.isArray(candidate.steps)) {
     issues.push({ path: "$.steps", message: "steps must be an array" })
@@ -1271,21 +1352,153 @@ function validateRuntimeEpisodeStep(
   }
   if (!nonEmptyString(step.action?.id)) {
     issues.push({ path: `${path}.action.id`, message: "action id is required" })
+  } else {
+    validateRuntimeEpisodeAction(step.action, `${path}.action`, issues)
   }
   if (!nonEmptyString(step.actionRef?.id)) {
     issues.push({ path: `${path}.actionRef.id`, message: "actionRef id is required" })
+  } else {
+    validateRuntimeEpisodeTraceRef(step.actionRef, `${path}.actionRef`, "action", issues)
+    validateRuntimeEpisodeRefDigest(step.actionRef, step.action?.digest, `${path}.actionRef`, issues)
   }
   if (!nonEmptyString(step.execution?.id)) {
     issues.push({ path: `${path}.execution.id`, message: "execution id is required" })
   }
   if (!nonEmptyString(step.executionRef?.id)) {
     issues.push({ path: `${path}.executionRef.id`, message: "executionRef id is required" })
+  } else {
+    validateRuntimeEpisodeTraceRef(step.executionRef, `${path}.executionRef`, "execution", issues)
+    validateRuntimeEpisodeRefDigest(step.executionRef, step.execution ? runtimeEpisodeDigest(step.execution) : undefined, `${path}.executionRef`, issues)
   }
   if (step.observation && !nonEmptyString(step.observation.id)) {
     issues.push({ path: `${path}.observation.id`, message: "observation id is required" })
+  } else if (step.observation) {
+    validateRuntimeEpisodeObservation(step.observation, `${path}.observation`, issues)
+  }
+  if (step.observationRef) {
+    validateRuntimeEpisodeTraceRef(step.observationRef, `${path}.observationRef`, "observation", issues)
+    if (step.observation) {
+      validateRuntimeEpisodeRefDigest(step.observationRef, step.observation.digest, `${path}.observationRef`, issues)
+    }
+  }
+}
+
+function validateRuntimeEpisodeAction(
+  action: RuntimeEpisodeActionRecord | unknown,
+  path: string,
+  issues: RuntimeEpisodeTraceValidationIssue[],
+): void {
+  if (!isRecord(action)) {
+    issues.push({ path, message: "action must be an object" })
+    return
+  }
+
+  if (action.schema !== RUNTIME_EPISODE_ACTION_SCHEMA) {
+    issues.push({ path: `${path}.schema`, message: `action schema must be ${RUNTIME_EPISODE_ACTION_SCHEMA}` })
+  }
+  if (action.kind !== "command") {
+    issues.push({ path: `${path}.kind`, message: "action kind must be command" })
+  }
+  if (!nonEmptyString(action.command)) {
+    issues.push({ path: `${path}.command`, message: "action command is required" })
+  }
+  if (!Array.isArray(action.args) || !action.args.every((arg) => typeof arg === "string")) {
+    issues.push({ path: `${path}.args`, message: "action args must be an array of strings" })
+  }
+  if (action.cwd !== undefined && typeof action.cwd !== "string") {
+    issues.push({ path: `${path}.cwd`, message: "action cwd must be a string when present" })
+  }
+  const timeoutMs = action.timeoutMs
+  if (timeoutMs !== undefined && (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs) || timeoutMs < 0)) {
+    issues.push({ path: `${path}.timeoutMs`, message: "action timeoutMs must be a non-negative number when present" })
+  }
+  if (!validDigest(action.digest)) {
+    issues.push({ path: `${path}.digest`, message: "action digest must be a sha256 digest" })
+    return
+  }
+
+  const expected = runtimeEpisodeDigest(runtimeEpisodeActionDigestPayload(action as unknown as RuntimeEpisodeActionRecord))
+  if (action.digest.value !== expected.value) {
+    issues.push({ path: `${path}.digest`, message: "action digest must match the canonical replay payload" })
+  }
+}
+
+function validateRuntimeEpisodeObservation(
+  observation: ObservationResult | unknown,
+  path: string,
+  issues: RuntimeEpisodeTraceValidationIssue[],
+): void {
+  if (!isRecord(observation)) {
+    issues.push({ path, message: "observation must be an object" })
+    return
+  }
+
+  if (observation.schema !== RUNTIME_EPISODE_OBSERVATION_SCHEMA) {
+    issues.push({ path: `${path}.schema`, message: `observation schema must be ${RUNTIME_EPISODE_OBSERVATION_SCHEMA}` })
+  }
+  if (!nonEmptyString(observation.id)) {
+    issues.push({ path: `${path}.id`, message: "observation id is required" })
+  }
+  if (!nonEmptyString(observation.type)) {
+    issues.push({ path: `${path}.type`, message: "observation type is required" })
+  }
+  if (!("data" in observation)) {
+    issues.push({ path: `${path}.data`, message: "observation data is required" })
+  }
+  if (!nonEmptyString(observation.observedAt)) {
+    issues.push({ path: `${path}.observedAt`, message: "observation observedAt is required" })
+  }
+  if (!validDigest(observation.digest)) {
+    issues.push({ path: `${path}.digest`, message: "observation digest must be a sha256 digest" })
+    return
+  }
+
+  const expected = runtimeEpisodeDigest(runtimeEpisodeObservationDigestPayload(observation as unknown as ObservationResult))
+  if (observation.digest.value !== expected.value) {
+    issues.push({ path: `${path}.digest`, message: "observation digest must match the canonical observation payload" })
+  }
+}
+
+function validateRuntimeEpisodeTraceRef(
+  ref: RuntimeEpisodeTraceRef | unknown,
+  path: string,
+  kind: RuntimeEpisodeTraceRef["kind"],
+  issues: RuntimeEpisodeTraceValidationIssue[],
+): void {
+  if (!isRecord(ref)) {
+    issues.push({ path, message: "ref must be an object" })
+    return
+  }
+
+  if (ref.kind !== kind) {
+    issues.push({ path: `${path}.kind`, message: `ref kind must be ${kind}` })
+  }
+  if (!nonEmptyString(ref.id)) {
+    issues.push({ path: `${path}.id`, message: "ref id is required" })
+  }
+  if (!validDigest(ref.digest)) {
+    issues.push({ path: `${path}.digest`, message: "ref digest must be a sha256 digest" })
   }
 }
 
+function validateRuntimeEpisodeRefDigest(
+  ref: RuntimeEpisodeTraceRef,
+  targetDigest: RuntimeEpisodeContentDigest | undefined,
+  path: string,
+  issues: RuntimeEpisodeTraceValidationIssue[],
+): void {
+  if (!validDigest(ref.digest) || !validDigest(targetDigest)) {
+    return
+  }
+  if (ref.digest.value !== targetDigest.value) {
+    issues.push({ path: `${path}.digest`, message: "ref digest must match the referenced envelope digest" })
+  }
+}
+
+function validDigest(value: unknown): value is RuntimeEpisodeContentDigest {
+  return isRecord(value) && value.algorithm === "sha256" && typeof value.value === "string" && /^[a-f0-9]{64}$/.test(value.value)
+}
+
 function collectForbiddenRuntimeEpisodeTraceFields(
   value: unknown,
   path: string,
@@ -1329,11 +1542,17 @@ function stableJson(value: unknown): string {
 }
 
 function observationRef(observation: ObservationResult, fallbackId: string): RuntimeEpisodeTraceRef {
-  return { kind: "observation", id: observation.id || fallbackId, digest: runtimeEpisodeDigest(observation) }
+  return { kind: "observation", id: observation.id || fallbackId, digest: observation.digest ?? runtimeEpisodeDigest(runtimeEpisodeObservationDigestPayload(observation)) }
 }
 
 function observationWithId(observation: ObservationResult, fallbackId: string): ObservationResult {
-  return { ...observation, id: observation.id || fallbackId }
+  const enveloped = {
+    ...observation,
+    schema: RUNTIME_EPISODE_OBSERVATION_SCHEMA,
+    id: observation.id || fallbackId,
+  }
+
+  return { ...enveloped, digest: runtimeEpisodeDigest(runtimeEpisodeObservationDigestPayload(enveloped)) }
 }
 
 function snapshotWithSemantics(snapshot: Snapshot): Snapshot {
@@ -1406,9 +1625,14 @@ class RuntimeEpisodeRunner implements RuntimeEpisode {
     const index = this.steps.length
     const stepId = `${execution.id}:step:${index}`
     const actionRecord = {
-      ...action,
+      schema: RUNTIME_EPISODE_ACTION_SCHEMA,
       id: `${stepId}:action`,
-      digest: runtimeEpisodeDigest(action),
+      kind: "command" as const,
+      command: action.command,
+      args: action.args ?? [],
+      ...(action.cwd ? { cwd: action.cwd } : {}),
+      ...(action.timeoutMs !== undefined ? { timeoutMs: action.timeoutMs } : {}),
+      digest: runtimeEpisodeDigest(runtimeEpisodeActionDigestPayload(action)),
     }
     const stepObservation = observation ? observationWithId(await runtime.observe(observation), `${stepId}:observation`) : undefined
     const result: RuntimeEpisodeStepResult = {