feat(artifacts): snapshot environment recipe provenance

Author: christso

apps/cli/src/commands/eval/artifact-writer.ts

@@ -3,6 +3,7 @@ import path from 'node:path';
 import {
   type AdditionalResultArtifactsWriter,
   type AggregateGradingArtifact,
+  type EnvironmentSummaryWire,
   type EvalTest,
   type EvaluationResult,
   type ExperimentArtifactMetadata,
@@ -64,6 +65,7 @@ export {
 export type {
   AggregateGradingArtifact,
   GradingArtifact,
+  EnvironmentSummaryWire,
   IndexArtifactEntry,
   ResultIndexArtifact,
   RunConfigArtifact,

apps/cli/src/commands/eval/task-bundle.ts

@@ -812,7 +812,10 @@ function serializeEnvironment(
   rewrites: ReadonlyMap<string, string>,
 ): Record<string, unknown> {
   const {
+    authoredReference: _authoredReference,
     recipeFilePath: _recipeFilePath,
+    recipeFileSha256: _recipeFileSha256,
+    recipeSha256: _recipeSha256,
     sourceDir: _sourceDir,
     ...portableEnvironment
   } = environment;

apps/cli/test/commands/eval/artifact-writer.test.ts

@@ -1316,6 +1316,125 @@ describe('writeArtifactsFromResults', () => {
     expect(indexLine.runtime_source).toBeUndefined();
   });
 
+  it('writes host environment provenance as a sidecar with redacted setup inputs and logs', async () => {
+    const result = makeResult({
+      testId: 'host-env',
+      environmentProvenance: {
+        schemaVersion: 'agentv.environment_provenance.v1',
+        authoredKind: 'file',
+        authoredReference: 'file://.agentv/environments/host.yaml',
+        recipeFilePath: '/repo/.agentv/environments/host.yaml',
+        recipeFileSha256: 'f'.repeat(64),
+        recipeSha256: 'a'.repeat(64),
+        type: 'host',
+        sourceDir: '/repo/.agentv/environments',
+        workdir: '/repo/workspaces/app',
+        setup: {
+          command: ['node', 'setup.mjs', '--api-key', '<redacted>'],
+          args: {
+            repo: 'example/app',
+            commit: 'abc123',
+            api_key: '<redacted>',
+          },
+        },
+        setupExecutions: [
+          {
+            scope: 'environment',
+            name: 'setup',
+            status: 'success',
+            testId: '__environment_setup__',
+            workdir: '/repo/workspaces/app',
+            command: ['node', 'setup.mjs', '--api-key', '<redacted>'],
+            cwd: '/repo/.agentv/environments',
+            output:
+              '{"repo_provenance":{"repo":"example/app","commit":"abc123"}}\\nused <redacted>',
+            exitCode: 0,
+          },
+        ],
+        repoProvenance: { repo: 'example/app', commit: 'abc123' },
+      },
+    });
+
+    const paths = await writeArtifactsFromResults([result], testDir, {
+      evalFile: 'evals/host.eval.yaml',
+    });
+    const [indexLine] = await readIndexLines(paths.indexPath);
+    const rowDir = expectRowDir(indexLine, 'host-env');
+    const environmentPath = path.join(testDir, indexLine.environment_path ?? '');
+    const environment = JSON.parse(await readFile(environmentPath, 'utf8'));
+    const resultJson = JSON.parse(
+      await readFile(path.join(testDir, rowDir, 'sample-1', 'result.json'), 'utf8'),
+    );
+    const summary: RunSummaryArtifact = JSON.parse(await readFile(paths.summaryPath, 'utf8'));
+
+    expect(indexLine.environment).toMatchObject({
+      schema_version: 'agentv.environment_summary.v1',
+      type: 'host',
+      workdir: '/repo/workspaces/app',
+      recipe_sha256: 'a'.repeat(64),
+      authored_reference: 'file://.agentv/environments/host.yaml',
+      setup_status: 'success',
+    });
+    expect(indexLine.environment_path).toBe(`${rowDir}/sample-1/environment.json`);
+    expect(indexLine.environment).not.toHaveProperty('setup_executions');
+    expect(JSON.stringify(indexLine)).not.toContain('used <redacted>');
+    expect(environment.setup.command).toEqual(['node', 'setup.mjs', '--api-key', '<redacted>']);
+    expect(environment.setup.args.api_key).toBe('<redacted>');
+    expect(environment.setup_executions[0].output).toContain('used <redacted>');
+    expect(environment.repo_provenance).toEqual({ repo: 'example/app', commit: 'abc123' });
+    expect(resultJson.environment_path).toBe('./environment.json');
+    expect(summary.metadata.environments?.[0]).toMatchObject({
+      type: 'host',
+      recipe_sha256: 'a'.repeat(64),
+    });
+  });
+
+  it('writes Docker environment provenance without setup logs in index rows', async () => {
+    const result = makeResult({
+      testId: 'docker-env',
+      environmentProvenance: {
+        schemaVersion: 'agentv.environment_provenance.v1',
+        authoredKind: 'inline',
+        recipeSha256: 'b'.repeat(64),
+        type: 'docker',
+        sourceDir: '/repo/evals',
+        workdir: '/app',
+        docker: {
+          context: '/repo/environment',
+          dockerfile: '/repo/environment/Dockerfile',
+          image: 'ghcr.io/example/app@sha256:1234567890abcdef',
+          imageDigest: 'sha256:1234567890abcdef',
+        },
+      },
+    });
+
+    const paths = await writeArtifactsFromResults([result], testDir, {
+      evalFile: 'evals/docker.eval.yaml',
+    });
+    const [indexLine] = await readIndexLines(paths.indexPath);
+    const environment = JSON.parse(
+      await readFile(path.join(testDir, indexLine.environment_path ?? ''), 'utf8'),
+    );
+
+    expect(indexLine.environment).toMatchObject({
+      type: 'docker',
+      workdir: '/app',
+      docker: {
+        context: '/repo/environment',
+        dockerfile: '/repo/environment/Dockerfile',
+        image: 'ghcr.io/example/app@sha256:1234567890abcdef',
+        image_digest: 'sha256:1234567890abcdef',
+      },
+    });
+    expect(indexLine.environment).not.toHaveProperty('setup_executions');
+    expect(environment).toMatchObject({
+      type: 'docker',
+      docker: {
+        image_digest: 'sha256:1234567890abcdef',
+      },
+    });
+  });
+
   it('does not write experiment config metadata into public run artifacts', async () => {
     const experimentMetadata = {
       name: 'native-exp',

apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx

@@ -45,6 +45,7 @@ The default local layout is:
         graders/
       sample-1/
         result.json
+        environment.json           # optional environment recipe provenance
         grading.json
         metrics.json
         target-execution.json     # optional target runtime envelope
@@ -97,6 +98,7 @@ reserved for rebuildable local state and are skipped by run discovery.
 | `summary.json` | Aggregate run metadata and rollups: run id, experiment label, tags, runtime source, counts, pass rate, score summaries, duration, token/cost totals, and writer metadata. | Listing runs, CI summaries, quick dashboards, trend cards, and validating that a run is complete enough to inspect. |
 | `.internal/index.jsonl` | Canonical per-run row index: one row per case/result aggregate, with identity fields, filter metadata, scores, status, and explicit run-relative paths to sidecars. | Filtering, compare/trend inputs, Dashboard detail routing, rerun/resume lookup, export adapters, and artifact discovery. |
 | `result.json` | Compact per-attempt manifest for one attempt directory, including AgentV `execution_status` and `verdict`. | Loading one attempt without scanning the whole run index. |
+| `environment.json` / `environment_path` | Redacted environment recipe provenance: authored inline/file reference, resolved recipe hash, host or Docker type, resolved workdir, setup command and typed args, setup log output/error, Docker context/image/digest fields when available, and repo provenance only when authored or emitted by setup. Index rows carry `environment_path` plus a compact `environment` summary; large setup logs stay in the sidecar. | Reproducing and reviewing the testbed without treating setup side effects as row metadata. Repository identity is opaque unless the environment recipe or setup output states it explicitly. |
 | `grading.json` | Grader outputs, `assertion_results`, rubric evidence, execution-metric grader facts, and scoring provenance. | Explaining why a row passed or failed. |
 | `metrics.json` | Duration, token usage, cost, execution status, trajectory, and derived executor behavior such as tool calls, files touched, shell commands, errors, turns, and output sizes. | Dashboard behavior views, cost/latency reporting, metric-style graders, adapter projections, and lightweight analysis. |
 | `target-execution.json` | Provider-neutral target runtime envelope, including command, cwd, timeout, exit code or signal, error kind, timestamps, log truncation metadata, and artifact paths. | Distinguishing target task failures, target crashes, timeouts, cancellation, malformed provider output, and sandbox/runner failures from AgentV orchestrator failures. |