fix(artifacts): flatten metrics sidecar (#1685)

Author: christso

apps/cli/src/commands/results/manifest.ts

@@ -4,11 +4,13 @@ import path from 'node:path';
 import {
   type EvaluationResult,
   type ExternalTraceMetadataWire,
+  type MetricsArtifactWire,
   type ResultArtifactPointersWire,
   type RunRuntimeSourceMetadata,
   type TraceSummary,
   buildTraceFromMessages,
   fromTraceEnvelopeWire,
+  normalizeMetricsArtifactWire,
   toCamelCaseDeep,
   traceEnvelopeToTraceSummary,
   traceEnvelopeToTranscriptMessages,
@@ -88,20 +90,6 @@ export interface ResultManifestRecord {
   readonly metadata?: Record<string, unknown>;
 }
 
-interface MetricsUsageArtifact {
-  readonly duration?: {
-    readonly total_ms?: number;
-  };
-  readonly tokens?: {
-    readonly input?: number;
-    readonly output?: number;
-    readonly reasoning?: number;
-  };
-  readonly cost?: {
-    readonly usd?: number | null;
-  };
-}
-
 interface LegacyTimingArtifact {
   readonly duration_ms?: number;
   readonly token_usage?: {
@@ -390,7 +378,9 @@ function hydrateManifestRecord(
   options: ManifestHydrationOptions,
 ): EvaluationResult {
   const grading = readOptionalJson<GradingArtifact>(baseDir, record.grading_path);
-  const metrics = readOptionalJson<MetricsUsageArtifact>(baseDir, record.metrics_path);
+  const metrics = normalizeMetricsArtifactWire(
+    readOptionalJson<MetricsArtifactWire | Record<string, unknown>>(baseDir, record.metrics_path),
+  );
   const timing = metrics ?? readOptionalJson<LegacyTimingArtifact>(baseDir, record.timing_path);
   const testId = record.test_id ?? 'unknown';
   const gradingAssertions = grading

apps/cli/src/commands/results/serve.ts

@@ -62,6 +62,7 @@ import {
   loadConfig,
   loadProjectRegistry,
   normalizeCategoryPath,
+  normalizeMetricsArtifactWire,
   normalizeTraceArtifactToTraceSessionResponse,
   omitExternalTraceMetadataKeys,
   readGitResultArtifact,
@@ -1154,7 +1155,7 @@ function buildRepeatTrialReadModels(
     const answerPath = caseTrialArtifactPath(resultDir, runPath, 'outputs/answer.md');
     const resultPath = caseTrialArtifactPath(resultDir, runPath, 'result.json');
     const runResult = readArtifactJsonObject(baseDir, resultPath);
-    const metrics = readArtifactJsonObject(baseDir, metricsPath);
+    const metrics = normalizeMetricsArtifactWire(readArtifactJsonObject(baseDir, metricsPath));
     const timing = readArtifactJsonObject(baseDir, timingPath);
     const toolCalls = objectField(metrics, 'tool_calls');
     const tokenUsage = objectField(metrics, 'tokens') ?? objectField(timing, 'token_usage');

apps/cli/src/commands/results/validate.ts

@@ -18,6 +18,11 @@
 import { existsSync, readFileSync, statSync } from 'node:fs';
 import path from 'node:path';
 
+import {
+  LEGACY_METRICS_SCHEMA_VERSION,
+  METRICS_SCHEMA_VERSION,
+  normalizeMetricsArtifactWire,
+} from '@agentv/core';
 import { command, positional, string } from 'cmd-ts';
 
 import { RESULT_INDEX_FILENAME, resolveExistingRunPrimaryPath } from '../eval/result-layout.js';
@@ -114,6 +119,50 @@ function validateNoLegacyGradingFields(value: unknown, pathLabel: string): strin
   return errors;
 }
 
+function validateMetricsArtifact(value: unknown): Diagnostic[] {
+  if (!isRecord(value)) {
+    return [{ severity: 'error', message: 'metrics.json must be an object' }];
+  }
+
+  const diagnostics: Diagnostic[] = [];
+  const schemaVersion = value.schema_version;
+  const isCurrentSchema = schemaVersion === METRICS_SCHEMA_VERSION;
+  const isLegacySchema = schemaVersion === LEGACY_METRICS_SCHEMA_VERSION;
+
+  if (isCurrentSchema && Object.hasOwn(value, 'trace')) {
+    diagnostics.push({
+      severity: 'error',
+      message: 'metrics.json must not include trace in agentv.metrics.v2',
+    });
+  } else if (isLegacySchema && Object.hasOwn(value, 'trace')) {
+    diagnostics.push({
+      severity: 'warning',
+      message: 'metrics.json uses legacy trace identity; readers ignore it',
+    });
+  }
+
+  if (isCurrentSchema && Object.hasOwn(value, 'metrics')) {
+    diagnostics.push({
+      severity: 'error',
+      message: 'metrics.json must not include nested metrics in agentv.metrics.v2',
+    });
+  } else if (isLegacySchema && Object.hasOwn(value, 'metrics')) {
+    diagnostics.push({
+      severity: 'warning',
+      message: 'metrics.json uses legacy nested metrics; readers flatten it',
+    });
+  }
+
+  if (!normalizeMetricsArtifactWire(value)) {
+    diagnostics.push({
+      severity: 'error',
+      message: 'metrics.json does not match a supported metrics artifact shape',
+    });
+  }
+
+  return diagnostics;
+}
+
 // ── Checks ───────────────────────────────────────────────────────────────
 
 function checkDirectoryNaming(runDir: string): Diagnostic[] {
@@ -372,6 +421,21 @@ function checkArtifactFiles(runDir: string, entries: IndexEntry[]): Diagnostic[]
           severity: 'warning',
           message: `${testId}: metrics.json not found at '${entry.metrics_path}'`,
         });
+      } else {
+        try {
+          const metrics = JSON.parse(readFileSync(metricsPath, 'utf8'));
+          for (const diagnostic of validateMetricsArtifact(metrics)) {
+            diagnostics.push({
+              severity: diagnostic.severity,
+              message: `${testId}: ${diagnostic.message}`,
+            });
+          }
+        } catch {
+          diagnostics.push({
+            severity: 'error',
+            message: `${testId}: metrics.json is not valid JSON`,
+          });
+        }
       }
     }
   }

apps/cli/test/commands/eval/artifact-writer.test.ts

@@ -2019,12 +2019,8 @@ describe('writeArtifactsFromResults', () => {
     );
 
     expect(summary.schema_version).toBe(METRICS_SCHEMA_VERSION);
-    expect(summary.trace).toMatchObject({
-      schema_version: 'agentv.trace.v1',
-      trace_id: expect.any(String),
-      root_span_id: expect.any(String),
-    });
-    expect(summary.trace).not.toHaveProperty('path');
+    expect(summary).not.toHaveProperty('trace');
+    expect(summary).not.toHaveProperty('metrics');
     expect(summary.source_artifacts).toMatchObject({
       transcript_path: 'transcript.json',
       grading_path: 'grading.json',
@@ -2034,29 +2030,29 @@ describe('writeArtifactsFromResults', () => {
     await expect(
       readFile(runArtifactPath(testDir, indexLine, 'sample-1', 'trace.json'), 'utf8'),
     ).rejects.toThrow();
-    expect(summary.metrics.total_turns).toBe(2);
-    expect(summary.metrics.total_tool_calls).toBe(4);
-    expect(summary.metrics.total_steps).toBe(2);
-    expect(summary.metrics.tool_calls).toMatchObject({
+    expect(summary.total_turns).toBe(2);
+    expect(summary.total_tool_calls).toBe(4);
+    expect(summary.total_steps).toBe(2);
+    expect(summary.tool_calls).toMatchObject({
       Read: 1,
       Bash: 1,
       WebFetch: 1,
       Edit: 1,
     });
-    expect(summary.metrics.tool_call_counts).toMatchObject({
+    expect(summary.tool_call_counts).toMatchObject({
       Read: 1,
       Bash: 1,
       WebFetch: 1,
       Edit: 1,
     });
-    expect(summary.metrics.tool_category_counts).toMatchObject({
+    expect(summary.tool_category_counts).toMatchObject({
       file_read: 1,
       shell: 1,
       web_fetch: 1,
       file_edit: 1,
     });
-    expect(summary.metrics.tool_call_events).toHaveLength(4);
-    expect(summary.metrics.shell_commands).toEqual([
+    expect(summary.tool_call_events).toHaveLength(4);
+    expect(summary.shell_commands).toEqual([
       {
         command: 'bun test apps/cli/test/commands/eval/artifact-writer.test.ts',
         tool_call_id: 'bash-1',
@@ -2065,25 +2061,25 @@ describe('writeArtifactsFromResults', () => {
         duration_ms: 1200,
       },
     ]);
-    expect(summary.metrics.files_read).toContainEqual({
+    expect(summary.files_read).toContainEqual({
       path: 'src/input.ts',
       tool_call_id: 'read-1',
       source: 'tool_input',
     });
-    expect(summary.metrics.files_modified).toContainEqual({
+    expect(summary.files_modified).toContainEqual({
       path: 'src/output.ts',
       operation: 'edit',
       tool_call_id: 'edit-1',
       source: 'tool_input',
     });
-    expect(summary.metrics.files_modified).toContainEqual({
+    expect(summary.files_modified).toContainEqual({
       path: 'src/output.ts',
       operation: 'workspace_diff',
       source: 'file_changes',
     });
-    expect(summary.metrics.files_created).toEqual(['src/new.ts']);
-    expect(summary.metrics.files_deleted).toEqual(['src/gone.ts']);
-    expect(summary.metrics.web_fetches).toEqual([
+    expect(summary.files_created).toEqual(['src/new.ts']);
+    expect(summary.files_deleted).toEqual(['src/gone.ts']);
+    expect(summary.web_fetches).toEqual([
       {
         url: 'https://example.com/spec',
         method: 'GET',
@@ -2092,17 +2088,14 @@ describe('writeArtifactsFromResults', () => {
         tool_call_id: 'web-1',
       },
     ]);
-    expect(summary.metrics.errors).toContainEqual({
+    expect(summary.errors).toContainEqual({
       message: 'quality gate failed',
     });
-    expect(summary.metrics.errors_encountered).toBe(1);
-    expect(summary.metrics.output_chars).toBe('Editing output.'.length);
-    expect(summary.metrics.transcript_chars).toBeGreaterThan(0);
-    expect(summary.metrics.thinking_blocks).toBe(2);
-    expect(summary.metrics.reasoning_blocks.map((block) => block.kind)).toEqual([
-      'reasoning',
-      'thinking',
-    ]);
+    expect(summary.errors_encountered).toBe(1);
+    expect(summary.output_chars).toBe('Editing output.'.length);
+    expect(summary.transcript_chars).toBeGreaterThan(0);
+    expect(summary.thinking_blocks).toBe(2);
+    expect(summary.reasoning_blocks.map((block) => block.kind)).toEqual(['reasoning', 'thinking']);
     expect(summary).not.toHaveProperty('usage_summary');
 
     expect(summary).toMatchObject({

apps/cli/test/commands/results/export-e2e-providers.test.ts

@@ -532,9 +532,11 @@ describe('export e2e — multi-provider metrics verification', () => {
           'utf8',
         ),
       );
-      expect(metrics.metrics.total_tool_calls).toBe(3);
-      expect(metrics.metrics.tool_call_counts.Read).toBe(2);
-      expect(metrics.metrics.tool_call_counts.Write).toBe(1);
+      expect(metrics).not.toHaveProperty('trace');
+      expect(metrics).not.toHaveProperty('metrics');
+      expect(metrics.total_tool_calls).toBe(3);
+      expect(metrics.tool_call_counts.Read).toBe(2);
+      expect(metrics.tool_call_counts.Write).toBe(1);
 
       expect(grading.component_results?.[0].component_results?.[0].assertion?.value).toBe(
         'Contains 42',
@@ -559,7 +561,9 @@ describe('export e2e — multi-provider metrics verification', () => {
       const metrics = JSON.parse(
         readFileSync(path.join(runArtifactDir(outputDir, COPILOT_RESULT), 'metrics.json'), 'utf8'),
       );
-      expect(metrics.metrics.total_tool_calls).toBe(0);
+      expect(metrics).not.toHaveProperty('trace');
+      expect(metrics).not.toHaveProperty('metrics');
+      expect(metrics.total_tool_calls).toBe(0);
     });
 
     it('should handle error result in grading', async () => {
@@ -577,7 +581,9 @@ describe('export e2e — multi-provider metrics verification', () => {
       const metrics = JSON.parse(
         readFileSync(path.join(runArtifactDir(outputDir, ERROR_RESULT), 'metrics.json'), 'utf8'),
       );
-      expect(metrics.metrics.errors_encountered).toBe(1);
+      expect(metrics).not.toHaveProperty('trace');
+      expect(metrics).not.toHaveProperty('metrics');
+      expect(metrics.errors_encountered).toBe(1);
     });
 
     it('should produce grading files for all test IDs in multi-target run', async () => {

apps/cli/test/commands/results/serve.test.ts

@@ -2293,6 +2293,80 @@ describe('serve app', () => {
         path.join(runDir, resultDir, 'attempt-2', 'result.json'),
         `${JSON.stringify({ transcript_summary: secondSummary })}\n`,
       );
+      writeFileSync(
+        path.join(runDir, resultDir, 'attempt-1', 'metrics.json'),
+        `${JSON.stringify({
+          schema_version: 'agentv.metrics.v2',
+          artifact_id: 'metrics-repeat-case-1',
+          generated_at: '2026-03-25T10:05:00.000Z',
+          test_id: 'repeat-case',
+          target: 'gpt-4o',
+          source_artifacts: {},
+          duration: { total_ms: 1000, total_seconds: 1, source: 'provider_reported' },
+          tokens: { total: 3, input: 1, output: 2, reasoning: 0, source: 'provider_reported' },
+          cost: { usd: 0.01, source: 'provider_reported' },
+          tool_calls: { Bash: 1 },
+          tool_call_counts: { Bash: 1 },
+          tool_category_counts: { shell: 1 },
+          total_tool_calls: 1,
+          total_steps: 1,
+          total_turns: 1,
+          tool_call_events: [],
+          shell_commands: [],
+          files_read: [],
+          files_modified: [],
+          files_created: [],
+          files_deleted: [],
+          web_fetches: [],
+          errors: [],
+          errors_encountered: 0,
+          output_chars: 0,
+          transcript_chars: 0,
+          reasoning_blocks: [],
+          thinking_blocks: 0,
+        })}\n`,
+      );
+      writeFileSync(
+        path.join(runDir, resultDir, 'attempt-2', 'metrics.json'),
+        `${JSON.stringify({
+          schema_version: 'agentv.metrics.v1',
+          artifact_id: 'metrics-repeat-case-2',
+          generated_at: '2026-03-25T10:05:01.000Z',
+          test_id: 'repeat-case',
+          target: 'gpt-4o',
+          trace: {
+            schema_version: 'agentv.trace.v1',
+            artifact_id: 'execution-trace-repeat-case-2',
+            trace_id: 'trace-repeat-case-2',
+            root_span_id: 'span-repeat-case-2',
+          },
+          source_artifacts: {},
+          duration: { total_ms: 2000, total_seconds: 2, source: 'provider_reported' },
+          tokens: { total: 7, input: 3, output: 4, reasoning: 0, source: 'provider_reported' },
+          cost: { usd: 0.02, source: 'provider_reported' },
+          metrics: {
+            tool_calls: { Read: 2 },
+            tool_call_counts: { Read: 2 },
+            tool_category_counts: { file_read: 2 },
+            total_tool_calls: 2,
+            total_steps: 1,
+            total_turns: 1,
+            tool_call_events: [],
+            shell_commands: [],
+            files_read: [],
+            files_modified: [],
+            files_created: [],
+            files_deleted: [],
+            web_fetches: [],
+            errors: [],
+            errors_encountered: 0,
+            output_chars: 0,
+            transcript_chars: 0,
+            reasoning_blocks: [],
+            thinking_blocks: 0,
+          },
+        })}\n`,
+      );
       writeFileSync(
         path.join(runDir, 'index.jsonl'),
         toJsonl({
@@ -2320,6 +2394,8 @@ describe('serve app', () => {
           attempts?: Array<{
             transcript_path?: string;
             transcript_summary?: Record<string, unknown>;
+            total_tool_calls?: number;
+            tool_calls?: Record<string, number>;
           }>;
         }>;
       };
@@ -2330,6 +2406,9 @@ describe('serve app', () => {
         `${resultDir}/attempt-1/transcript.json`,
         `${resultDir}/attempt-2/transcript.json`,
       ]);
+      expect(data.results[0]?.attempts?.map((trial) => trial.total_tool_calls)).toEqual([1, 2]);
+      expect(data.results[0]?.attempts?.[0]?.tool_calls).toEqual({ Bash: 1 });
+      expect(data.results[0]?.attempts?.[1]?.tool_calls).toEqual({ Read: 2 });
     });
 
     it('loads historical runs without test bundle metadata', async () => {