feat(core): replay normalized transcripts (#1658)

Author: christso

apps/cli/src/commands/eval/run-eval.ts

@@ -2274,9 +2274,29 @@ export async function runEvalCommand(
       (sum, meta) => sum + meta.testCases.length,
       0,
     );
+    const evalTestIds = [...fileMetadata.values()].flatMap((meta) =>
+      meta.testCases.map((testCase) => testCase.id),
+    );
     if (transcriptProvider.lineCount !== totalTests) {
       throw new Error(
-        `Transcript has ${transcriptProvider.lineCount} entr${transcriptProvider.lineCount === 1 ? 'y' : 'ies'} but eval defines ${totalTests} test(s). Each transcript entry maps positionally to one test case.`,
+        `Transcript has ${transcriptProvider.lineCount} entr${transcriptProvider.lineCount === 1 ? 'y' : 'ies'} but eval defines ${totalTests} test(s). Each transcript entry must map to one test case by test_id.`,
+      );
+    }
+    const transcriptTestIds = new Set(transcriptProvider.testIds);
+    const evalTestIdSet = new Set(evalTestIds);
+    const missing = evalTestIds.filter((testId) => !transcriptTestIds.has(testId));
+    const extra = transcriptProvider.testIds.filter((testId) => !evalTestIdSet.has(testId));
+    if (missing.length > 0 || extra.length > 0) {
+      throw new Error(
+        [
+          'Transcript test_id values must match eval test ids for replay.',
+          missing.length > 0 ? `Missing transcript entries: ${missing.join(', ')}` : undefined,
+          extra.length > 0
+            ? `Transcript entries without eval tests: ${extra.join(', ')}`
+            : undefined,
+        ]
+          .filter((line): line is string => line !== undefined)
+          .join(' '),
       );
     }
 

apps/cli/src/commands/import/claude.ts

@@ -29,6 +29,16 @@ export const importClaudeCommand = command({
       description:
         'Output file path (default: .agentv/transcripts/claude-<session-id-short>.jsonl)',
     }),
+    testId: option({
+      type: optional(string),
+      long: 'test-id',
+      description: 'Set the transcript test_id to match an eval test id',
+    }),
+    target: option({
+      type: optional(string),
+      long: 'target',
+      description: 'Set the transcript target/source_target value (default: claude)',
+    }),
     projectsDir: option({
       type: optional(string),
       long: 'projects-dir',
@@ -39,7 +49,7 @@ export const importClaudeCommand = command({
       description: 'List available sessions instead of importing',
     }),
   },
-  handler: async ({ sessionId, projectPath, output, projectsDir, list }) => {
+  handler: async ({ sessionId, projectPath, output, testId, target, projectsDir, list }) => {
     if (list) {
       const sessions = await discoverClaudeSessions({
         projectPath,
@@ -95,7 +105,7 @@ export const importClaudeCommand = command({
     await mkdir(path.dirname(outputPath), { recursive: true });
 
     // Write transcript as JSONL (one message per line, grouped by test_id)
-    const jsonLines = toTranscriptJsonLines(transcript);
+    const jsonLines = toTranscriptJsonLines(transcript, transcriptWriteOptions(testId, target));
     await writeFile(
       outputPath,
       `${jsonLines.map((line) => JSON.stringify(line)).join('\n')}\n`,
@@ -131,6 +141,16 @@ function formatAge(date: Date): string {
   return `${diffDays}d ago`;
 }
 
+function transcriptWriteOptions(
+  testId: string | undefined,
+  target: string | undefined,
+): { testId?: string; target?: string } {
+  return {
+    ...(testId ? { testId } : {}),
+    ...(target ? { target } : {}),
+  };
+}
+
 function formatDurationMs(ms: number): string {
   if (ms < 1000) return `${ms}ms`;
   const seconds = Math.floor(ms / 1000);

apps/cli/src/commands/import/codex.ts

@@ -28,6 +28,16 @@ export const importCodexCommand = command({
       short: 'o',
       description: 'Output file path (default: .agentv/transcripts/codex-<timestamp>.jsonl)',
     }),
+    testId: option({
+      type: optional(string),
+      long: 'test-id',
+      description: 'Set the transcript test_id to match an eval test id',
+    }),
+    target: option({
+      type: optional(string),
+      long: 'target',
+      description: 'Set the transcript target/source_target value (default: codex)',
+    }),
     sessionsDir: option({
       type: optional(string),
       long: 'sessions-dir',
@@ -38,7 +48,7 @@ export const importCodexCommand = command({
       description: 'List available sessions instead of importing',
     }),
   },
-  handler: async ({ sessionId, date, output, sessionsDir, list }) => {
+  handler: async ({ sessionId, date, output, testId, target, sessionsDir, list }) => {
     if (list) {
       const sessions = await discoverCodexSessions({
         date,
@@ -92,7 +102,7 @@ export const importCodexCommand = command({
     await mkdir(path.dirname(outputPath), { recursive: true });
 
     // Write transcript as JSONL (one message per line, grouped by test_id)
-    const jsonLines = toTranscriptJsonLines(transcript);
+    const jsonLines = toTranscriptJsonLines(transcript, transcriptWriteOptions(testId, target));
     await writeFile(
       outputPath,
       `${jsonLines.map((line) => JSON.stringify(line)).join('\n')}\n`,
@@ -123,6 +133,16 @@ function formatAge(date: Date): string {
   return `${diffDays}d ago`;
 }
 
+function transcriptWriteOptions(
+  testId: string | undefined,
+  target: string | undefined,
+): { testId?: string; target?: string } {
+  return {
+    ...(testId ? { testId } : {}),
+    ...(target ? { target } : {}),
+  };
+}
+
 function formatDurationMs(ms: number): string {
   if (ms < 1000) return `${ms}ms`;
   const seconds = Math.floor(ms / 1000);

apps/cli/src/commands/import/copilot.ts

@@ -19,6 +19,16 @@ export const importCopilotCommand = command({
       description:
         'Output file path (default: .agentv/transcripts/copilot-<session-id-short>.jsonl)',
     }),
+    testId: option({
+      type: optional(string),
+      long: 'test-id',
+      description: 'Set the transcript test_id to match an eval test id',
+    }),
+    target: option({
+      type: optional(string),
+      long: 'target',
+      description: 'Set the transcript target/source_target value (default: copilot)',
+    }),
     sessionStateDir: option({
       type: optional(string),
       long: 'session-state-dir',
@@ -29,7 +39,7 @@ export const importCopilotCommand = command({
       description: 'List available sessions instead of importing',
     }),
   },
-  handler: async ({ sessionId, output, sessionStateDir, list }) => {
+  handler: async ({ sessionId, output, testId, target, sessionStateDir, list }) => {
     if (list) {
       const sessions = await discoverCopilotSessions({
         sessionStateDir,
@@ -100,7 +110,7 @@ export const importCopilotCommand = command({
     await mkdir(path.dirname(outputPath), { recursive: true });
 
     // Write transcript as JSONL (one message per line, grouped by test_id)
-    const jsonLines = toTranscriptJsonLines(transcript);
+    const jsonLines = toTranscriptJsonLines(transcript, transcriptWriteOptions(testId, target));
     await writeFile(
       outputPath,
       `${jsonLines.map((line) => JSON.stringify(line)).join('\n')}\n`,
@@ -136,6 +146,16 @@ function formatAge(date: Date): string {
   return `${diffDays}d ago`;
 }
 
+function transcriptWriteOptions(
+  testId: string | undefined,
+  target: string | undefined,
+): { testId?: string; target?: string } {
+  return {
+    ...(testId ? { testId } : {}),
+    ...(target ? { target } : {}),
+  };
+}
+
 function formatDurationMs(ms: number): string {
   if (ms < 1000) return `${ms}ms`;
   const seconds = Math.floor(ms / 1000);

apps/web/src/content/docs/docs/next/guides/skill-improvement-workflow.mdx

@@ -132,7 +132,7 @@ agentv import claude --list
 agentv import claude --session-id <uuid>
 
 # Run deterministic graders against the imported transcript
-agentv eval EVAL.yaml --target copilot-log
+agentv eval EVAL.yaml --transcript .agentv/transcripts/claude-<session-id-short>.jsonl
 ```
 
 Offline grading is useful when you want to evaluate skills with agents that don't have a direct API integration — import the session transcript and run deterministic graders.

apps/web/src/content/docs/docs/next/integrations/agent-skills-evals.mdx

@@ -122,7 +122,7 @@ Grade existing agent sessions offline by importing transcripts and running the a
 agentv import claude --list
 agentv import claude --session-id <uuid>
 
-agentv eval evals.json --target copilot-log
+agentv eval evals.json --transcript .agentv/transcripts/claude-<session-id-short>.jsonl
 ```
 
 If another tool owns the original `evals.json`, keep that file as the source and run it through the read adapter. Convert only when you need to edit the AgentV-native form.

apps/web/src/content/docs/docs/next/targets/coding-agents.mdx

@@ -110,7 +110,6 @@ targets:
 | `claude-cli` | Claude CLI subprocess. | Default Claude path; captures structured stream output when available. |
 | `claude-sdk` | Claude Agent SDK in an AgentV child runner. | Explicit SDK path; useful when SDK-native events matter more than matching a local CLI invocation. |
 | `copilot-cli` | Copilot CLI subprocess/protocol path. | Active Copilot eval run through the installed process. |
-| `copilot-log` | Passive Copilot session-log reader. | Zero-cost transcript grading for existing sessions; it does not run a new agent. |
 | `copilot-sdk` | Copilot SDK in an AgentV child runner. | Explicit SDK path with child-process isolation. |
 
 Every coding-agent provider returns a structured target execution envelope.
@@ -271,15 +270,20 @@ targets:
       api_format: responses
 ```
 
-Read an existing Copilot session log without running a new agent:
+Replay an existing Copilot session without running a new agent by importing the
+native log into AgentV transcript rows, then using the generic replay target:
+
+```bash
+agentv import copilot --session-id <uuid> -o .agentv/transcripts/copilot-case.jsonl
+```
 
 ```yaml
 targets:
-  - id: copilot-session-log
-    provider: copilot-log
-    runtime: host
+  - id: copilot-session-replay
+    provider: replay
     config:
-      discover: latest
+      transcripts: .agentv/transcripts/copilot-case.jsonl
+      source_target: copilot-cli
 ```
 
 Use `copilot-sdk` only when you intentionally want the SDK path:
@@ -295,8 +299,9 @@ targets:
 
 Copilot config fields include `command`, `model`, `cwd`, `timeout_seconds`,
 `subprovider`, `base_url`, `api_key`, `bearer_token`, `api_version`,
-`api_format`, `log_dir`, `stream_log`, `system_prompt`, and session-log fields
-such as `discover`, `session_id`, and `session_dir` for `copilot-log`.
+`api_format`, `log_dir`, `stream_log`, and `system_prompt`. Copilot
+`events.jsonl` parsing is available through `agentv import copilot`; graders
+and Dashboard views consume normalized AgentV transcript/replay artifacts.
 
 ## File inputs
 

apps/web/src/content/docs/docs/next/targets/configuration.mdx

@@ -206,7 +206,6 @@ already-exported secrets into `.env`.
 | `codex-app-server` | Agent | Codex app-server subprocess |
 | `codex-sdk` | Agent | Codex SDK in an isolated child runner |
 | `copilot-cli` | Agent | Copilot CLI subprocess |
-| `copilot-log` | Agent | Passive Copilot CLI session log reader |
 | `copilot-sdk` | Agent | Copilot SDK in an isolated child runner |
 | `pi-sdk` | Agent | Pi SDK in an isolated child runner |
 | `pi-cli` | Agent | Pi CLI subprocess |

apps/web/src/content/docs/docs/next/tools/import.mdx

@@ -108,6 +108,8 @@ The transcript providers share the same core flags:
 | `--session-id <uuid>` | Import a specific session by UUID |
 | `--list` | List available sessions instead of importing |
 | `--output, -o <path>` | Custom output file path |
+| `--test-id <id>` | Set the transcript `test_id` to match an eval test id |
+| `--target <id>` | Set the transcript `target` / replay `source_target` value |
 
 Provider-specific flags:
 
@@ -189,19 +191,36 @@ Token usage is aggregated from the final cumulative value per LLM request. Durat
 
 ## Workflow
 
-Import a session, then run graders against it:
+Import a session, then run graders against it. The transcript `test_id` values
+must match the eval test ids so replay cannot silently grade the wrong
+trajectory.
 
 ```bash
 # 1. List sessions and pick one
 agentv import claude --list
 
 # 2. Import a session by ID
-agentv import claude --session-id 4c4f9e4e-e6f1-490b-a1b1-9aef543ebf22
+agentv import claude --session-id 4c4f9e4e-e6f1-490b-a1b1-9aef543ebf22 --test-id my-case
 
 # 3. Run graders against the imported transcript
 agentv eval evals/my-eval.yaml --transcript .agentv/transcripts/claude-4c4f9e4e.jsonl
 ```
 
+You can also use the same normalized transcript rows through a replay target:
+
+```yaml
+targets:
+  - id: recorded-agent
+    provider: replay
+    transcripts: .agentv/transcripts/claude-4c4f9e4e.jsonl
+    source_target: claude-cli
+```
+
+Replay targets match by `test_id` and `source_target`, return the recorded
+AgentV `Message[]` trajectory, and run graders fresh. Raw provider logs remain
+provenance/debug input for importers; graders and Dashboard views consume the
+normalized transcript/replay artifacts.
+
 See `examples/features/import-claude/` for a complete working example.
 
 ## HuggingFace Datasets (SWE-bench)

apps/web/src/content/docs/docs/v4.42.4/guides/skill-improvement-workflow.mdx

@@ -130,7 +130,7 @@ agentv import claude --list
 agentv import claude --session-id <uuid>
 
 # Run deterministic graders against the imported transcript
-agentv eval evals.json --target copilot-log
+agentv eval evals.json --transcript .agentv/transcripts/claude-<session-id-short>.jsonl
 ```
 
 Offline grading is useful when you want to evaluate skills with agents that don't have a direct API integration — import the session transcript and run deterministic graders.