Make AgentCI diffs easier to debug at the step level

AgentCI could tell users that an episode changed, but the public backlog
correctly identified that regression debugging was still too flat and too
shallow. This change adds structured step-level diff items, carries them
through regression results and JSON output, and exposes field-level step
changes in the HTML report so users can see exactly where a candidate run
starts to diverge.

Constraint: Preserve existing text diff prefixes so ignore rules such as metric:* keep working
Rejected: Replace flat diff items entirely with structured output | too disruptive for current CLI and regression consumers
Rejected: Limit the enhancement to HTML only | lower value than fixing the core compare pipeline once
Confidence: high
Scope-risk: narrow
Reversibility: clean
Directive: Keep future diff enrichments additive and compatibility-aware; downstream tooling may already depend on diff item prefixes
Tested: AgentCI unittest suite; direct CLI validation for diff JSON, assert-regression output, and generated HTML report field-level change rendering
Not-tested: Very large deeply nested payload diffs across long episodes

Author: weiyi

projects/agentci/src/agentci/compare.py

@@ -6,15 +6,47 @@
 from .schema import Episode
 
 
+@dataclass
+class StepDiffItem:
+    step_index: int
+    kind: str
+    name: str
+    field_path: str
+    left: Any
+    right: Any
+    change_type: str = "changed"
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "step_index": self.step_index,
+            "kind": self.kind,
+            "name": self.name,
+            "field_path": self.field_path,
+            "left": self.left,
+            "right": self.right,
+            "change_type": self.change_type,
+        }
+
+    def to_text(self) -> str:
+        prefix = f"step {self.step_index} [{self.kind}:{self.name}] {self.field_path}"
+        if self.change_type == "added":
+            return f"{prefix}: <missing> -> {self.right!r}"
+        if self.change_type == "removed":
+            return f"{prefix}: {self.left!r} -> <missing>"
+        return f"{prefix}: {self.left!r} -> {self.right!r}"
+
+
 @dataclass
 class EpisodeDiff:
     changed: bool
     items: list[str] = field(default_factory=list)
+    step_items: list[StepDiffItem] = field(default_factory=list)
 
     def to_dict(self) -> dict[str, object]:
         return {
             "changed": self.changed,
             "items": list(self.items),
+            "step_items": [item.to_dict() for item in self.step_items],
         }
 
 
@@ -23,32 +55,116 @@ def _append_if_changed(items: list[str], label: str, left: Any, right: Any) -> N
         items.append(f"{label}: {left!r} -> {right!r}")
 
 
+def _append_nested_step_diffs(
+    step_items: list[StepDiffItem],
+    *,
+    step_index: int,
+    kind: str,
+    name: str,
+    field_path: str,
+    left: Any,
+    right: Any,
+) -> None:
+    if left == right:
+        return
+    if isinstance(left, dict) and isinstance(right, dict):
+        for key in sorted(set(left) | set(right)):
+            child_path = f"{field_path}.{key}" if field_path else str(key)
+            _append_nested_step_diffs(
+                step_items,
+                step_index=step_index,
+                kind=kind,
+                name=name,
+                field_path=child_path,
+                left=left.get(key),
+                right=right.get(key),
+            )
+        return
+    step_items.append(
+        StepDiffItem(
+            step_index=step_index,
+            kind=kind,
+            name=name,
+            field_path=field_path,
+            left=left,
+            right=right,
+        )
+    )
+
+
 def compare_episodes(baseline: Episode, candidate: Episode) -> EpisodeDiff:
     items: list[str] = []
+    step_items: list[StepDiffItem] = []
 
     _append_if_changed(items, "model", baseline.model, candidate.model)
     _append_if_changed(items, "prompt_version", baseline.prompt_version, candidate.prompt_version)
     _append_if_changed(items, "success", baseline.success, candidate.success)
     _append_if_changed(items, "final_output", baseline.final_output, candidate.final_output)
     _append_if_changed(items, "step_count", len(baseline.steps), len(candidate.steps))
 
-    max_steps = min(len(baseline.steps), len(candidate.steps))
+    max_steps = max(len(baseline.steps), len(candidate.steps))
     for index in range(max_steps):
-        left = baseline.steps[index]
-        right = candidate.steps[index]
+        left = baseline.steps[index] if index < len(baseline.steps) else None
+        right = candidate.steps[index] if index < len(candidate.steps) else None
+        if left is None and right is not None:
+            step_items.append(
+                StepDiffItem(
+                    step_index=index + 1,
+                    kind=right.kind,
+                    name=right.name,
+                    field_path="step",
+                    left=None,
+                    right=right.payload,
+                    change_type="added",
+                )
+            )
+            continue
+        if left is not None and right is None:
+            step_items.append(
+                StepDiffItem(
+                    step_index=index + 1,
+                    kind=left.kind,
+                    name=left.name,
+                    field_path="step",
+                    left=left.payload,
+                    right=None,
+                    change_type="removed",
+                )
+            )
+            continue
+        assert left is not None and right is not None
         if left.kind != right.kind:
-            items.append(f"step {index + 1} kind: {left.kind!r} -> {right.kind!r}")
+            step_items.append(
+                StepDiffItem(
+                    step_index=index + 1,
+                    kind=left.kind,
+                    name=left.name,
+                    field_path="kind",
+                    left=left.kind,
+                    right=right.kind,
+                )
+            )
             continue
         if left.name != right.name:
-            items.append(f"step {index + 1} name: {left.name!r} -> {right.name!r}")
-        if left.payload != right.payload:
-            all_payload_keys = sorted(set(left.payload) | set(right.payload))
-            for key in all_payload_keys:
-                if left.payload.get(key) != right.payload.get(key):
-                    items.append(
-                        f"step {index + 1} payload.{key}: "
-                        f"{left.payload.get(key)!r} -> {right.payload.get(key)!r}"
-                    )
+            step_items.append(
+                StepDiffItem(
+                    step_index=index + 1,
+                    kind=left.kind,
+                    name=left.name,
+                    field_path="name",
+                    left=left.name,
+                    right=right.name,
+                )
+            )
+        _append_nested_step_diffs(
+            step_items,
+            step_index=index + 1,
+            kind=left.kind,
+            name=left.name,
+            field_path="payload",
+            left=left.payload,
+            right=right.payload,
+        )
 
     all_metric_keys = sorted(set(baseline.metrics) | set(candidate.metrics))
     for key in all_metric_keys:
@@ -59,4 +175,5 @@ def compare_episodes(baseline: Episode, candidate: Episode) -> EpisodeDiff:
             candidate.metrics.get(key),
         )
 
-    return EpisodeDiff(changed=bool(items), items=items)
+    items.extend(item.to_text() for item in step_items)
+    return EpisodeDiff(changed=bool(items), items=items, step_items=step_items)

projects/agentci/src/agentci/html_report.py

@@ -4,32 +4,43 @@
 import json
 from pathlib import Path
 
-from .compare import compare_episodes
+from .compare import StepDiffItem, compare_episodes
 from .schema import Episode, EpisodeStep
 
 
 def _pretty_json(value: object) -> str:
     return json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False)
 
 
-def _step_rows(baseline: Episode, candidate: Episode) -> str:
+def _step_rows(baseline: Episode, candidate: Episode, step_items: list[StepDiffItem]) -> str:
     rows: list[str] = []
+    grouped: dict[int, list[StepDiffItem]] = {}
+    for item in step_items:
+        grouped.setdefault(item.step_index, []).append(item)
     max_steps = max(len(baseline.steps), len(candidate.steps))
     for index in range(max_steps):
         left = baseline.steps[index] if index < len(baseline.steps) else None
         right = candidate.steps[index] if index < len(candidate.steps) else None
-        changed = left != right
+        item_group = grouped.get(index + 1, [])
+        changed = left != right or bool(item_group)
         rows.append(
             "<tr>"
             f"<td>{index + 1}</td>"
             f"<td>{_render_step(left)}</td>"
             f"<td>{_render_step(right)}</td>"
-            f"<td><span class='badge {'changed' if changed else 'same'}'>{'changed' if changed else 'same'}</span></td>"
+            f"<td><span class='badge {'changed' if changed else 'same'}'>{'changed' if changed else 'same'}</span>{_render_step_diff_items(item_group)}</td>"
             "</tr>"
         )
     return "\n".join(rows)
 
 
+def _render_step_diff_items(items: list[StepDiffItem]) -> str:
+    if not items:
+        return ""
+    details = "".join(f"<li><code>{escape(item.to_text())}</code></li>" for item in items)
+    return f"<details><summary>field-level changes</summary><ul>{details}</ul></details>"
+
+
 def _render_step(step: EpisodeStep | None) -> str:
     if step is None:
         return "<span class='muted'>missing</span>"
@@ -241,7 +252,7 @@ def render_diff_html_report(baseline: Episode, candidate: Episode) -> str:
           </tr>
         </thead>
         <tbody>
-          {_step_rows(baseline, candidate)}
+          {_step_rows(baseline, candidate, diff.step_items)}
         </tbody>
       </table>
     </section>

projects/agentci/src/agentci/regression.py

@@ -3,7 +3,7 @@
 from dataclasses import dataclass, field
 from pathlib import Path
 
-from .compare import compare_episodes
+from .compare import StepDiffItem, compare_episodes
 from .replay import replay_episode
 from .schema import Episode
 
@@ -14,6 +14,7 @@ class RegressionResult:
     baseline_path: str | None = None
     candidate_path: str | None = None
     diff_items: list[str] = field(default_factory=list)
+    step_items: list[StepDiffItem] = field(default_factory=list)
     replay_mismatches: list[str] = field(default_factory=list)
 
     def to_dict(self) -> dict[str, object]:
@@ -22,6 +23,7 @@ def to_dict(self) -> dict[str, object]:
             "baseline_path": self.baseline_path,
             "candidate_path": self.candidate_path,
             "diff_items": list(self.diff_items),
+            "step_items": [item.to_dict() for item in self.step_items],
             "replay_mismatches": list(self.replay_mismatches),
         }
 
@@ -33,6 +35,9 @@ def failure_message(self) -> str:
         if self.diff_items:
             lines.append("- diff:")
             lines.extend(f"  - {item}" for item in self.diff_items)
+        if self.step_items:
+            lines.append("- step details:")
+            lines.extend(f"  - {item.to_text()}" for item in self.step_items)
         if self.replay_mismatches:
             lines.append("- replay mismatches:")
             lines.extend(f"  - {item}" for item in self.replay_mismatches)
@@ -58,6 +63,7 @@ def run_regression_check(
 ) -> RegressionResult:
     diff = compare_episodes(baseline, candidate)
     kept_items = [item for item in diff.items if _keep_item(item, ignore_diff_prefixes)]
+    kept_step_items = [item for item in diff.step_items if _keep_item(item.to_text(), ignore_diff_prefixes)]
     replay_mismatches: list[str] = []
     if check_candidate_replay:
         replay = replay_episode(candidate, strict=True)
@@ -68,6 +74,7 @@ def run_regression_check(
         baseline_path=baseline_path,
         candidate_path=candidate_path,
         diff_items=kept_items,
+        step_items=kept_step_items,
         replay_mismatches=replay_mismatches,
     )
 

projects/agentci/tests/test_agentci.py

@@ -178,6 +178,45 @@ def test_summarize_cli_can_emit_json(self):
         self.assertEqual(payload["tool_calls"], 1)
         self.assertEqual(payload["model_calls"], 1)
 
+
+    def test_diff_json_includes_step_items_for_nested_payload_changes(self):
+        baseline = self._build_episode()
+        candidate = self._build_episode()
+        candidate.steps[1].payload["output"] = {"ok": False, "reason": "timeout"}
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            baseline_path = root / "baseline.json"
+            candidate_path = root / "candidate.json"
+            baseline.save(baseline_path)
+            candidate.save(candidate_path)
+            output = StringIO()
+            with redirect_stdout(output):
+                code = cli_main(["diff", str(baseline_path), str(candidate_path), "--json"])
+        payload = json.loads(output.getvalue())
+        self.assertEqual(code, 0)
+        self.assertTrue(payload["changed"])
+        self.assertTrue(payload["step_items"])
+        self.assertEqual(payload["step_items"][0]["step_index"], 2)
+        self.assertIn("payload.output", payload["step_items"][0]["field_path"])
+
+    def test_assert_regression_failure_message_includes_step_details(self):
+        baseline = self._build_episode()
+        candidate = self._build_episode()
+        candidate.steps[1].payload["status"] = "error"
+        result = run_regression_check(baseline, candidate, check_candidate_replay=False)
+        self.assertFalse(result.passed)
+        message = result.failure_message()
+        self.assertIn("step details:", message)
+        self.assertIn("step 2 [tool_call:tool] payload.status", message)
+
+    def test_html_report_renders_field_level_step_changes(self):
+        baseline = self._build_episode()
+        candidate = self._build_episode()
+        candidate.steps.append(type(baseline.steps[0])(kind="note", name="extra", payload={"x": 1}))
+        html = render_diff_html_report(baseline, candidate)
+        self.assertIn("field-level changes", html)
+        self.assertIn("step 3 [note:extra] step", html)
+
     def test_diff_cli_can_emit_json(self):
         baseline = self._build_episode()
         candidate = self._build_episode()