Generate complete benchmark reports

Author: AntoineRichard

benchmarks/kamino_dvi/README.md

@@ -0,0 +1,53 @@
+# Kamino DVI runtime benchmark
+
+This package runs the approved RSL-RL comparison across current Kamino, Newton PR 3570 P-ADMM and DVI,
+MJWarp, and PhysX. The matrix uses 300 training iterations and seeds 42–46. It starts at 4096 environments and
+only moves down the declared capacity ladder after an explicit capacity failure.
+
+## Locked environments
+
+Create `.venv-current` from the project environment with Newton at the `newton_current` revision in `matrix.yaml`.
+Create `.venv-pr3570` from the same environment, then install the exact PR revision without applying IsaacLab's
+baseline package-source override:
+
+```bash
+uv pip install --python .venv-pr3570/bin/python --no-deps --reinstall --no-cache --no-config \
+  "git+https://github.com/newton-physics/newton.git@7906676b2e5061273db96af179d7081fc6cbbba0"
+```
+
+For PhysX, `_isaac_sim` must point to a working Isaac Sim binary installation. The runner probes and validates both
+Newton revisions before executing any selected identity.
+
+## Execute
+
+Run five-iteration construction preflights first:
+
+```bash
+./isaaclab.sh -p -m benchmarks.kamino_dvi.run --preflight-only --resume
+```
+
+Then execute the full single-GPU matrix. Output is streamed to ignored per-run directories under
+`benchmark_artifacts/kamino_dvi/runs`; atomic manifests make the command safe to resume.
+
+```bash
+./isaaclab.sh -p -m benchmarks.kamino_dvi.run --full-only --resume
+```
+
+Use `--task`, `--variant`, and `--seed` to select a subset. `--dry-run` prints the exact commands without probing or
+launching an environment.
+
+## Analyze
+
+Generate compact JSON, a 95% CI runtime figure, and Markdown/PDF reports:
+
+```bash
+./isaaclab.sh -p -m benchmarks.kamino_dvi.analyze \
+  --artifact-root benchmark_artifacts/kamino_dvi/runs \
+  --logs-root logs \
+  --output-dir benchmarks/kamino_dvi/results
+```
+
+Runtime summaries exclude iterations 1–10. Reward, episode length, and success summaries average the final 20
+iterations. Confidence intervals use the two-sided five-seed Student-t critical value. Reward and episode length are
+read from schema v1.1; success is read from the matching TensorBoard trace because the original v1.1 generator stored
+live step averages instead of the logged per-iteration values. That generator bug is fixed separately in PR 6624.

benchmarks/kamino_dvi/analysis.py

@@ -30,6 +30,7 @@ class RunMetrics:
     reward: tuple[float, ...]
     ep_length: tuple[float, ...]
     success_rate: tuple[float, ...] | None
+    success_schema_mismatch: bool = False
 
 
 @dataclass(frozen=True)
@@ -82,6 +83,17 @@ def summarize_records(records: list[RunMetrics]) -> list[VariantSummary]:
     return summaries
 
 
+def complete_five_seed_records(records: list[RunMetrics]) -> list[RunMetrics]:
+    """Return only task/variant groups with all five unique approved seeds."""
+    complete: list[RunMetrics] = []
+    ordered = sorted(records, key=lambda record: (record.task, record.variant, record.seed))
+    for _, grouped in groupby(ordered, key=lambda record: (record.task, record.variant)):
+        runs = list(grouped)
+        if len(runs) == 5 and len({run.seed for run in runs}) == 5:
+            complete.extend(runs)
+    return complete
+
+
 def load_records(artifact_root: Path, logs_root: Path, task: str | None = None) -> list[RunMetrics]:
     """Load every completed full-run manifest and its matched TensorBoard trace."""
     records: list[RunMetrics] = []
@@ -91,10 +103,11 @@ def load_records(artifact_root: Path, logs_root: Path, task: str | None = None)
         if manifest.get("state") != "completed" or (task is not None and identity.get("task") != task):
             continue
         bundles = tuple(manifest_path.parent.glob("benchmark_training_*.json"))
-        if len(bundles) != 1:
-            raise ValueError(f"{manifest_path.parent} must contain exactly one schema bundle")
-        event_path = locate_rsl_rl_events(bundles[0], logs_root)
-        trace = parse_training_trace(bundles[0], event_path)
+        if not bundles:
+            raise ValueError(f"{manifest_path.parent} contains no schema bundle")
+        bundle = max(bundles, key=lambda path: path.stat().st_mtime)
+        event_path = locate_rsl_rl_events(bundle, logs_root)
+        trace = parse_training_trace(bundle, event_path)
         records.append(
             RunMetrics(
                 task=trace.task,
@@ -106,6 +119,7 @@ def load_records(artifact_root: Path, logs_root: Path, task: str | None = None)
                 reward=trace.reward,
                 ep_length=trace.ep_length,
                 success_rate=trace.success_rate,
+                success_schema_mismatch=trace.success_schema_mismatch,
             )
         )
     return records

benchmarks/kamino_dvi/analyze.py

@@ -0,0 +1,91 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Generate compact results, figures, and reports from completed benchmark runs."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from dataclasses import asdict
+from pathlib import Path
+
+from .analysis import VariantSummary, complete_five_seed_records, load_records, summarize_records
+from .plotting import VARIANT_LABELS, plot_runtime
+from .reporting import write_reports
+
+
+def quality_issues(records, summaries: list[VariantSummary], artifact_root: Path) -> list[str]:
+    """Return explicit schema, capacity, failure, and learning-quality warnings."""
+    issues: list[str] = []
+    if any(record.success_schema_mismatch for record in records):
+        issues.append(
+            "Schema v1.1 success series differs from TensorBoard for at least one run; report uses TensorBoard success."
+        )
+    for summary in summaries:
+        if summary.num_envs < 4096:
+            issues.append(f"{summary.task} used {summary.num_envs} environments after a documented capacity fallback.")
+    for manifest_path in sorted(artifact_root.glob("*/manifest.json")):
+        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+        if manifest.get("state") == "failed":
+            identity = manifest["identity"]
+            issues.append(
+                f"Failed {identity['task']} / {identity['variant']} / seed {identity['seed']}: "
+                f"{manifest.get('failure_category')}."
+            )
+    by_task = {summary.task for summary in summaries}
+    for task in sorted(by_task):
+        rows = {summary.variant: summary for summary in summaries if summary.task == task}
+        baseline = rows.get("kamino_current")
+        if baseline is None:
+            continue
+        baseline_floor = baseline.reward.mean - baseline.reward.half_width
+        for variant, row in rows.items():
+            if variant == "kamino_current":
+                continue
+            if row.reward.mean + row.reward.half_width < baseline_floor:
+                issues.append(
+                    f"{task} {VARIANT_LABELS.get(variant, variant)} has materially lower final-window reward than "
+                    "current Kamino."
+                )
+    return issues
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the report generator command-line parser."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--artifact-root", type=Path, default=Path("benchmark_artifacts/kamino_dvi/runs"))
+    parser.add_argument("--logs-root", type=Path, default=Path("logs"))
+    parser.add_argument("--output-dir", type=Path, default=Path("benchmarks/kamino_dvi/results"))
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Load validated runs and generate all compact report artifacts."""
+    args = build_parser().parse_args(argv)
+    records = load_records(args.artifact_root, args.logs_root)
+    summaries = summarize_records(complete_five_seed_records(records))
+    if not summaries:
+        raise RuntimeError("no complete five-seed task/variant groups are available")
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    runtime_figure = args.output_dir / "runtime.png"
+    plot_runtime(summaries, runtime_figure)
+    issues = quality_issues(records, summaries, args.artifact_root)
+    (args.output_dir / "summary.json").write_text(
+        json.dumps([asdict(summary) for summary in summaries], indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    write_reports(
+        summaries,
+        issues,
+        [runtime_figure],
+        args.output_dir / "kamino_dvi_benchmark.md",
+        args.output_dir / "kamino_dvi_benchmark.pdf",
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmarks/kamino_dvi/reporting.py

@@ -58,17 +58,17 @@ def _markdown(summaries: list[VariantSummary], issues: list[str], figure_paths:
             "## Protocol",
             "",
             "- RSL-RL, 300 training iterations, seeds 42–46.",
-            "- A common environment count is selected per task from 4096 downward only after explicit capacity failures.",
-            "- Runs are sequential on one GPU and validated against immutable IsaacLab/Newton revisions and schema v1.1.",
+            "- A common environment count is selected per task from 4096 downward only after explicit capacity "
+            "failures.",
+            "- Runs are sequential on one GPU and validated against immutable IsaacLab/Newton revisions and schema "
+            "v1.1.",
             "- Reward and episode length use schema series; success rate uses the matching TensorBoard trace.",
         ]
     )
     return "\n".join(lines) + "\n"
 
 
-def _write_pdf(
-    summaries: list[VariantSummary], issues: list[str], figure_paths: list[Path], output_path: Path
-) -> None:
+def _write_pdf(summaries: list[VariantSummary], issues: list[str], figure_paths: list[Path], output_path: Path) -> None:
     with PdfPages(output_path) as pdf:
         figure = plt.figure(figsize=(11.69, 8.27))
         figure.text(0.07, 0.93, "Kamino DVI Solver Benchmark", fontsize=22, fontweight="bold")