feat(report): record run_config (seeds + load/warmup) in result_summary.json (#377)

Snapshot the run's load_pattern, warmup, and the scheduler/dataloader RNG seeds
into the report so result_summary.json is self-describing and a valid run is
identified by its settings. Rendered as a 'Run config:' block in report.txt.
endpoint_config (api_key/URLs) is a sibling of settings and never included, so no
secrets leak.

Replaces the standalone 'seeds' field (#353). The resolved/effective runtime
settings (sample count + ordering, which can differ per audit phase) are left for
a follow-up.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: nvzhihanj

src/inference_endpoint/commands/benchmark/execute.py

@@ -1110,16 +1110,28 @@ def _on_phase_start(phase: PhaseConfig) -> None:
 
                 if snap_dict is not None:
                     try:
-                        runtime = ctx.config.settings.runtime
-                        warmup = ctx.config.settings.warmup
                         load_pattern = ctx.config.settings.load_pattern
+                        runtime_cfg = ctx.config.settings.runtime
+                        # load_pattern + warmup config and the RNG seeds, so
+                        # result_summary.json is self-describing and a valid run is
+                        # identified by its settings. The full, re-runnable config
+                        # lives in config.yaml alongside. The resolved/effective
+                        # runtime settings (sample count + ordering, which can differ
+                        # per audit phase) are deferred to a follow-up. endpoint_config
+                        # (api_key/URLs) is a sibling of settings and never included,
+                        # so no secrets.
+                        run_config = ctx.config.settings.model_dump(
+                            mode="json", include={"load_pattern", "warmup"}
+                        )
+                        run_config["scheduler_random_seed"] = (
+                            runtime_cfg.scheduler_random_seed
+                        )
+                        run_config["dataloader_random_seed"] = (
+                            runtime_cfg.dataloader_random_seed
+                        )
                         report = Report.from_snapshot(
                             snap_dict,
-                            seeds={
-                                "scheduler_random_seed": runtime.scheduler_random_seed,
-                                "dataloader_random_seed": runtime.dataloader_random_seed,
-                                "warmup_random_seed": warmup.warmup_random_seed,
-                            },
+                            run_config=run_config,
                             use_legacy_loadgen_qps_metrics=(
                                 load_pattern.type == LoadPatternType.POISSON
                                 and load_pattern.use_legacy_loadgen_qps_metrics

src/inference_endpoint/metrics/report.py

@@ -146,25 +146,28 @@ class Report(msgspec.Struct, frozen=True):  # type: ignore[call-arg]
     qps: float | None = None
     tps: float | None = None
 
-    # RNG seeds for this run (scheduler/dataloader/warmup, from config). Carried
-    # so result_summary.json is self-validating: a reproducible run is identified
-    # by its seeds. These are config, not a measured metric, so the from_snapshot
-    # caller supplies them rather than reading them from the metrics snapshot.
-    seeds: dict[str, int] | None = None
+    # Run configuration (load_pattern, warmup, and the scheduler/dataloader RNG
+    # seeds), from config. Carried so result_summary.json is self-describing and a
+    # valid run is identified by its settings. Config, not a measured metric, so
+    # the from_snapshot caller supplies it rather than reading it from the metrics
+    # snapshot. (Resolved/effective runtime settings — sample count + ordering,
+    # which can differ per audit phase — are deferred to a follow-up.)
+    run_config: dict[str, Any] | None = None
 
     @classmethod
     def from_snapshot(
         cls,
         snap: dict[str, Any],
         *,
-        seeds: dict[str, int] | None = None,
+        run_config: dict[str, Any] | None = None,
         use_legacy_loadgen_qps_metrics: bool = True,
     ) -> Report:
         """Build a Report from a snapshot dict.
 
-        ``seeds`` (optional) carries the run's RNG seeds from config into the
-        report so result_summary.json is self-validating; it is keyword-only
-        because it is config, not part of the metrics snapshot.
+        ``run_config`` (optional, keyword-only) carries the run's configuration
+        (load_pattern, warmup, and the scheduler/dataloader RNG seeds) into the
+        report so result_summary.json is self-describing; it is config, not part
+        of the metrics snapshot.
 
         Input is the dict form produced by
         ``inference_endpoint.async_utils.services.metrics_aggregator.snapshot
@@ -274,7 +277,7 @@ def _series_dict(key: str) -> dict[str, Any]:
             legacy_loadgen_window_duration_ns=legacy_loadgen_window_duration_ns,
             qps=qps,
             tps=tps,
-            seeds=seeds,
+            run_config=run_config,
         )
 
     def to_json(self, save_to: os.PathLike | None = None) -> bytes:
@@ -304,9 +307,14 @@ def display(
         fn(f"Version: {self.version}{newline}")
         if self.git_sha:
             fn(f"Git SHA: {self.git_sha}{newline}")
-        if self.seeds:
-            seed_str = ", ".join(f"{k}={v}" for k, v in self.seeds.items())
-            fn(f"Seeds: {seed_str}{newline}")
+        if self.run_config:
+            fn(f"Run config:{newline}")
+            for section, params in self.run_config.items():
+                if isinstance(params, dict):
+                    inner = ", ".join(f"{k}={v}" for k, v in params.items())
+                    fn(f"  {section}: {inner}{newline}")
+                else:
+                    fn(f"  {section}: {params}{newline}")
         if self.test_started_at > 0:
             approx = monotime_to_datetime(self.test_started_at)
             fn(f"Test started at: {approx.strftime('%Y-%m-%d %H:%M:%S')}{newline}")

tests/unit/metrics/test_report_builder.py

@@ -181,16 +181,23 @@ def test_with_metrics(self):
         # OSL data was written → tps is computable.
         assert report.tps is not None
 
-    def test_seeds_keyword_only_passthrough(self):
-        """seeds is config, not a snapshot metric: None unless the caller
+    def test_run_config_keyword_only_passthrough(self):
+        """run_config is config, not a snapshot metric: None unless the caller
         supplies it, and carried verbatim into the report when it does."""
         registry = _make_registry(n_samples=5)
         snap = snapshot_to_dict(
             registry.build_snapshot(state=SessionState.COMPLETE, n_pending_tasks=0)
         )
-        assert Report.from_snapshot(snap).seeds is None
-        seeds = {"scheduler_random_seed": 42, "dataloader_random_seed": 7}
-        assert Report.from_snapshot(snap, seeds=seeds).seeds == seeds
+        assert Report.from_snapshot(snap).run_config is None
+        run_config = {
+            "load_pattern": {"type": "poisson", "target_qps": 14.75},
+            "warmup": {"enabled": False, "warmup_random_seed": 42},
+            "scheduler_random_seed": 42,
+            "dataloader_random_seed": 42,
+        }
+        assert (
+            Report.from_snapshot(snap, run_config=run_config).run_config == run_config
+        )
 
     def test_failed_uses_tracked_counter(self):
         """``n_samples_failed`` reads from ``tracked_samples_failed``, not
@@ -279,23 +286,28 @@ def test_to_json_qps_tps_null_without_duration(self):
         assert data["qps"] is None
         assert data["tps"] is None
 
-    def test_to_json_serializes_seeds(self):
-        """result_summary.json carries the run's RNG seeds so a run can be
-        validated as reproducible; absent seeds serialize as null."""
+    def test_to_json_and_display_carry_run_config(self):
+        """result_summary.json + report.txt carry the run's config so a run is
+        self-describing/reproducible; absent run_config serializes as null."""
         registry = _make_registry(n_samples=5)
         snap = snapshot_to_dict(
             registry.build_snapshot(state=SessionState.COMPLETE, n_pending_tasks=0)
         )
-        seeds = {"scheduler_random_seed": 42, "dataloader_random_seed": 42}
-        report = Report.from_snapshot(snap, seeds=seeds)
-        assert json.loads(report.to_json())["seeds"] == seeds
+        run_config = {
+            "load_pattern": {"type": "poisson", "target_qps": 14.75},
+            "scheduler_random_seed": 42,
+            "dataloader_random_seed": 42,
+        }
+        report = Report.from_snapshot(snap, run_config=run_config)
+        assert json.loads(report.to_json())["run_config"] == run_config
 
         lines: list[str] = []
         report.display(fn=lines.append, summary_only=True)
-        assert any("Seeds:" in ln for ln in lines)
+        assert any("Run config:" in ln for ln in lines)
+        assert any("load_pattern:" in ln and "poisson" in ln for ln in lines)
 
-        # Absent seeds -> null, not omitted.
-        assert json.loads(Report.from_snapshot(snap).to_json())["seeds"] is None
+        # Absent run_config -> null, not omitted.
+        assert json.loads(Report.from_snapshot(snap).to_json())["run_config"] is None
 
     def test_to_json_save(self, tmp_path: Path):
         registry = _make_registry(n_samples=5)