feat(metrics): consolidate gpt-oss/deepseek accuracy, dedup reporting, surface TPS

Deduplicate the reporting layer, give gpt-oss-120b and deepseek-r1 a uniform
per-subset accuracy breakdown, and surface TPS everywhere QPS appears.

- Add shared evaluation/accuracy_results.py (to_float, find_accuracy_breakdown,
  ACCURACY_METRIC_KEYS, build_breakdown); metrics/results_plots.py and
  compliance/checker.py import it instead of copy-pasted helpers.
- DeepSeek-R1 scorer surfaces its per-subset breakdown via score_breakdown()
  (BFCL-shaped: overall_accuracy/subset_scores/total_samples, 0-100).
- Add GptOssAccuracyScorer: runs the 3 gpt-oss member scorers once and rolls
  them up sample-weighted by unique problem count. finalize emits both the
  per-subset <subset>::gptoss entries (audit + downstream compat) and one
  consolidated `gptoss` entry with a breakdown, from the single scoring pass.
- Consolidation is opt-in via explicit accuracy_config.group; the ::variant
  suffix is a naming convention, not a grouping signal. Group members must
  return scalar scores in [0,1].
- Add accuracy to Report.display()/report.txt/result_summary.json; write the
  report in a finally so a scoring failure can't discard perf artifacts.
- Single-source results.json qps/tps from Report (drop the duplicate QPS
  recompute); add tps + a symmetric "TPS: N/A" display branch.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: arekay-nv

examples/04_GPTOSS120B_Example/sglang_gptoss_120b_example.yaml

@@ -23,20 +23,23 @@ datasets:
       eval_method: "code_bench_scorer"
       extractor: "python_code_extractor"
       num_repeats: 3
+      group: "gptoss"
   - name: "aime25::gptoss"
     type: "accuracy"
     accuracy_config:
       eval_method: "pass_at_1"
       ground_truth: "answer"
       extractor: "boxed_math_extractor"
       num_repeats: 8
+      group: "gptoss"
   - name: "gpqa::gptoss"
     type: "accuracy"
     accuracy_config:
       eval_method: "pass_at_1"
       extractor: "abcd_extractor"
       ground_truth: "ground_truth"
       num_repeats: 5
+      group: "gptoss"
 settings:
   runtime:
     min_duration_ms: 3000

examples/04_GPTOSS120B_Example/vllm_gptoss_120b_example.yaml

@@ -25,20 +25,23 @@ datasets:
       eval_method: "code_bench_scorer"
       extractor: "python_code_extractor"
       num_repeats: 3
+      group: "gptoss"
   - name: "aime25::gptoss"
     type: "accuracy"
     accuracy_config:
       eval_method: "pass_at_1"
       ground_truth: "answer"
       extractor: "boxed_math_extractor"
       num_repeats: 8
+      group: "gptoss"
   - name: "gpqa::gptoss"
     type: "accuracy"
     accuracy_config:
       eval_method: "pass_at_1"
       extractor: "abcd_extractor"
       ground_truth: "ground_truth"
       num_repeats: 5
+      group: "gptoss"
 
 settings:
   runtime:

src/inference_endpoint/commands/benchmark/execute.py

@@ -44,6 +44,7 @@
 
 import msgspec
 import msgspec.json
+import msgspec.structs
 from huggingface_hub import model_info
 from tqdm import tqdm
 from transformers.utils import logging as transformers_logging
@@ -83,7 +84,11 @@
 from inference_endpoint.endpoint_client.http_client import HTTPEndpointClient
 from inference_endpoint.endpoint_client.http_sample_issuer import HttpClientSampleIssuer
 from inference_endpoint.evaluation import Extractor
-from inference_endpoint.evaluation.scoring import Scorer
+from inference_endpoint.evaluation.scoring import (
+    GptOssAccuracyScorer,
+    GptOssMember,
+    Scorer,
+)
 from inference_endpoint.exceptions import (
     ExecutionError,
     InputValidationError,
@@ -160,6 +165,9 @@ class AccuracyConfiguration:
     ground_truth_column: str | None
     num_repeats: int
     extras: dict[str, Any] = field(default_factory=dict)
+    # Consolidation group tag; members sharing a group (>=2) roll up into one
+    # combined score. None -> scored on its own.
+    group: str | None = None
 
 
 @dataclass
@@ -274,6 +282,19 @@ def _resolve_accuracy_components(
     return scorer_cls, extractor_cls
 
 
+def _accuracy_group(accuracy_config: Any | None) -> str | None:
+    """Explicit consolidation group for an accuracy dataset.
+
+    Only an explicit ``accuracy_config.group`` triggers consolidation. The
+    ``::<variant>`` name suffix is a naming convention (``open_orca::llama2_70b``,
+    ``cnn_dailymail::llama3_8b``), NOT a grouping signal, so it must not roll
+    datasets up implicitly — that would silently consolidate any future config
+    with two accuracy datasets sharing a model suffix. The gpt-oss example
+    configs set ``group: "gptoss"`` explicitly.
+    """
+    return getattr(accuracy_config, "group", None) if accuracy_config else None
+
+
 def _load_datasets(
     config: BenchmarkConfig,
     report_dir: Path,
@@ -319,6 +340,7 @@ def _load_datasets(
                 acc_cfg.accuracy_config.ground_truth,
                 acc_cfg.accuracy_config.num_repeats,
                 acc_cfg.accuracy_config.extras or {},
+                group=_accuracy_group(acc_cfg.accuracy_config),
             )
         )
         ds.load(
@@ -373,6 +395,7 @@ def _load_datasets(
                     accuracy_config.ground_truth,
                     accuracy_config.num_repeats,
                     accuracy_config.extras or {},
+                    group=_accuracy_group(accuracy_config),
                 )
             )
 
@@ -1169,41 +1192,76 @@ def _salvage_tmpfs(report_dir: Path, tmpfs_dir: Path) -> None:
         logger.debug(f"Copied {src_events} -> {dst_events}")
 
 
-def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None:
-    """Score accuracy, aggregate results, write JSON."""
-    config = ctx.config
-    result = bench.session
-    collector = bench.collector
-    report = bench.report
+def _score_accuracy(ctx: BenchmarkContext, result: SessionResult) -> dict[str, Any]:
+    """Run accuracy scorers, consolidating grouped datasets into one entry each.
 
-    # Display report if available (from MetricsAggregator pub/sub snapshot).
-    # result_summary.json is the self-complete machine-readable report (carries
-    # qps/tps/seeds via Report.to_json); report.txt is the full human-readable
-    # dump (histograms + percentiles); the console log shows just the summary.
-    if report is not None:
-        report.display(fn=lambda s: logger.info(s), summary_only=True)
-        report.to_json(save_to=ctx.report_dir / "result_summary.json")
-
-        report_txt = ctx.report_dir / "report.txt"
-        with report_txt.open("w") as f:
-            report.display(fn=lambda s: print(s, file=f))
-            if bench.profiling is not None:
-                _write_profiling_section(f, bench.profiling)
-        logger.info("Report written to %s", report_txt)
+    Datasets sharing a consolidation group with >=2 members (e.g. gpt-oss's
+    aime25/gpqa/livecodebench subsets) are rolled up by GptOssAccuracyScorer into
+    a single combined entry keyed by the group name; everything else is scored on
+    its own exactly as before.
+    """
+    accuracy_scores: dict[str, Any] = {}
 
-    # Sibling profiling.json — kept separate so Report stays a pure
-    # snapshot-derived struct.
-    if bench.profiling is not None:
-        (ctx.report_dir / "profiling.json").write_text(
-            json.dumps(bench.profiling, indent=2)
+    groups: dict[str, list[AccuracyConfiguration]] = {}
+    for cfg in ctx.eval_configs:
+        if cfg.group:
+            groups.setdefault(cfg.group, []).append(cfg)
+    combined = {g: members for g, members in groups.items() if len(members) >= 2}
+    combined_ids = {id(cfg) for members in combined.values() for cfg in members}
+
+    for group_name, members in combined.items():
+        gpt_members = [
+            GptOssMember(
+                full_name=cfg.dataset_name,
+                subset=cfg.dataset_name.split("::")[0],
+                scorer_cls=cfg.scorer,
+                extractor=cfg.extractor,
+                dataset=cfg.dataset,
+                ground_truth_column=cfg.ground_truth_column,
+                extras=cfg.extras,
+            )
+            for cfg in members
+        ]
+        scorer = GptOssAccuracyScorer(gpt_members, members[0].report_dir)
+        score, n_repeats = scorer.score()
+        breakdown = scorer.score_breakdown()
+        entry: dict[str, Any] = {
+            "dataset_name": group_name,
+            "num_samples": (breakdown or {}).get("total_samples", 0),
+            "score": score,
+            "complete": scorer.complete,
+        }
+        if breakdown is not None:
+            entry["breakdown"] = breakdown
+        accuracy_scores[group_name] = entry
+        logger.info(
+            f"Score for {group_name} (combined {len(members)} subsets): {score} "
+            f"({n_repeats} repeats, complete={scorer.complete})"
         )
 
-    # Write scoring artifacts + copy event log from tmpfs to disk
-    _write_scoring_artifacts(ctx, result, bench.tmpfs_dir)
+        # Per-subset audit entries from the same scoring pass (no re-scoring):
+        # keeps the individual <subset>::gptoss scores in results.json for audit
+        # and for downstream consumers that read them. These carry no breakdown,
+        # so only the consolidated entry surfaces in the report headline.
+        for cfg in members:
+            member_score, member_complete = scorer.member_scores.get(
+                cfg.dataset_name, (None, False)
+            )
+            assert cfg.dataset.data is not None
+            accuracy_scores[cfg.dataset_name] = {
+                "dataset_name": cfg.dataset_name,
+                "num_samples": len(cfg.dataset.data),
+                "extractor": (
+                    cfg.extractor.__name__ if cfg.extractor is not None else None
+                ),
+                "ground_truth_column": cfg.ground_truth_column,
+                "score": member_score,
+                "complete": member_complete,
+            }
 
-    # Accuracy scoring
-    accuracy_scores: dict[str, Any] = {}
     for eval_cfg in ctx.eval_configs:
+        if id(eval_cfg) in combined_ids:
+            continue
         try:
             scorer_instance = eval_cfg.scorer(
                 eval_cfg.dataset_name,
@@ -1223,7 +1281,7 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None:
         num_samples = len(eval_cfg.dataset.data)
         if eval_cfg.dataset_name == "performance":
             num_samples = sum(phase.issued_count for phase in result.perf_results)
-        entry: dict[str, Any] = {
+        singleton_entry: dict[str, Any] = {
             "dataset_name": eval_cfg.dataset_name,
             "num_samples": num_samples,
             "extractor": (
@@ -1232,25 +1290,84 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None:
             "ground_truth_column": eval_cfg.ground_truth_column,
             "score": score,
             # False when the scorer produced only a partial headline (e.g.
-            # LegacyMLPerfDeepSeekR1Scorer when the lcb-service container was unreachable),
-            # so a partial number is never mistaken for a complete one.
+            # LegacyMLPerfDeepSeekR1Scorer when the lcb-service container was
+            # unreachable), so a partial number is never mistaken for a complete one.
             "complete": scorer_instance.complete,
         }
         breakdown = scorer_instance.score_breakdown()
         if breakdown is not None:
-            entry["breakdown"] = breakdown
-        accuracy_scores[eval_cfg.dataset_name] = entry
+            singleton_entry["breakdown"] = breakdown
+        accuracy_scores[eval_cfg.dataset_name] = singleton_entry
         logger.info(
             f"Score for {eval_cfg.dataset_name}: {score} "
             f"({n_repeats} repeats, complete={scorer_instance.complete})"
         )
 
+    return accuracy_scores
+
+
+def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None:
+    """Score accuracy, aggregate results, write JSON."""
+    config = ctx.config
+    result = bench.session
+    collector = bench.collector
+    report = bench.report
+
+    # Sibling profiling.json — kept separate so Report stays a pure
+    # snapshot-derived struct.
+    if bench.profiling is not None:
+        (ctx.report_dir / "profiling.json").write_text(
+            json.dumps(bench.profiling, indent=2)
+        )
+
+    # Write scoring artifacts + copy event log from tmpfs to disk (scorers read
+    # sample_idx_map.json + events.jsonl from here).
+    _write_scoring_artifacts(ctx, result, bench.tmpfs_dir)
+
+    # Accuracy scoring (grouped datasets consolidated into one entry each).
+    # Scoring runs before the report is written so the accuracy headline can be
+    # attached, but the report is written in the `finally` below so a scoring
+    # failure (e.g. lcb-service unreachable, missing eval subproject, bad extras)
+    # still leaves the perf run's result_summary.json / report.txt on disk
+    # instead of discarding them — then the exception propagates as before.
+    accuracy_scores: dict[str, Any] = {}
+    try:
+        accuracy_scores = _score_accuracy(ctx, result)
+    finally:
+        # Attach accuracy breakdowns so result_summary.json, the console summary,
+        # and report.txt all carry the headline from this one scoring pass
+        # (absent on a scoring failure — accuracy_scores stays {}).
+        if report is not None:
+            accuracy_for_report = {
+                name: entry["breakdown"]
+                for name, entry in accuracy_scores.items()
+                if "breakdown" in entry
+            }
+            if accuracy_for_report:
+                report = msgspec.structs.replace(report, accuracy=accuracy_for_report)
+
+        # Display report if available (from MetricsAggregator pub/sub snapshot).
+        # result_summary.json is the self-complete machine-readable report
+        # (carries qps/tps/seeds/accuracy via Report.to_json); report.txt is the
+        # full human-readable dump; the console log shows the summary.
+        if report is not None:
+            report.display(fn=lambda s: logger.info(s), summary_only=True)
+            report.to_json(save_to=ctx.report_dir / "result_summary.json")
+
+            report_txt = ctx.report_dir / "report.txt"
+            with report_txt.open("w") as f:
+                report.display(fn=lambda s: print(s, file=f))
+                if bench.profiling is not None:
+                    _write_profiling_section(f, bench.profiling)
+            logger.info("Report written to %s", report_txt)
+
     # Report metrics: prefer Report from MetricsSnapshot, fall back to SessionResult
     if report is not None and report.duration_ns is not None:
         perf_elapsed = report.duration_ns / 1e9
         total_issued = report.n_samples_issued
         n_errors = report.n_samples_failed
         qps = report.qps or 0.0
+        tps = report.tps
     else:
         perf = result.perf_results[0] if result.perf_results else None
         if perf:
@@ -1261,6 +1378,8 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None:
             total_issued = 0
         n_errors = len(collector.errors)
         qps = total_issued / perf_elapsed if perf_elapsed > 0 else 0.0
+        # No OSL source outside the metrics snapshot, so TPS is unknown here.
+        tps = None
 
     logger.info(f"Completed in {perf_elapsed:.1f}s")
     if ctx.accuracy_only:
@@ -1295,6 +1414,7 @@ def finalize_benchmark(ctx: BenchmarkContext, bench: BenchmarkResult) -> None:
                 "failed": n_errors,
                 "elapsed_time": perf_elapsed,
                 "qps": qps,
+                "tps": tps,
             },
         }
         if accuracy_scores: