Report per-trial token and latency distributions in the aggregator

Give every token and latency measure a mean, median, and max over trials: these
distributions are heavy-tailed, so a mean alone hides that a few trials carry
much of an evaluation's spend. Fix the run label for a session passed as "."
(an empty name blanked the run column) and key the single-session JSON payload
off the grid rather than the argument name. Document the per-case statistics in
CONFIGURATION.md and the run-benchmark skill.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: varunursekar

harness-engineering-bench/CONFIGURATION.md

@@ -33,7 +33,17 @@ benchmark can be checked against the others at a glance.
   (`inference_input_tokens`, `inference_cached_input_tokens`,
   `inference_output_tokens`, `inference_total_tokens`, `inference_requests`) in
   `reward_metrics`; each trial's `result.json` also carries agent-self-reported
-  tokens (`agent_reported_*`) and its wall window. With
+  tokens (`agent_reported_*`) and its wall window. Cost is reported per case the
+  same way latency is, and because token and latency distributions are unbounded
+  and heavy-tailed (a few cases carry much of the total) each carries a **mean, a
+  median, and a max** — `mean/median/max_case_wall_seconds` and
+  `mean/median/max_case_agent_reported_{input,cached_input,output}_tokens`.
+  Accuracy, being bounded, keeps mean plus stddev. The trusted gateway figure is
+  metered per evaluation rather than per case, so it contributes the sum plus a
+  derived `mean_case_inference_*`; its median and max come from post-hoc
+  attribution (`scripts/per_trial_tokens.py`, which reports mean/median/max over
+  trials for every token and latency measure). All of these are budget signal and
+  are redacted from the agent under `disclose_budget: false`; latency is not. With
   `inference_gateway.request_log_attribution: true` the gateway stamps every
   request-log record with a `thread_id`, so `scripts/per_trial_tokens.py`
   attributes gateway tokens to individual trials (trusted, versus a content-match

harness-engineering-bench/scripts/per_trial_tokens.py

@@ -40,6 +40,7 @@
 import csv
 import json
 import re
+import statistics
 import sys
 from collections import defaultdict
 from datetime import datetime, timedelta
@@ -216,6 +217,35 @@ def assign_thread(
     return None
 
 
+_DISTRIBUTION_KEYS = (*TOKEN_KEYS, "requests", "latency_ms", "wall_s")
+
+
+def distribution(entries: list[dict]) -> dict[str, dict[str, float]]:
+    """mean/median/max per-trial, for each token and latency measure.
+
+    Token and latency distributions are unbounded and heavy-tailed — a few
+    trials carry much of an evaluation's total — so the median is reported
+    beside the mean and the max names the tail. (Accuracy, being bounded, is
+    summarized by its mean alone.)
+    """
+    summary: dict[str, dict[str, float]] = {}
+    for key in _DISTRIBUTION_KEYS:
+        values = [
+            entry[key]
+            for entry in entries
+            if isinstance(entry.get(key), (int, float))
+        ]
+        if not values:
+            continue
+        summary[key] = {
+            "mean": round(sum(values) / len(values), 1),
+            "median": round(statistics.median(values), 1),
+            "max": round(float(max(values)), 1),
+            "n": len(values),
+        }
+    return summary
+
+
 def trial_wall_seconds(trial: dict) -> float | None:
     started, finished = trial.get("started"), trial.get("finished")
     if started is None or finished is None:
@@ -283,6 +313,7 @@ def analyze_session(
         }
         report[evaluation_id] = {
             "trials": per_trial,
+            "distribution": distribution(list(per_trial.values())),
             "gateway_total": gateway_total,
             "attributed": attributed,
             "agent_reported_total": agent_reported_total,
@@ -385,6 +416,13 @@ def print_report(run_label: str, report: dict) -> None:
             f"coverage: {data['coverage_pct']}% of {gateway['total_tokens']} "
             f"gateway-metered tokens attributed to trials"
         )
+        if data["distribution"]:
+            print(f"per-trial distribution  {'mean':>12} {'median':>12} {'max':>12}")
+            for key, stats in data["distribution"].items():
+                print(
+                    f"  {key:<20} {stats['mean']:>12,.1f} "
+                    f"{stats['median']:>12,.1f} {stats['max']:>12,.1f}"
+                )
         # Reconcile the trusted envelope against the two independent token sources.
         print(
             f"  input tokens  gateway {gateway['input_tokens']} | "
@@ -414,7 +452,9 @@ def main() -> int:
         if not requests_dir.is_dir():
             print(f"no request log at {requests_dir}", file=sys.stderr)
             continue
-        grid[session.name] = analyze_session(session, requests_dir, task_texts)
+        # "." resolves to an empty name, which would label every row blank.
+        run_label = session.resolve().name or str(session)
+        grid[run_label] = analyze_session(session, requests_dir, task_texts)
 
     if not grid:
         return 1
@@ -430,7 +470,7 @@ def main() -> int:
     if arguments.as_json:
         # One session keeps the original flat {evaluation: ...} schema; several
         # nest under their run label.
-        payload = grid[arguments.sessions[0].name] if len(grid) == 1 else grid
+        payload = next(iter(grid.values())) if len(grid) == 1 else grid
         print(json.dumps(payload, indent=1, default=str))
         return 0
 

harness-engineering-bench/scripts/test_per_trial_tokens.py

@@ -212,7 +212,7 @@ def test_analyze_session_stamped_full_coverage_wall_and_reconciliation(tmp_path)
 
     # report schema
     assert set(data) == {
-        "trials", "gateway_total", "attributed",
+        "trials", "distribution", "gateway_total", "attributed",
         "agent_reported_total", "residual", "coverage_pct",
     }
     # stamped chains attribute fully
@@ -261,3 +261,41 @@ def test_csv_rows_schema_and_residual_row(tmp_path):
     assert trial_row["total_tokens"] == 13
     residual_row = next(r for r in rows if r["task"] == "(unattributed)")
     assert residual_row["total_tokens"] == 8
+
+
+def test_distribution_reports_mean_median_max_for_skewed_trials(tmp_path):
+    # Heavy tail: one trial dominates. mean alone hides the shape, so the
+    # aggregator reports median beside it and max names the tail.
+    evaluation = "eval-1"
+    sizes = {"a": 100, "b": 200, "c": 3000}
+    for index, (suffix, tokens) in enumerate(sizes.items()):
+        _trial(
+            tmp_path, evaluation, index, f"org/task-{suffix}",
+            f"Question about {suffix} widgets in the treasury bulletin corpus",
+        )
+    records = [
+        {
+            "scope": "evaluation", "attribution": evaluation,
+            "thread_id": f"th{suffix}",
+            "root_snippet": (
+                f"Question about {suffix} widgets in the treasury bulletin corpus"
+            ),
+            "input_tokens": tokens, "cached_input_tokens": tokens // 10,
+            "output_tokens": tokens // 20, "total_tokens": tokens,
+            "latency_ms": tokens, "ts": "2026-01-01T00:01:00Z",
+        }
+        for suffix, tokens in sizes.items()
+    ]
+    requests = _write_log(tmp_path, records)
+
+    report = p.analyze_session(tmp_path, requests, [])
+    dist = report[evaluation]["distribution"]
+
+    assert dist["input_tokens"]["mean"] == 1100.0     # dragged up by the tail
+    assert dist["input_tokens"]["median"] == 200.0    # unmoved by it
+    assert dist["input_tokens"]["max"] == 3000.0
+    assert dist["input_tokens"]["n"] == 3
+    assert dist["output_tokens"]["median"] == 10.0
+    assert dist["latency_ms"]["median"] == 200.0
+    # every trial shares one 5-minute window in the fixture
+    assert dist["wall_s"]["median"] == 300.0

harness-engineering-bench/skills/run-benchmark/SKILL.md

@@ -182,8 +182,16 @@ The reported unit is tokens, not dollars: the trusted per-evaluation split lands
 in `reward_metrics` as `inference_input_tokens` / `inference_cached_input_tokens`
 / `inference_output_tokens` / `inference_total_tokens` (→ W&B). Because the target
 model is fixed per benchmark, dollars are a downstream linear function of that
-triple (per-model rate vector) and are not stored. For per-trial breakdowns, run
-the post-hoc aggregator on the exported session:
+triple (per-model rate vector) and are not stored.
+
+Read the per-case statistics, not just the totals: token and latency
+distributions are heavy-tailed, so `reward_metrics` carries a mean, median, and
+max per case (`mean/median/max_case_wall_seconds`,
+`mean/median/max_case_agent_reported_*_tokens`, plus a derived
+`mean_case_inference_*`). A mean far above the median means a few cases dominate
+the spend; the max is the case most likely to hit its wall budget and score the
+failure value. For per-trial breakdowns, run the post-hoc aggregator on the
+exported session:
 
 ```bash
 python harness-engineering-bench/scripts/per_trial_tokens.py <session-dir> --json