feat: accuracy issuer inherits perf concurrency in online mode (#357) (#379)

* feat: accuracy issuer inherits perf concurrency in online mode (#357)

When the performance phase runs the CONCURRENCY load pattern (online), the
accuracy phase now mirrors that same fixed concurrency instead of always
bursting at MAX_THROUGHPUT, so evaluation exercises the endpoint the same way
as the performance run.

All other patterns are unchanged: POISSON and offline MAX_THROUGHPUT perf
phases keep the accuracy phase at MAX_THROUGHPUT, since inheriting POISSON
would silently rate-limit evaluation to the perf QPS (no accuracy QPS-budgeting
yet). The gate is purely load_pattern.type == CONCURRENCY, which the schema
already constrains to online mode.

Also logs the accuracy issuer's chosen load mode (pattern + target_concurrency)
per accuracy dataset. Adds unit tests for the concurrency-inheritance,
POISSON-stays-max-throughput, offline-stays-max-throughput, and logging cases.

Co-authored-by: Claude Opus 4.8 (10M context) <noreply@anthropic.com>

---------

Signed-off-by: arekay-nv <230885705+arekay-nv@users.noreply.github.com>

Author: arekay-nv

src/inference_endpoint/commands/benchmark/execute.py

@@ -562,6 +562,16 @@ def _build_phases(
             )
         )
 
+    # Accuracy mirrors the perf load pattern so evaluation exercises the
+    # endpoint the same way it was benchmarked. AGENTIC_INFERENCE can't drive
+    # the (non-agentic) accuracy datasets — create_load_strategy rejects it —
+    # so it (and a missing perf pattern) falls back to MAX_THROUGHPUT.
+    perf_lp = ctx.rt_settings.load_pattern if ctx.rt_settings is not None else None
+    if perf_lp is None or perf_lp.type == LoadPatternType.AGENTIC_INFERENCE:
+        acc_load_pattern = LoadPattern(type=LoadPatternType.MAX_THROUGHPUT)
+    else:
+        acc_load_pattern = perf_lp
+
     # Accuracy phases — use eval_cfg.dataset_name as phase name so it matches
     # what Scorer._load_sample_index_map() looks up in sample_idx_map.json
     for eval_cfg in ctx.eval_configs:
@@ -574,15 +584,14 @@ def _build_phases(
                 "AgenticInferenceDataset, which is not yet supported for "
                 "accuracy evaluation."
             )
-        # Accuracy phases run at MAX_THROUGHPUT; inheriting perf_lp (e.g. POISSON)
-        # would silently rate-limit evaluation until an agentic inference accuracy strategy
-        # and QPS-budgeting support are added.
+        logger.info(
+            "Accuracy issuer '%s' load mode: %s",
+            eval_cfg.dataset_name,
+            acc_load_pattern,
+        )
         rng_settings = ctx.rt_settings or RuntimeSettings.from_config(
             ctx.config, acc_ds.num_samples()
         )
-        acc_load_pattern: LoadPattern | None = LoadPattern(
-            type=LoadPatternType.MAX_THROUGHPUT
-        )
         acc_settings = RuntimeSettings(
             metric_target=rng_settings.metric_target,
             reported_metrics=rng_settings.reported_metrics,

src/inference_endpoint/config/schema.py

@@ -718,6 +718,20 @@ def _validate_completeness(self) -> Self:
             )
         return self
 
+    def __str__(self) -> str:
+        """Human-readable "type (param=value)" form for logging, e.g.
+        ``concurrency (target_concurrency=7)`` / ``poisson (target_qps=10.0)``.
+        Patterns without a driving parameter render as just the type name.
+        """
+        if self.type in (
+            LoadPatternType.CONCURRENCY,
+            LoadPatternType.AGENTIC_INFERENCE,
+        ):
+            return f"{self.type.value} (target_concurrency={self.target_concurrency})"
+        if self.type == LoadPatternType.POISSON:
+            return f"{self.type.value} (target_qps={self.target_qps})"
+        return self.type.value
+
 
 @cyclopts.Parameter(name="*")
 class WarmupConfig(BaseModel):

tests/unit/commands/test_benchmark.py

@@ -16,8 +16,10 @@
 """Tests for benchmark CLI models, config building, and command handlers."""
 
 import asyncio
+import dataclasses
 import io
 import json
+import logging
 import random
 import tempfile
 from pathlib import Path
@@ -1165,6 +1167,151 @@ def test_accuracy_drain_timeout_defaults_to_unbounded(
         acc = next(p for p in phases if p.phase_type == PhaseType.ACCURACY)
         assert acc.drain_timeout is None
 
+    @pytest.mark.unit
+    def test_accuracy_phase_inherits_perf_concurrency(
+        self, base_rt_settings, simple_dataset
+    ):
+        """When the perf phase runs CONCURRENCY, the accuracy phase mirrors the
+        same fixed concurrency instead of bursting at MAX_THROUGHPUT."""
+        rt = dataclasses.replace(
+            base_rt_settings,
+            load_pattern=LoadPattern(
+                type=LoadPatternType.CONCURRENCY, target_concurrency=7
+            ),
+        )
+        config = OfflineConfig(**_OFFLINE_KWARGS)
+        ctx = self._make_ctx(config, rt, simple_dataset)
+        ctx.eval_configs = [self._make_eval_config(simple_dataset)]
+        phases = _build_phases(ctx)
+
+        acc = next(p for p in phases if p.phase_type == PhaseType.ACCURACY)
+        assert acc.runtime_settings.load_pattern is not None
+        assert acc.runtime_settings.load_pattern.type == LoadPatternType.CONCURRENCY
+        assert acc.runtime_settings.load_pattern.target_concurrency == 7
+
+    @pytest.mark.unit
+    def test_accuracy_phase_inherits_perf_poisson(
+        self, base_rt_settings, simple_dataset
+    ):
+        """POISSON perf: accuracy mirrors the same POISSON config (target_qps)."""
+        rt = dataclasses.replace(
+            base_rt_settings,
+            load_pattern=LoadPattern(type=LoadPatternType.POISSON, target_qps=10.0),
+        )
+        config = OfflineConfig(**_OFFLINE_KWARGS)
+        ctx = self._make_ctx(config, rt, simple_dataset)
+        ctx.eval_configs = [self._make_eval_config(simple_dataset)]
+        phases = _build_phases(ctx)
+
+        acc = next(p for p in phases if p.phase_type == PhaseType.ACCURACY)
+        assert acc.runtime_settings.load_pattern is not None
+        assert acc.runtime_settings.load_pattern.type == LoadPatternType.POISSON
+        assert acc.runtime_settings.load_pattern.target_qps == 10.0
+
+    @pytest.mark.unit
+    def test_accuracy_phase_max_throughput_when_perf_offline(
+        self, base_rt_settings, simple_dataset
+    ):
+        """Offline (MAX_THROUGHPUT) perf leaves accuracy at MAX_THROUGHPUT."""
+        config = OfflineConfig(**_OFFLINE_KWARGS)
+        ctx = self._make_ctx(config, base_rt_settings, simple_dataset)
+        ctx.eval_configs = [self._make_eval_config(simple_dataset)]
+        phases = _build_phases(ctx)
+
+        acc = next(p for p in phases if p.phase_type == PhaseType.ACCURACY)
+        assert acc.runtime_settings.load_pattern is not None
+        assert acc.runtime_settings.load_pattern.type == LoadPatternType.MAX_THROUGHPUT
+
+    @pytest.mark.unit
+    def test_accuracy_phase_max_throughput_when_perf_agentic(
+        self, base_rt_settings, simple_dataset
+    ):
+        """AGENTIC_INFERENCE can't drive a non-agentic accuracy dataset, so the
+        accuracy phase falls back to MAX_THROUGHPUT instead of crashing."""
+        rt = dataclasses.replace(
+            base_rt_settings,
+            load_pattern=LoadPattern(
+                type=LoadPatternType.AGENTIC_INFERENCE, target_concurrency=8
+            ),
+        )
+        config = OfflineConfig(**_OFFLINE_KWARGS)
+        ctx = self._make_ctx(config, rt, simple_dataset)
+        ctx.eval_configs = [self._make_eval_config(simple_dataset)]
+        phases = _build_phases(ctx)
+
+        acc = next(p for p in phases if p.phase_type == PhaseType.ACCURACY)
+        assert acc.runtime_settings.load_pattern is not None
+        assert acc.runtime_settings.load_pattern.type == LoadPatternType.MAX_THROUGHPUT
+
+    @pytest.mark.unit
+    def test_accuracy_phase_max_throughput_when_perf_none(
+        self, base_rt_settings, simple_dataset
+    ):
+        """A missing perf load pattern falls back to MAX_THROUGHPUT for accuracy."""
+        rt = dataclasses.replace(base_rt_settings, load_pattern=None)
+        config = OfflineConfig(**_OFFLINE_KWARGS)
+        ctx = self._make_ctx(config, rt, simple_dataset)
+        ctx.eval_configs = [self._make_eval_config(simple_dataset)]
+        phases = _build_phases(ctx)
+
+        acc = next(p for p in phases if p.phase_type == PhaseType.ACCURACY)
+        assert acc.runtime_settings.load_pattern is not None
+        assert acc.runtime_settings.load_pattern.type == LoadPatternType.MAX_THROUGHPUT
+
+    @pytest.mark.unit
+    def test_accuracy_issuer_logs_load_mode(
+        self, base_rt_settings, simple_dataset, caplog
+    ):
+        """The accuracy issuer logs which load mode it will run in."""
+        rt = dataclasses.replace(
+            base_rt_settings,
+            load_pattern=LoadPattern(
+                type=LoadPatternType.CONCURRENCY, target_concurrency=4
+            ),
+        )
+        config = OfflineConfig(**_OFFLINE_KWARGS)
+        ctx = self._make_ctx(config, rt, simple_dataset)
+        ctx.eval_configs = [self._make_eval_config(simple_dataset)]
+
+        with caplog.at_level(
+            logging.INFO, logger="inference_endpoint.commands.benchmark.execute"
+        ):
+            _build_phases(ctx)
+
+        msgs = [r.getMessage() for r in caplog.records]
+        assert any(
+            "load mode" in m and "concurrency" in m and "4" in m for m in msgs
+        ), msgs
+
+    @pytest.mark.unit
+    def test_accuracy_issuer_logs_poisson_when_perf_poisson(
+        self, base_rt_settings, simple_dataset, caplog
+    ):
+        """POISSON perf logs the inherited poisson mode with its target_qps and
+        must NOT emit a target_concurrency suffix (the concurrency-only branch)."""
+        rt = dataclasses.replace(
+            base_rt_settings,
+            load_pattern=LoadPattern(type=LoadPatternType.POISSON, target_qps=10.0),
+        )
+        config = OfflineConfig(**_OFFLINE_KWARGS)
+        ctx = self._make_ctx(config, rt, simple_dataset)
+        ctx.eval_configs = [self._make_eval_config(simple_dataset)]
+
+        with caplog.at_level(
+            logging.INFO, logger="inference_endpoint.commands.benchmark.execute"
+        ):
+            _build_phases(ctx)
+
+        msgs = [r.getMessage() for r in caplog.records]
+        load_mode_msgs = [m for m in msgs if "load mode" in m]
+        assert load_mode_msgs, msgs
+        assert any(
+            "poisson" in m and "target_qps=10.0" in m for m in load_mode_msgs
+        ), load_mode_msgs
+        assert all(
+            "target_concurrency" not in m for m in load_mode_msgs
+        ), load_mode_msgs
+
     @pytest.mark.unit
     def test_warmup_uses_independent_rng_instances(
         self, base_rt_settings, simple_dataset