Merge remote-tracking branch 'upstream/main'

Author: vishbhat

benchmarks/benchmark.py

@@ -5,13 +5,14 @@
 Orchestrates guidellm burst/idle cycles alongside batch workloads to measure
 whether gated dispatch protects interactive latency while batch makes SLO progress.
 
-Supports 6 scenarios:
+Supports 7 scenarios:
   0 - Interactive only (baseline)
   1 - No batch-gateway (batch as regular requests)
   2 - Ungated batch (aggressive concurrency, no AIMD)
-  3 - AIMD only (processor-side adaptive concurrency)
-  4 - AIMD + llm-d Router flow control (two-layer protection)
+  3 - Admission control + AIMD (saturation-based batch rejection + adaptive concurrency)
+  4 - Flow control + AIMD (priority dispatch ordering + adaptive concurrency)
   5 - Async processor (blocked on integration)
+  6 - Low batch concurrency (fixed perEndpoint cap, no AIMD)
 
 Usage:
     python3 benchmarks/benchmark.py \
@@ -43,9 +44,10 @@
     0: "interactive-only",
     1: "no-batch-gateway",
     2: "ungated",
-    3: "aimd",
-    4: "aimd-flow-control",
+    3: "admission-control-aimd",
+    4: "flow-control-aimd",
     5: "async",
+    6: "low-concurrency",
 }
 
 SCRIPT_DIR = Path(__file__).parent.resolve()
@@ -1183,6 +1185,7 @@ def _generate_narrative(results, cfg):
     ungated = next((r for r in results if r.scenario == 2), None)
     aimd = next((r for r in results if r.scenario == 3), None)
     fc = next((r for r in results if r.scenario == 4), None)
+    low_conc = next((r for r in results if r.scenario == 6), None)
 
     lines = []
     baseline_burst = _aggregate_phases(baseline.phases, "burst") if baseline else None
@@ -1204,17 +1207,25 @@ def _generate_narrative(results, cfg):
         aimd_burst = _aggregate_phases(aimd.phases, "burst")
         if aimd_burst and baseline_ttft:
             overhead = ((aimd_burst["ttft_p99"] - baseline_ttft) / baseline_ttft) * 100
-            lines.append(f"AIMD-gated batch (S3) protected interactive TTFT p99 within "
+            lines.append(f"Admission control + AIMD (S3) protected interactive TTFT p99 within "
                          f"{overhead:.0f}% of baseline ({aimd_burst['ttft_p99']:.0f} ms) "
-                         f"using lower static concurrency (perEndpoint: 30).")
+                         f"using saturation-based batch rejection and adaptive concurrency.")
 
     if fc:
         fc_burst = _aggregate_phases(fc.phases, "burst")
         if fc_burst and baseline_ttft:
             overhead = ((fc_burst["ttft_p99"] - baseline_ttft) / baseline_ttft) * 100
-            lines.append(f"AIMD + flow control (S4) achieved the best protection at "
+            lines.append(f"Flow control + AIMD (S4) achieved the best protection at "
                          f"{overhead:.0f}% above baseline ({fc_burst['ttft_p99']:.0f} ms) "
-                         f"with proactive Router-side batch shedding.")
+                         f"with priority dispatch ordering (interactive before batch).")
+
+    if low_conc:
+        low_conc_burst = _aggregate_phases(low_conc.phases, "burst")
+        if low_conc_burst and baseline_ttft:
+            overhead = ((low_conc_burst["ttft_p99"] - baseline_ttft) / baseline_ttft) * 100
+            lines.append(f"Low concurrency (S6) limited interactive TTFT p99 impact to "
+                         f"{overhead:.0f}% above baseline ({low_conc_burst['ttft_p99']:.0f} ms) "
+                         f"using a fixed per-endpoint concurrency cap.")
 
     # Batch completion summary with per-job SLO status
     for r in results:
@@ -1318,7 +1329,7 @@ def generate_html_report(cfg, results):
         if result.batch_timeline:
             timelines_json[f"S{result.scenario} ({result.name})"] = result.batch_timeline
 
-    # AIMD dynamics data (scenarios 3-4)
+    # AIMD dynamics data (scenarios with AIMD enabled)
     aimd_chart_data = []
     for result in results:
         if result.aimd_metrics and "aimd_concurrency_limit_series" in result.aimd_metrics:
@@ -1376,8 +1387,12 @@ def generate_html_report(cfg, results):
         burst_agg = _aggregate_phases(result.phases, "burst")
         idle_agg = _aggregate_phases(result.phases, "idle")
 
-        ttft_p99_burst = f"{burst_agg['ttft_p99']:.1f}" if burst_agg else "N/A"
-        itl_p99_burst = f"{burst_agg['itl_p99']:.1f}" if burst_agg else "N/A"
+        if burst_agg:
+            ttft_burst = f"{burst_agg['ttft_p50']:.0f} / {burst_agg['ttft_p95']:.0f} / {burst_agg['ttft_p99']:.0f}"
+            itl_burst = f"{burst_agg['itl_p50']:.0f} / {burst_agg['itl_p95']:.0f} / {burst_agg['itl_p99']:.0f}"
+        else:
+            ttft_burst = "N/A"
+            itl_burst = "N/A"
         idle_rps = f"{idle_agg['completed'] / max(1, cfg.idle_seconds * (cfg.cycles - cfg.warmup_cycles)):.2f}" if idle_agg else "N/A"
 
         if result.scenario <= 1:
@@ -1399,8 +1414,8 @@ def generate_html_report(cfg, results):
 
         summary_rows.append(
             f"<tr><td>S{result.scenario} ({result.name})</td>"
-            f"<td>{ttft_p99_burst}</td>"
-            f"<td>{itl_p99_burst}</td>"
+            f"<td>{ttft_burst}</td>"
+            f"<td>{itl_burst}</td>"
             f"<td>{idle_rps}</td>"
             f"<td>{batch_slo}</td>"
             f"<td>{batch_ttft_p50}</td></tr>"
@@ -1501,8 +1516,8 @@ def generate_html_report(cfg, results):
             <table>
                 <tr>
                     <th>Scenario</th>
-                    <th>Interactive TTFT p99 (burst)</th>
-                    <th>Interactive ITL p99 (burst)</th>
+                    <th>Interactive TTFT at peak (p50 / p95 / p99)</th>
+                    <th>Interactive ITL at peak (p50 / p95 / p99)</th>
                     <th>Batch idle throughput (req/s)</th>
                     <th>Batch SLO completion</th>
                     <th>Batch TTFT p50</th>
@@ -1563,7 +1578,7 @@ def generate_html_report(cfg, results):
             </div>
         </div>
 
-        <h2>AIMD Concurrency Dynamics (Scenarios 3-4)</h2>
+        <h2>AIMD Concurrency Dynamics</h2>
         <div class="chart-container">
             <canvas id="aimdChart"></canvas>
         </div>
@@ -1965,7 +1980,7 @@ def run_scenario(cfg, scenario):
              '{"claimName":"benchmark-results"}}]}}', "--"],
             cfg.context, namespace, check=False)
 
-    # Submit batch (scenarios 2-4 only)
+    # Submit batch (scenarios 2-6)
     if scenario >= 2:
         submit_batches(cfg, namespace)
         time.sleep(10)
@@ -2022,7 +2037,7 @@ def run_scenario(cfg, scenario):
             if metrics.completed > 0:
                 phases.append(metrics)
 
-    # Collect AIMD metrics for scenarios 3-4
+    # Collect AIMD metrics (scenarios with AIMD enabled: S3, S4)
     aimd_metrics = {}
     if scenario >= 3 and os.environ.get("PROMETHEUS_URL"):
         end_time = datetime.datetime.utcfromtimestamp(time.time())
@@ -2506,7 +2521,7 @@ def main():
     # Validate scenarios
     for s in cfg.scenarios:
         if s not in SCENARIO_NAMES:
-            log(f"ERROR: Unknown scenario {s}. Valid: 0-5")
+            log(f"ERROR: Unknown scenario {s}. Valid: 0-6")
             sys.exit(1)
 
     # Rate sweep mode
@@ -2674,7 +2689,7 @@ def main():
                 "total_errors": sum(p.errors for p in burst_phases),
             }
 
-        # AIMD and flow control metrics (scenarios 3-4)
+        # AIMD and flow control metrics
         if result.aimd_metrics:
             scenario_data["aimd"] = {
                 k: v for k, v in result.aimd_metrics.items()

benchmarks/helm-values/scenario-3-admission-control-aimd.yaml

@@ -0,0 +1,26 @@
+# Scenario 3: Admission control + AIMD
+# Processor sends x-gateway-inference-objective: "batch-sheddable" header.
+# The llm-d Router's admission controller (flow control feature gate OFF)
+# rejects sheddable requests (priority < 0) when saturation >= 1.0,
+# generating 429s that feed AIMD multiplicative decrease.
+#
+# InferenceObjective CRDs and Router overlay are deployed by setup.sh.
+processor:
+  config:
+    dispatchMode: sync
+    concurrency:
+      global: 60
+      perEndpoint: 30
+      recovery: 5
+      aimd:
+        enabled: true
+        min: 5
+        backoffFactor: 0.5
+        additiveIncrease: 1
+    globalInferenceGateway:
+      url: "http://llm-d-inference-gateway-istio:80"
+      inferenceObjective: "batch-sheddable"
+      requestTimeout: "30s"
+      maxRetries: 3
+      initialBackoff: "1s"
+      maxBackoff: "10s"

benchmarks/helm-values/scenario-3-admission-control-overlay-router.yaml

@@ -0,0 +1,23 @@
+# Admission control overlay for llm-d Router (router.epp.* format for v0.9.0+ charts).
+# Applied in scenario 3 to enable saturation-based admission control:
+#   - Flow control feature gate is OFF (no priority dispatch ordering)
+#   - Concurrency-detector provides the saturation signal
+#   - LegacyAdmissionController rejects sheddable requests (priority < 0)
+#     when saturation >= 1.0
+#
+# maxConcurrency should be tuned to the model's observed peak in-flight
+# requests. When total in-flight exceeds maxConcurrency, saturation
+# crosses 1.0 and batch requests are rejected with 429.
+router:
+  epp:
+    pluginsConfigFile: "admission-control-plugins.yaml"
+    pluginsCustomConfig:
+      admission-control-plugins.yaml: |
+        apiVersion: inference.networking.x-k8s.io/v1alpha1
+        kind: EndpointPickerConfig
+        plugins:
+          - type: concurrency-detector
+            parameters:
+              maxConcurrency: 50
+        saturationDetector:
+          pluginRef: concurrency-detector

benchmarks/helm-values/scenario-3-admission-control-overlay.yaml

@@ -0,0 +1,22 @@
+# Admission control overlay for llm-d Router (InferencePool EPP chart).
+# Applied in scenario 3 to enable saturation-based admission control:
+#   - Flow control feature gate is OFF (no priority dispatch ordering)
+#   - Concurrency-detector provides the saturation signal
+#   - LegacyAdmissionController rejects sheddable requests (priority < 0)
+#     when saturation >= 1.0
+#
+# maxConcurrency should be tuned to the model's observed peak in-flight
+# requests. When total in-flight exceeds maxConcurrency, saturation
+# crosses 1.0 and batch requests are rejected with 429.
+inferenceExtension:
+  pluginsConfigFile: "admission-control-plugins.yaml"
+  pluginsCustomConfig:
+    admission-control-plugins.yaml: |
+      apiVersion: inference.networking.x-k8s.io/v1alpha1
+      kind: EndpointPickerConfig
+      plugins:
+        - type: concurrency-detector
+          parameters:
+            maxConcurrency: 50
+      saturationDetector:
+        pluginRef: concurrency-detector

benchmarks/helm-values/scenario-3-aimd.yaml

@@ -1,16 +1,16 @@
-# Scenario 4: AIMD + llm-d Router flow control
-# Same processor AIMD config as scenario 3, plus two-layer protection:
+# Scenario 4: Flow control + AIMD
+# Same processor AIMD config as scenario 3, plus priority dispatch ordering:
 # - llm-d Router flowControl feature gate enabled (EndpointPickerConfig)
 # - Priority bands: interactive (100) + batch (-1, sheddable)
 # - InferenceObjective CRDs: interactive-default (100), batch-sheddable (-1)
 # - Processor sends x-gateway-inference-objective: "batch-sheddable" header
 #
-# With flow control active, the Router returns 429 for batch requests during
-# saturation. This triggers AIMD multiplicative decrease, producing the
-# expected sawtooth pattern. Lower perEndpoint (30) same as S3 for fair
-# comparison; the differentiator here is the Router's proactive 429 shedding.
+# Flow control's dispatch cycle dispatches one request per tick, highest
+# priority first. Interactive (100) always dispatches before batch (-1).
+# This priority ordering passively protects interactive latency.
+# AIMD is enabled but stays flat without 429s (ready for future integration).
 #
-# CRDs and Router overlay are deployed by setup.sh (scenario 4 block).
+# CRDs and Router overlay are deployed by setup.sh.
 processor:
   config:
     dispatchMode: sync

…values/scenario-4-aimd-flow-control.yaml …values/scenario-4-flow-control-aimd.yamlbenchmarks/helm-values/scenario-4-aimd-flow-control.yaml renamed to benchmarks/helm-values/scenario-4-flow-control-aimd.yaml benchmarks/helm-values/scenario-4-aimd-flow-control.yaml renamed to benchmarks/helm-values/scenario-4-flow-control-aimd.yaml

@@ -1,7 +1,19 @@
 # Flow control overlay for llm-d Router (router.epp.* format for v0.9.0+ charts).
-# Applied in scenario 4 to enable two-layer protection:
-#   Layer 1: Router EPP flow control (priority-based request scheduling)
-#   Layer 2: Processor AIMD (adaptive concurrency per endpoint)
+# Applied in scenario 4 to enable priority dispatch ordering:
+#   Flow control's dispatch cycle dispatches one request per tick, highest
+#   priority first. Interactive (priority 100) always dispatches before
+#   batch (priority -1). This priority ordering passively protects
+#   interactive latency.
+#
+# Saturation detector choice: concurrency-detector with generous maxConcurrency
+# (1000) is a benchmarking workaround — it keeps saturation artificially low so
+# the dispatch cycle runs freely without rejection. This isolates priority
+# ordering as the sole protection mechanism. For production, consider:
+#   - utilization-detector (reacts to real GPU pressure: queue depth, KV cache)
+#   - concurrency-detector with a tuned maxConcurrency matching model capacity
+# Both are currently affected by llm-d-router bugs (#1474, #2060) that make the
+# saturation signal unreliable. Once fixed, combine with priority-holdback-policy
+# for graduated per-band shedding.
 #
 # Priority bands:
 #   Interactive (100): low-latency, round-robin fairness, FCFS ordering
@@ -19,10 +31,9 @@ router:
           - type: round-robin-fairness-policy
           - type: global-strict-fairness-policy
           - type: slo-deadline-ordering-policy
-          - type: utilization-detector
+          - type: concurrency-detector
             parameters:
-              queueDepthThreshold: 2
-              kvCacheUtilThreshold: 0.5
+              maxConcurrency: 1000
         flowControl:
           maxBytes: 4294967296
           defaultRequestTTL: 30s
@@ -39,5 +50,5 @@ router:
             maxBytes: 536870912
             fairnessPolicyRef: global-strict-fairness-policy
             orderingPolicyRef: fcfs-ordering-policy
-          saturationDetector:
-            pluginRef: utilization-detector
+        saturationDetector:
+          pluginRef: concurrency-detector

benchmarks/helm-values/scenario-4-flow-control-overlay-router.yaml

@@ -1,7 +1,19 @@
 # Flow control overlay for llm-d Router (InferencePool EPP chart).
-# Applied in scenario 4 to enable two-layer protection:
-#   Layer 1: Router EPP flow control (priority-based request scheduling)
-#   Layer 2: Processor AIMD (adaptive concurrency per endpoint)
+# Applied in scenario 4 to enable priority dispatch ordering:
+#   Flow control's dispatch cycle dispatches one request per tick, highest
+#   priority first. Interactive (priority 100) always dispatches before
+#   batch (priority -1). This priority ordering passively protects
+#   interactive latency.
+#
+# Saturation detector choice: concurrency-detector with generous maxConcurrency
+# (1000) is a benchmarking workaround — it keeps saturation artificially low so
+# the dispatch cycle runs freely without rejection. This isolates priority
+# ordering as the sole protection mechanism. For production, consider:
+#   - utilization-detector (reacts to real GPU pressure: queue depth, KV cache)
+#   - concurrency-detector with a tuned maxConcurrency matching model capacity
+# Both are currently affected by llm-d-router bugs (#1474, #2060) that make the
+# saturation signal unreliable. Once fixed, combine with priority-holdback-policy
+# for graduated per-band shedding.
 #
 # Priority bands:
 #   Interactive (100): low-latency, round-robin fairness, FCFS ordering
@@ -18,10 +30,9 @@ inferenceExtension:
         - type: round-robin-fairness-policy
         - type: global-strict-fairness-policy
         - type: slo-deadline-ordering-policy
-        - type: utilization-detector
+        - type: concurrency-detector
           parameters:
-            queueDepthThreshold: 2
-            kvCacheUtilThreshold: 0.5
+            maxConcurrency: 1000
       flowControl:
         maxBytes: 4294967296
         defaultRequestTTL: 30s
@@ -38,5 +49,5 @@ inferenceExtension:
           maxBytes: 536870912
           fairnessPolicyRef: global-strict-fairness-policy
           orderingPolicyRef: fcfs-ordering-policy
-        saturationDetector:
-          pluginRef: utilization-detector
+      saturationDetector:
+        pluginRef: concurrency-detector