[Feat][Router] Add per-model request latency histogram (#940)

vllm:request_latency_seconds with server/model/status labels. observed
once per request in finally, so http 4xx/5xx and exceptions both land
as status=error.

Refs #699.

Signed-off-by: Mikhail Basov <Michael.S.Sinclair@protonmail.com>
Co-authored-by: Rui Zhang <51696593+ruizhang0101@users.noreply.github.com>

Author: banlor

src/tests/test_metrics.py

@@ -0,0 +1,20 @@
+from vllm_router.services.metrics_service import request_latency_seconds
+
+
+def test_records_both_status_labels():
+    request_latency_seconds.labels(
+        server="http://test-engine:8000", model="test-model", status="success"
+    ).observe(0.5)
+    request_latency_seconds.labels(
+        server="http://test-engine:8000", model="test-model", status="error"
+    ).observe(30.0)
+
+    statuses_seen = {
+        s.labels.get("status")
+        for metric in request_latency_seconds.collect()
+        for s in metric.samples
+        if s.name == "vllm:request_latency_seconds_bucket"
+        and s.labels.get("server") == "http://test-engine:8000"
+    }
+
+    assert {"success", "error"} <= statuses_seen

src/vllm_router/services/metrics_service/__init__.py

@@ -1,4 +1,4 @@
-from prometheus_client import Counter, Gauge
+from prometheus_client import Counter, Gauge, Histogram
 
 # --- Prometheus Gauges ---
 # Existing metrics
@@ -63,3 +63,9 @@
     "Total request errors",
     ["server", "model", "error_type"],
 )
+request_latency_seconds = Histogram(
+    "vllm:request_latency_seconds",
+    "End-to-end request latency observed at the router",
+    ["server", "model", "status"],
+    buckets=(0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0),
+)

src/vllm_router/services/request_service/request.py

@@ -74,6 +74,7 @@
     num_incoming_requests_total,
     output_tokens_total,
     request_errors_total,
+    request_latency_seconds,
 )
 
 logger = init_logger(__name__)
@@ -298,6 +299,9 @@ async def process_request(
     # For non-streaming requests, collect the full response to cache it properly
     full_response = bytearray()
 
+    request_status = "success"
+    http_status_code = None
+
     try:
         async with request.app.state.aiohttp_client_wrapper().request(
             method=request.method,
@@ -306,6 +310,7 @@ async def process_request(
             data=body,
             timeout=aiohttp.ClientTimeout(total=None),
         ) as backend_response:
+            http_status_code = backend_response.status
             # Set response status on span if tracing
             if span is not None:
                 span.set_attribute("http.status_code", backend_response.status)
@@ -325,10 +330,14 @@ async def process_request(
                     full_response.extend(chunk)
                 yield chunk
 
+        end_time = time.time()
         request.app.state.request_stats_monitor.on_request_complete(
-            backend_url, request_id, time.time()
+            backend_url, request_id, end_time
         )
 
+        if http_status_code is not None and http_status_code >= 400:
+            request_status = "error"
+
         # Track token usage for non-streaming requests
         if not is_streaming and full_response:
             try:
@@ -357,13 +366,17 @@ async def process_request(
                 request.app.state.callbacks.post_request, request, full_response
             )
     except Exception as e:
+        request_status = "error"
         # Track other errors
         request_errors_total.labels(
             server=backend_url, model=model_name, error_type=type(e).__name__
         ).inc()
         end_span(span, error=e) if tracing_active else None
         raise
     finally:
+        request_latency_seconds.labels(
+            server=backend_url, model=model_name, status=request_status
+        ).observe(time.time() - start_time)
         end_span(span) if tracing_active else None