feat: Extend histogram metrics to non-decoupled models

docs/user_guide/metrics.md

@@ -1,5 +1,5 @@
 <!--
-# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -215,9 +215,9 @@ By default, the following
 [Histogram](https://prometheus.io/docs/concepts/metric_types/#histogram)
 metrics are used for latencies:
 
-|Category      |Metric          |Metric Name |Description                |Granularity|Frequency    |Model Type
-|--------------|----------------|------------|---------------------------|-----------|-------------|-------------|
-|Latency       |Request to First Response Time    |`nv_inference_first_response_histogram_ms` |Histogram of end-to-end inference request to the first response time |Per model  |Per request  | Decoupled |
+|Category      |Metric          |Metric Name |Description                |Granularity|Frequency    |
+|--------------|----------------|------------|---------------------------|-----------|-------------|
+|Latency       |Request to First Response Time    |`nv_inference_first_response_histogram_ms` |Histogram of end-to-end inference request to the first response time |Per model  |Per request  |
 
 To enable these metrics specifically, you can set `--metrics-config histogram_latencies=true`
 

qa/L0_metrics/ensemble_decoupled/async_execute_decouple/1/model.py

@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -45,9 +45,9 @@ async def execute(self, requests):
         ]
 
         # Wait
-        time.sleep(wait_secs.item())
         response_sender = request.get_response_sender()
         for i in range(response_num):
+            time.sleep(wait_secs.item())
             response = pb_utils.InferenceResponse(output_tensors)
             if i != response_num - 1:
                 response_sender.send(response)

qa/L0_metrics/histogram_metrics_test.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -114,7 +114,7 @@ def callback(user_data, result, error):
     def test_ensemble_decoupled(self):
         wait_secs = 1
         responses_per_req = 3
-        total_reqs = 3
+        total_iters = 3
         delta = 0.2
 
         # Infer
@@ -133,7 +133,7 @@ def test_ensemble_decoupled(self):
         inputs[1].set_data_from_numpy(input_data_1)
 
         # Send requests to ensemble decoupled model
-        for request_num in range(1, total_reqs + 1):
+        for iter_cnt in range(1, total_iters + 1):
             ensemble_model_name = "ensemble"
             decoupled_model_name = "async_execute_decouple"
             non_decoupled_model_name = "async_execute"
@@ -144,40 +144,40 @@ def test_ensemble_decoupled(self):
             # Checks metrics output
             histogram_dict = self.get_histogram_metrics(FIRST_RESPONSE_HISTOGRAM)
 
-            def check_existing_metrics(model_name, wait_secs_per_req, delta):
-                metric_count = get_histogram_metric_key(
+            def check_histogram(model_name, request_cnt, wait_secs_per_req, delta):
+                histogram_count_key = get_histogram_metric_key(
                     FIRST_RESPONSE_HISTOGRAM, model_name, "1", "count"
                 )
-                metric_sum = get_histogram_metric_key(
+                histogram_sum_key = get_histogram_metric_key(
                     FIRST_RESPONSE_HISTOGRAM, model_name, "1", "sum"
                 )
                 # Test histogram count
-                self.assertIn(metric_count, histogram_dict)
-                self.assertEqual(histogram_dict[metric_count], request_num)
+                self.assertIn(histogram_count_key, histogram_dict)
+                self.assertEqual(
+                    histogram_dict[histogram_count_key], request_cnt * iter_cnt
+                )
                 # Test histogram sum
-                self.assertIn(metric_sum, histogram_dict)
+                self.assertIn(histogram_sum_key, histogram_dict)
                 self.assertTrue(
-                    wait_secs_per_req * MILLIS_PER_SEC * request_num
-                    <= histogram_dict[metric_sum]
-                    < (wait_secs_per_req + delta) * MILLIS_PER_SEC * request_num
+                    wait_secs_per_req * MILLIS_PER_SEC * request_cnt * iter_cnt
+                    <= histogram_dict[histogram_sum_key]
+                    < (wait_secs_per_req + delta)
+                    * MILLIS_PER_SEC
+                    * request_cnt
+                    * iter_cnt
                 )
                 # Prometheus histogram buckets are tested in metrics_api_test.cc::HistogramAPIHelper
 
             # Test ensemble model metrics
-            check_existing_metrics(ensemble_model_name, 2 * wait_secs, 2 * delta)
+            check_histogram(ensemble_model_name, 1, wait_secs * 2, 2 * delta)
 
             # Test decoupled model metrics
-            check_existing_metrics(decoupled_model_name, wait_secs, delta)
+            check_histogram(decoupled_model_name, 1, wait_secs, delta)
 
             # Test non-decoupled model metrics
-            non_decoupled_model_count = get_histogram_metric_key(
-                FIRST_RESPONSE_HISTOGRAM, non_decoupled_model_name, "1", "count"
-            )
-            non_decoupled_model_sum = get_histogram_metric_key(
-                FIRST_RESPONSE_HISTOGRAM, non_decoupled_model_name, "1", "sum"
+            check_histogram(
+                non_decoupled_model_name, responses_per_req, wait_secs, delta
             )
-            self.assertNotIn(non_decoupled_model_count, histogram_dict)
-            self.assertNotIn(non_decoupled_model_sum, histogram_dict)
 
     def test_buckets_override(self):
         model_name = "async_execute_decouple"

qa/L0_metrics/metrics_config_test.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -44,7 +44,7 @@
     "nv_inference_compute_infer_duration",
     "nv_inference_compute_output_duration",
 ]
-INF_HISTOGRAM_DECOUPLED_PATTERNS = ["nv_inference_first_response_histogram_ms"]
+INF_HISTOGRAM_PATTERNS = ["nv_inference_first_response_histogram_ms"]
 INF_SUMMARY_PATTERNS = [
     "nv_inference_request_summary",
     "nv_inference_queue_summary",
@@ -99,15 +99,15 @@ def test_cache_counters_missing(self):
             self.assertNotIn(metric, metrics)
 
     # Histograms
-    def test_inf_histograms_decoupled_exist(self):
+    def test_inf_histograms_exist(self):
         metrics = self._get_metrics()
-        for metric in INF_HISTOGRAM_DECOUPLED_PATTERNS:
+        for metric in INF_HISTOGRAM_PATTERNS:
             for suffix in ["_count", "_sum", "_bucket"]:
                 self.assertIn(metric + suffix, metrics)
 
-    def test_inf_histograms_decoupled_missing(self):
+    def test_inf_histograms_missing(self):
         metrics = self._get_metrics()
-        for metric in INF_HISTOGRAM_DECOUPLED_PATTERNS:
+        for metric in INF_HISTOGRAM_PATTERNS:
             self.assertNotIn(metric, metrics)
 
     # Summaries

qa/L0_metrics/test.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -276,7 +276,7 @@ SERVER_ARGS="${BASE_SERVER_ARGS} --load-model=identity_cache_off"
 run_and_check_server
 python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_counters_exist 2>&1 | tee ${CLIENT_LOG}
 check_unit_test
-python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_missing 2>&1 | tee ${CLIENT_LOG}
+python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_missing 2>&1 | tee ${CLIENT_LOG}
 check_unit_test
 python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_summaries_missing 2>&1 | tee ${CLIENT_LOG}
 check_unit_test
@@ -286,47 +286,12 @@ python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_summaries_missing 2>&1 | tee
 check_unit_test
 kill_server
 
-# Check default settings: Histograms should be always disabled in non-decoupled model.
+# Enable histograms
 SERVER_ARGS="${BASE_SERVER_ARGS} --load-model=identity_cache_off --metrics-config histogram_latencies=true"
 run_and_check_server
 python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_counters_exist 2>&1 | tee ${CLIENT_LOG}
 check_unit_test
-python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_missing 2>&1 | tee ${CLIENT_LOG}
-check_unit_test
-python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_summaries_missing 2>&1 | tee ${CLIENT_LOG}
-check_unit_test
-python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_counters_missing 2>&1 | tee ${CLIENT_LOG}
-check_unit_test
-python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_summaries_missing 2>&1 | tee ${CLIENT_LOG}
-check_unit_test
-kill_server
-
-# Check default settings: Histograms should be disabled in decoupled model
-decoupled_model="async_execute_decouple"
-mkdir -p "${MODELDIR}/${decoupled_model}/1/"
-cp ../python_models/${decoupled_model}/model.py ${MODELDIR}/${decoupled_model}/1/
-cp ../python_models/${decoupled_model}/config.pbtxt ${MODELDIR}/${decoupled_model}/
-
-SERVER_ARGS="${BASE_SERVER_ARGS} --load-model=${decoupled_model}"
-run_and_check_server
-python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_counters_exist 2>&1 | tee ${CLIENT_LOG}
-check_unit_test
-python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_missing 2>&1 | tee ${CLIENT_LOG}
-check_unit_test
-python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_summaries_missing 2>&1 | tee ${CLIENT_LOG}
-check_unit_test
-python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_counters_missing 2>&1 | tee ${CLIENT_LOG}
-check_unit_test
-python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_summaries_missing 2>&1 | tee ${CLIENT_LOG}
-check_unit_test
-kill_server
-
-# Enable histograms in decoupled model
-SERVER_ARGS="${BASE_SERVER_ARGS} --load-model=${decoupled_model} --metrics-config histogram_latencies=true"
-run_and_check_server
-python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_counters_exist 2>&1 | tee ${CLIENT_LOG}
-check_unit_test
-python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_exist 2>&1 | tee ${CLIENT_LOG}
+python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_exist 2>&1 | tee ${CLIENT_LOG}
 check_unit_test
 python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_summaries_missing 2>&1 | tee ${CLIENT_LOG}
 check_unit_test
@@ -465,6 +430,7 @@ SERVER_LOG="./histogram_ensemble_decoupled_server.log"
 CLIENT_LOG="./histogram_ensemble_decoupled_client.log"
 SERVER_ARGS="--model-repository=${MODELDIR} --metrics-config histogram_latencies=true --log-verbose=1"
 mkdir -p "${MODELDIR}"/ensemble/1
+rm -rf "${MODELDIR}"/async_execute
 cp -r "${MODELDIR}"/async_execute_decouple "${MODELDIR}"/async_execute
 sed -i "s/model_transaction_policy { decoupled: True }//" "${MODELDIR}"/async_execute/config.pbtxt
 
@@ -510,7 +476,7 @@ kill_server
 PYTHON_TEST="metrics_config_test.py"
 SERVER_ARGS="--model-repository=${MODELDIR} --model-control-mode=explicit --load-model=${decoupled_model} --metrics-config histogram_latencies=false --log-verbose=1"
 run_and_check_server
-python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_missing 2>&1 | tee ${CLIENT_LOG}
+python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_missing 2>&1 | tee ${CLIENT_LOG}
 check_unit_test
 kill_server