Skip to content

Commit eb04f5f

Browse files
yinggehyinggeh
authored
feat: Extend histogram metrics to non-decoupled models (#8580)
1 parent dd2fc4f commit eb04f5f

5 files changed

Lines changed: 39 additions & 73 deletions

File tree

‎docs/user_guide/metrics.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<!--
2-
# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
33
#
44
# Redistribution and use in source and binary forms, with or without
55
# modification, are permitted provided that the following conditions
@@ -215,9 +215,9 @@ By default, the following
215215
[Histogram](https://prometheus.io/docs/concepts/metric_types/#histogram)
216216
metrics are used for latencies:
217217

218-
|Category |Metric |Metric Name |Description |Granularity|Frequency |Model Type
219-
|--------------|----------------|------------|---------------------------|-----------|-------------|-------------|
220-
|Latency |Request to First Response Time |`nv_inference_first_response_histogram_ms` |Histogram of end-to-end inference request to the first response time |Per model |Per request | Decoupled |
218+
|Category |Metric |Metric Name |Description |Granularity|Frequency |
219+
|--------------|----------------|------------|---------------------------|-----------|-------------|
220+
|Latency |Request to First Response Time |`nv_inference_first_response_histogram_ms` |Histogram of end-to-end inference request to the first response time |Per model |Per request |
221221

222222
To enable these metrics specifically, you can set `--metrics-config histogram_latencies=true`
223223

‎qa/L0_metrics/ensemble_decoupled/async_execute_decouple/1/model.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
1+
# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
#
33
# Redistribution and use in source and binary forms, with or without
44
# modification, are permitted provided that the following conditions
@@ -45,9 +45,9 @@ async def execute(self, requests):
4545
]
4646

4747
# Wait
48-
time.sleep(wait_secs.item())
4948
response_sender = request.get_response_sender()
5049
for i in range(response_num):
50+
time.sleep(wait_secs.item())
5151
response = pb_utils.InferenceResponse(output_tensors)
5252
if i != response_num - 1:
5353
response_sender.send(response)

‎qa/L0_metrics/histogram_metrics_test.py‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/usr/bin/python
2-
# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
33
#
44
# Redistribution and use in source and binary forms, with or without
55
# modification, are permitted provided that the following conditions
@@ -114,7 +114,7 @@ def callback(user_data, result, error):
114114
def test_ensemble_decoupled(self):
115115
wait_secs = 1
116116
responses_per_req = 3
117-
total_reqs = 3
117+
total_iters = 3
118118
delta = 0.2
119119

120120
# Infer
@@ -133,7 +133,7 @@ def test_ensemble_decoupled(self):
133133
inputs[1].set_data_from_numpy(input_data_1)
134134

135135
# Send requests to ensemble decoupled model
136-
for request_num in range(1, total_reqs + 1):
136+
for iter_cnt in range(1, total_iters + 1):
137137
ensemble_model_name = "ensemble"
138138
decoupled_model_name = "async_execute_decouple"
139139
non_decoupled_model_name = "async_execute"
@@ -144,40 +144,40 @@ def test_ensemble_decoupled(self):
144144
# Checks metrics output
145145
histogram_dict = self.get_histogram_metrics(FIRST_RESPONSE_HISTOGRAM)
146146

147-
def check_existing_metrics(model_name, wait_secs_per_req, delta):
148-
metric_count = get_histogram_metric_key(
147+
def check_histogram(model_name, request_cnt, wait_secs_per_req, delta):
148+
histogram_count_key = get_histogram_metric_key(
149149
FIRST_RESPONSE_HISTOGRAM, model_name, "1", "count"
150150
)
151-
metric_sum = get_histogram_metric_key(
151+
histogram_sum_key = get_histogram_metric_key(
152152
FIRST_RESPONSE_HISTOGRAM, model_name, "1", "sum"
153153
)
154154
# Test histogram count
155-
self.assertIn(metric_count, histogram_dict)
156-
self.assertEqual(histogram_dict[metric_count], request_num)
155+
self.assertIn(histogram_count_key, histogram_dict)
156+
self.assertEqual(
157+
histogram_dict[histogram_count_key], request_cnt * iter_cnt
158+
)
157159
# Test histogram sum
158-
self.assertIn(metric_sum, histogram_dict)
160+
self.assertIn(histogram_sum_key, histogram_dict)
159161
self.assertTrue(
160-
wait_secs_per_req * MILLIS_PER_SEC * request_num
161-
<= histogram_dict[metric_sum]
162-
< (wait_secs_per_req + delta) * MILLIS_PER_SEC * request_num
162+
wait_secs_per_req * MILLIS_PER_SEC * request_cnt * iter_cnt
163+
<= histogram_dict[histogram_sum_key]
164+
< (wait_secs_per_req + delta)
165+
* MILLIS_PER_SEC
166+
* request_cnt
167+
* iter_cnt
163168
)
164169
# Prometheus histogram buckets are tested in metrics_api_test.cc::HistogramAPIHelper
165170

166171
# Test ensemble model metrics
167-
check_existing_metrics(ensemble_model_name, 2 * wait_secs, 2 * delta)
172+
check_histogram(ensemble_model_name, 1, wait_secs * 2, 2 * delta)
168173

169174
# Test decoupled model metrics
170-
check_existing_metrics(decoupled_model_name, wait_secs, delta)
175+
check_histogram(decoupled_model_name, 1, wait_secs, delta)
171176

172177
# Test non-decoupled model metrics
173-
non_decoupled_model_count = get_histogram_metric_key(
174-
FIRST_RESPONSE_HISTOGRAM, non_decoupled_model_name, "1", "count"
175-
)
176-
non_decoupled_model_sum = get_histogram_metric_key(
177-
FIRST_RESPONSE_HISTOGRAM, non_decoupled_model_name, "1", "sum"
178+
check_histogram(
179+
non_decoupled_model_name, responses_per_req, wait_secs, delta
178180
)
179-
self.assertNotIn(non_decoupled_model_count, histogram_dict)
180-
self.assertNotIn(non_decoupled_model_sum, histogram_dict)
181181

182182
def test_buckets_override(self):
183183
model_name = "async_execute_decouple"

‎qa/L0_metrics/metrics_config_test.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/usr/bin/python
2-
# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
33
#
44
# Redistribution and use in source and binary forms, with or without
55
# modification, are permitted provided that the following conditions
@@ -44,7 +44,7 @@
4444
"nv_inference_compute_infer_duration",
4545
"nv_inference_compute_output_duration",
4646
]
47-
INF_HISTOGRAM_DECOUPLED_PATTERNS = ["nv_inference_first_response_histogram_ms"]
47+
INF_HISTOGRAM_PATTERNS = ["nv_inference_first_response_histogram_ms"]
4848
INF_SUMMARY_PATTERNS = [
4949
"nv_inference_request_summary",
5050
"nv_inference_queue_summary",
@@ -99,15 +99,15 @@ def test_cache_counters_missing(self):
9999
self.assertNotIn(metric, metrics)
100100

101101
# Histograms
102-
def test_inf_histograms_decoupled_exist(self):
102+
def test_inf_histograms_exist(self):
103103
metrics = self._get_metrics()
104-
for metric in INF_HISTOGRAM_DECOUPLED_PATTERNS:
104+
for metric in INF_HISTOGRAM_PATTERNS:
105105
for suffix in ["_count", "_sum", "_bucket"]:
106106
self.assertIn(metric + suffix, metrics)
107107

108-
def test_inf_histograms_decoupled_missing(self):
108+
def test_inf_histograms_missing(self):
109109
metrics = self._get_metrics()
110-
for metric in INF_HISTOGRAM_DECOUPLED_PATTERNS:
110+
for metric in INF_HISTOGRAM_PATTERNS:
111111
self.assertNotIn(metric, metrics)
112112

113113
# Summaries

‎qa/L0_metrics/test.sh‎

Lines changed: 6 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
33
#
44
# Redistribution and use in source and binary forms, with or without
55
# modification, are permitted provided that the following conditions
@@ -276,7 +276,7 @@ SERVER_ARGS="${BASE_SERVER_ARGS} --load-model=identity_cache_off"
276276
run_and_check_server
277277
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_counters_exist 2>&1 | tee ${CLIENT_LOG}
278278
check_unit_test
279-
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_missing 2>&1 | tee ${CLIENT_LOG}
279+
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_missing 2>&1 | tee ${CLIENT_LOG}
280280
check_unit_test
281281
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_summaries_missing 2>&1 | tee ${CLIENT_LOG}
282282
check_unit_test
@@ -286,47 +286,12 @@ python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_summaries_missing 2>&1 | tee
286286
check_unit_test
287287
kill_server
288288

289-
# Check default settings: Histograms should be always disabled in non-decoupled model.
289+
# Enable histograms
290290
SERVER_ARGS="${BASE_SERVER_ARGS} --load-model=identity_cache_off --metrics-config histogram_latencies=true"
291291
run_and_check_server
292292
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_counters_exist 2>&1 | tee ${CLIENT_LOG}
293293
check_unit_test
294-
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_missing 2>&1 | tee ${CLIENT_LOG}
295-
check_unit_test
296-
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_summaries_missing 2>&1 | tee ${CLIENT_LOG}
297-
check_unit_test
298-
python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_counters_missing 2>&1 | tee ${CLIENT_LOG}
299-
check_unit_test
300-
python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_summaries_missing 2>&1 | tee ${CLIENT_LOG}
301-
check_unit_test
302-
kill_server
303-
304-
# Check default settings: Histograms should be disabled in decoupled model
305-
decoupled_model="async_execute_decouple"
306-
mkdir -p "${MODELDIR}/${decoupled_model}/1/"
307-
cp ../python_models/${decoupled_model}/model.py ${MODELDIR}/${decoupled_model}/1/
308-
cp ../python_models/${decoupled_model}/config.pbtxt ${MODELDIR}/${decoupled_model}/
309-
310-
SERVER_ARGS="${BASE_SERVER_ARGS} --load-model=${decoupled_model}"
311-
run_and_check_server
312-
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_counters_exist 2>&1 | tee ${CLIENT_LOG}
313-
check_unit_test
314-
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_missing 2>&1 | tee ${CLIENT_LOG}
315-
check_unit_test
316-
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_summaries_missing 2>&1 | tee ${CLIENT_LOG}
317-
check_unit_test
318-
python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_counters_missing 2>&1 | tee ${CLIENT_LOG}
319-
check_unit_test
320-
python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_summaries_missing 2>&1 | tee ${CLIENT_LOG}
321-
check_unit_test
322-
kill_server
323-
324-
# Enable histograms in decoupled model
325-
SERVER_ARGS="${BASE_SERVER_ARGS} --load-model=${decoupled_model} --metrics-config histogram_latencies=true"
326-
run_and_check_server
327-
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_counters_exist 2>&1 | tee ${CLIENT_LOG}
328-
check_unit_test
329-
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_exist 2>&1 | tee ${CLIENT_LOG}
294+
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_exist 2>&1 | tee ${CLIENT_LOG}
330295
check_unit_test
331296
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_summaries_missing 2>&1 | tee ${CLIENT_LOG}
332297
check_unit_test
@@ -465,6 +430,7 @@ SERVER_LOG="./histogram_ensemble_decoupled_server.log"
465430
CLIENT_LOG="./histogram_ensemble_decoupled_client.log"
466431
SERVER_ARGS="--model-repository=${MODELDIR} --metrics-config histogram_latencies=true --log-verbose=1"
467432
mkdir -p "${MODELDIR}"/ensemble/1
433+
rm -rf "${MODELDIR}"/async_execute
468434
cp -r "${MODELDIR}"/async_execute_decouple "${MODELDIR}"/async_execute
469435
sed -i "s/model_transaction_policy { decoupled: True }//" "${MODELDIR}"/async_execute/config.pbtxt
470436

@@ -510,7 +476,7 @@ kill_server
510476
PYTHON_TEST="metrics_config_test.py"
511477
SERVER_ARGS="--model-repository=${MODELDIR} --model-control-mode=explicit --load-model=${decoupled_model} --metrics-config histogram_latencies=false --log-verbose=1"
512478
run_and_check_server
513-
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_missing 2>&1 | tee ${CLIENT_LOG}
479+
python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_missing 2>&1 | tee ${CLIENT_LOG}
514480
check_unit_test
515481
kill_server
516482

0 commit comments

Comments
 (0)