From a2b7b4e94d1dcf23068e3c75579e096d7c672a7b Mon Sep 17 00:00:00 2001 From: Yingge He Date: Fri, 13 Mar 2026 18:27:42 -0700 Subject: [PATCH 1/3] Warm up CUDA cache before all tests --- qa/L0_batcher/batcher_test.py | 111 +++++++++++++++++----------------- qa/L0_batcher/test.sh | 61 ++++++++++++++++++- 2 files changed, 114 insertions(+), 58 deletions(-) diff --git a/qa/L0_batcher/batcher_test.py b/qa/L0_batcher/batcher_test.py index 780eb1e68f..6a133238ef 100755 --- a/qa/L0_batcher/batcher_test.py +++ b/qa/L0_batcher/batcher_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -74,6 +74,7 @@ _ragged_batch_supported_trials.append("libtorch") _max_queue_delay_ms = 10000 +_max_expected_response_ms = 3000 _deferred_exceptions_lock = threading.Lock() _deferred_exceptions = [] @@ -311,13 +312,13 @@ def test_static_batch_preferred(self): self.check_response( trial, 2, - (3000, None), + (_max_expected_response_ms, None), precreated_shm_regions=precreated_shm_regions, ) self.check_response( trial, 6, - (3000, None), + (_max_expected_response_ms, None), precreated_shm_regions=precreated_shm_regions, ) self.check_deferred_exception() @@ -392,7 +393,7 @@ def test_static_batch_gt_max_preferred(self): self.check_response( trial, 7, - (3000, None), + (_max_expected_response_ms, None), precreated_shm_regions=precreated_shm_regions, ) self.check_deferred_exception() @@ -475,7 +476,7 @@ def test_multi_batch_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "input_size": 16, "shm_region_names": shm0_region_names, @@ -600,7 +601,7 @@ def test_multi_batch_not_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -610,7 +611,7 @@ def test_multi_batch_not_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 3, (6000, None)), + args=(trial, 3, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -678,7 +679,7 @@ def test_multi_batch_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -688,7 +689,7 @@ def test_multi_batch_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 3, (6000, None)), + args=(trial, 3, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -698,7 +699,7 @@ def test_multi_batch_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "input_size": 8, "shm_region_names": shm2_region_names, @@ -709,7 +710,7 @@ def test_multi_batch_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 5, (6000, None)), + args=(trial, 5, (_max_expected_response_ms * 2, None)), kwargs={ "input_size": 8, "shm_region_names": shm3_region_names, @@ -756,7 +757,7 @@ def test_multi_batch_gt_max_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 3, (3000, None)), + args=(trial, 3, (_max_expected_response_ms, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -766,7 +767,7 @@ def test_multi_batch_gt_max_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 7, (3000, None)), + args=(trial, 7, (_max_expected_response_ms, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -813,7 +814,7 @@ def test_multi_batch_sum_gt_max_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 3, (3000, None)), + args=(trial, 3, (_max_expected_response_ms, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -870,7 +871,7 @@ def test_multi_same_output0(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (3000, None)), + args=(trial, 1, (_max_expected_response_ms, None)), kwargs={ "requested_outputs": ("OUTPUT0",), "shm_region_names": shm0_region_names, @@ -881,7 +882,7 @@ def test_multi_same_output0(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (3000, None)), + args=(trial, 1, (_max_expected_response_ms, None)), kwargs={ "requested_outputs": ("OUTPUT0",), "shm_region_names": shm1_region_names, @@ -924,7 +925,7 @@ def test_multi_same_output1(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (3000, None)), + args=(trial, 1, (_max_expected_response_ms, None)), kwargs={ "requested_outputs": ("OUTPUT1",), "shm_region_names": shm0_region_names, @@ -935,7 +936,7 @@ def test_multi_same_output1(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (3000, None)), + args=(trial, 1, (_max_expected_response_ms, None)), kwargs={ "requested_outputs": ("OUTPUT1",), "shm_region_names": shm1_region_names, @@ -979,7 +980,7 @@ def test_multi_different_outputs(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "requested_outputs": ("OUTPUT0",), "shm_region_names": shm0_region_names, @@ -990,7 +991,7 @@ def test_multi_different_outputs(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "requested_outputs": ("OUTPUT1",), "shm_region_names": shm1_region_names, @@ -1031,7 +1032,7 @@ def test_multi_different_output_order(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "requested_outputs": ("OUTPUT0", "OUTPUT1"), "shm_region_names": shm0_region_names, @@ -1041,7 +1042,7 @@ def test_multi_different_output_order(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "requested_outputs": ("OUTPUT1", "OUTPUT0"), "shm_region_names": shm1_region_names, @@ -1090,7 +1091,7 @@ def test_multi_batch_delayed_sum_gt_max_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 3, (6000, None)), + args=(trial, 3, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1187,7 +1188,7 @@ def test_multi_batch_delayed_use_max_batch(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm2_region_names, "precreated_shm_regions": precreated_shm2_regions, @@ -1245,7 +1246,7 @@ def test_multi_batch_delayed_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (3000, None)), + args=(trial, 1, (_max_expected_response_ms, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1255,7 +1256,7 @@ def test_multi_batch_delayed_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 3, (3000, None)), + args=(trial, 3, (_max_expected_response_ms, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -1265,7 +1266,7 @@ def test_multi_batch_delayed_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (3000, None)), + args=(trial, 1, (_max_expected_response_ms, None)), kwargs={ "input_size": 8, "shm_region_names": shm2_region_names, @@ -1276,7 +1277,7 @@ def test_multi_batch_delayed_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 5, (3000, None)), + args=(trial, 5, (_max_expected_response_ms, None)), kwargs={ "input_size": 8, "shm_region_names": shm3_region_names, @@ -1338,7 +1339,7 @@ def test_multi_batch_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1348,7 +1349,7 @@ def test_multi_batch_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -1358,7 +1359,7 @@ def test_multi_batch_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm2_region_names, "precreated_shm_regions": precreated_shm2_regions, @@ -1368,7 +1369,7 @@ def test_multi_batch_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm3_region_names, "precreated_shm_regions": precreated_shm3_regions, @@ -1378,7 +1379,7 @@ def test_multi_batch_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm4_region_names, "precreated_shm_regions": precreated_shm4_regions, @@ -1388,7 +1389,7 @@ def test_multi_batch_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm5_region_names, "precreated_shm_regions": precreated_shm5_regions, @@ -1439,7 +1440,7 @@ def test_multi_batch_use_best_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1449,7 +1450,7 @@ def test_multi_batch_use_best_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -1560,7 +1561,7 @@ def test_preferred_batch_only_aligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1570,7 +1571,7 @@ def test_preferred_batch_only_aligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -1580,7 +1581,7 @@ def test_preferred_batch_only_aligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm2_region_names, "precreated_shm_regions": precreated_shm2_regions, @@ -1590,7 +1591,7 @@ def test_preferred_batch_only_aligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm3_region_names, "precreated_shm_regions": precreated_shm3_regions, @@ -1645,7 +1646,7 @@ def test_preferred_batch_only_unaligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1655,7 +1656,7 @@ def test_preferred_batch_only_unaligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -1665,7 +1666,7 @@ def test_preferred_batch_only_unaligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm2_region_names, "precreated_shm_regions": precreated_shm2_regions, @@ -1675,7 +1676,7 @@ def test_preferred_batch_only_unaligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm3_region_names, "precreated_shm_regions": precreated_shm3_regions, @@ -1685,7 +1686,7 @@ def test_preferred_batch_only_unaligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm4_region_names, "precreated_shm_regions": precreated_shm4_regions, @@ -1746,7 +1747,7 @@ def test_preferred_batch_only_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1756,7 +1757,7 @@ def test_preferred_batch_only_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -1766,7 +1767,7 @@ def test_preferred_batch_only_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm2_region_names, "precreated_shm_regions": precreated_shm2_regions, @@ -1776,7 +1777,7 @@ def test_preferred_batch_only_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm3_region_names, "precreated_shm_regions": precreated_shm3_regions, @@ -1786,7 +1787,7 @@ def test_preferred_batch_only_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm4_region_names, "precreated_shm_regions": precreated_shm4_regions, @@ -1796,7 +1797,7 @@ def test_preferred_batch_only_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm5_region_names, "precreated_shm_regions": precreated_shm5_regions, @@ -1806,7 +1807,7 @@ def test_preferred_batch_only_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm6_region_names, "precreated_shm_regions": precreated_shm6_regions, @@ -1854,7 +1855,7 @@ def test_preferred_batch_only_use_no_preferred_size(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1864,7 +1865,7 @@ def test_preferred_batch_only_use_no_preferred_size(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -1874,7 +1875,7 @@ def test_preferred_batch_only_use_no_preferred_size(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (6000, None)), + args=(trial, 1, (_max_expected_response_ms * 2, None)), kwargs={ "shm_region_names": shm2_region_names, "precreated_shm_regions": precreated_shm2_regions, diff --git a/qa/L0_batcher/test.sh b/qa/L0_batcher/test.sh index 136fbe586d..0997acc0df 100755 --- a/qa/L0_batcher/test.sh +++ b/qa/L0_batcher/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -42,11 +42,63 @@ fi # can fail when the requests are distributed to multiple devices. export CUDA_VISIBLE_DEVICES=0 -CLIENT_LOG="./client.log" BATCHER_TEST=batcher_test.py VERIFY_TIMESTAMPS=verify_timestamps.py TEST_RESULT_FILE='test_results.txt' +warmup_cuda_cache() { + local backend=$1 + local batch_size=$2 + local model="${backend}_float32_float32_float32" + local n=$((16 * batch_size)) + local input0_data=$(printf '1,%.0s' $(seq 1 $n) | sed 's/,$//') + local input1_data=$(printf '2,%.0s' $(seq 1 $n) | sed 's/,$//') + + SERVER_ARGS="--model-repository=$MODELDIR/models --model-control-mode=explicit --load-model=$model" + SERVER_LOG="./warmup_cuda_cache.server.log" + + run_server + + if [ "$SERVER_PID" == "0" ]; then + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 + fi + + set +e + + curl -X POST "http://localhost:8000/v2/models/${model}/versions/1/infer" \ + -H "Content-Type: application/json" \ + -d '{ + "inputs": [ + { + "name": "INPUT0", + "datatype": "FP32", + "shape": ['"$batch_size"', 16], + "data": ['"$input0_data"'] + }, + { + "name": "INPUT1", + "datatype": "FP32", + "shape": ['"$batch_size"', 16], + "data": ['"$input1_data"'] + } + ], + "outputs": [ + { "name": "OUTPUT0" }, + { "name": "OUTPUT1" } + ] + }' + + if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + + kill_server +} + if [ -z "$TEST_VALGRIND" ]; then TEST_VALGRIND="0" fi @@ -265,17 +317,20 @@ if [[ $BACKENDS == *"libtorch"* ]]; then dynamic_batching { preferred_batch_size: [ 2, 6 ], max_queue_delay_microseconds: 10000000 }" >> config.pbtxt) fi +# [TRI-830] Give extra time for GB300 to warmup CUDA_CACHE before testing. +warmup_cuda_cache onnx 2 + # Need to launch the server for each test so that the model status is # reset (which is used to make sure the correctly batch size was used # for execution). Test everything with fixed-tensor-size models and # variable-tensor-size models. - for model_type in FIXED VARIABLE; do export BATCHER_TYPE=$model_type MODEL_PATH=models && [[ "$model_type" == "VARIABLE" ]] && MODEL_PATH=var_models for i in $NO_DELAY_TESTS ; do SERVER_ARGS="--model-repository=$MODELDIR/$MODEL_PATH ${SERVER_ARGS_EXTRA}" SERVER_LOG="./$i.$model_type.server.log" + CLIENT_LOG="./$i.$model_type.client.log" if [ "$TEST_VALGRIND" -eq 1 ]; then LEAKCHECK_LOG="./$i.$model_type.valgrind.log" From 89fb1415799fc801d4ded5855c6f04e3377dff32 Mon Sep 17 00:00:00 2001 From: Yingge He Date: Fri, 13 Mar 2026 18:48:07 -0700 Subject: [PATCH 2/3] Apply only to GB300 runner --- qa/L0_batcher/test.sh | 110 +++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 54 deletions(-) diff --git a/qa/L0_batcher/test.sh b/qa/L0_batcher/test.sh index 0997acc0df..c73cb54b27 100755 --- a/qa/L0_batcher/test.sh +++ b/qa/L0_batcher/test.sh @@ -46,59 +46,6 @@ BATCHER_TEST=batcher_test.py VERIFY_TIMESTAMPS=verify_timestamps.py TEST_RESULT_FILE='test_results.txt' -warmup_cuda_cache() { - local backend=$1 - local batch_size=$2 - local model="${backend}_float32_float32_float32" - local n=$((16 * batch_size)) - local input0_data=$(printf '1,%.0s' $(seq 1 $n) | sed 's/,$//') - local input1_data=$(printf '2,%.0s' $(seq 1 $n) | sed 's/,$//') - - SERVER_ARGS="--model-repository=$MODELDIR/models --model-control-mode=explicit --load-model=$model" - SERVER_LOG="./warmup_cuda_cache.server.log" - - run_server - - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi - - set +e - - curl -X POST "http://localhost:8000/v2/models/${model}/versions/1/infer" \ - -H "Content-Type: application/json" \ - -d '{ - "inputs": [ - { - "name": "INPUT0", - "datatype": "FP32", - "shape": ['"$batch_size"', 16], - "data": ['"$input0_data"'] - }, - { - "name": "INPUT1", - "datatype": "FP32", - "shape": ['"$batch_size"', 16], - "data": ['"$input1_data"'] - } - ], - "outputs": [ - { "name": "OUTPUT0" }, - { "name": "OUTPUT1" } - ] - }' - - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Test Failed\n***" - RET=1 - fi - set -e - - kill_server -} - if [ -z "$TEST_VALGRIND" ]; then TEST_VALGRIND="0" fi @@ -317,8 +264,63 @@ if [[ $BACKENDS == *"libtorch"* ]]; then dynamic_batching { preferred_batch_size: [ 2, 6 ], max_queue_delay_microseconds: 10000000 }" >> config.pbtxt) fi +warmup_cuda_cache() { + local backend=$1 + local batch_size=$2 + local model="${backend}_float32_float32_float32" + local n=$((16 * batch_size)) + local input0_data=$(printf '1,%.0s' $(seq 1 $n) | sed 's/,$//') + local input1_data=$(printf '2,%.0s' $(seq 1 $n) | sed 's/,$//') + + SERVER_ARGS="--model-repository=$MODELDIR/models --model-control-mode=explicit --load-model=$model" + SERVER_LOG="./warmup_cuda_cache.server.log" + + run_server + + if [ "$SERVER_PID" == "0" ]; then + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 + fi + + set +e + + curl -X POST "http://localhost:8000/v2/models/${model}/versions/1/infer" \ + -H "Content-Type: application/json" \ + -d '{ + "inputs": [ + { + "name": "INPUT0", + "datatype": "FP32", + "shape": ['"$batch_size"', 16], + "data": ['"$input0_data"'] + }, + { + "name": "INPUT1", + "datatype": "FP32", + "shape": ['"$batch_size"', 16], + "data": ['"$input1_data"'] + } + ], + "outputs": [ + { "name": "OUTPUT0" }, + { "name": "OUTPUT1" } + ] + }' + + if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + + kill_server +} + # [TRI-830] Give extra time for GB300 to warmup CUDA_CACHE before testing. -warmup_cuda_cache onnx 2 +if [[ $TEST_REPO_ARCH == "gb300_arm_103" ]]; then + warmup_cuda_cache onnx 2 +fi # Need to launch the server for each test so that the model status is # reset (which is used to make sure the correctly batch size was used From 493a34167b1064ccd8009f8a84f778b15857de6a Mon Sep 17 00:00:00 2001 From: Yingge He Date: Fri, 13 Mar 2026 22:57:53 -0700 Subject: [PATCH 3/3] Update comment --- qa/L0_batcher/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/L0_batcher/test.sh b/qa/L0_batcher/test.sh index c73cb54b27..dae31d5aa6 100755 --- a/qa/L0_batcher/test.sh +++ b/qa/L0_batcher/test.sh @@ -317,7 +317,7 @@ warmup_cuda_cache() { kill_server } -# [TRI-830] Give extra time for GB300 to warmup CUDA_CACHE before testing. +# [TRI-830] Send a simpple request to warmup CUDA_CACHE for GB300 before testing. if [[ $TEST_REPO_ARCH == "gb300_arm_103" ]]; then warmup_cuda_cache onnx 2 fi