From ebbe9fef000f61594dab62cbfa9425f130c5c0e7 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Tue, 24 Feb 2026 16:32:03 -0500 Subject: [PATCH 01/19] Avoid overflows when reading json inputs --- src/http_server.cc | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/src/http_server.cc b/src/http_server.cc index cce57ee254..cc28524cc8 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -3593,7 +3593,48 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( if (dtype == TRITONSERVER_TYPE_BYTES) { RETURN_IF_ERR(JsonBytesArrayByteSize(tensor_data, &byte_size)); } else { - byte_size = element_cnt * TRITONSERVER_DataTypeByteSize(dtype); + size_t element_size = TRITONSERVER_DataTypeByteSize(dtype); + + // Ensure that we do not have an integer overflow when calculating + // byte_size = element_cnt * element_size. + // For element_size == 1, there is no risk of integer overflow so we can + // skip the check. + switch (element_size) { + case 1: + break; + case 2: + case 4: + case 8: + if (element_cnt > (SIZE_MAX / element_size)) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + (std::string("input '") + name + + "' has too many elements of datatype " + value) + .c_str()); + } + break; + default: + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + (std::string("input '") + name + "' has unsupported datatype " + + value) + .c_str()); + } + + byte_size = element_cnt * element_size; + } + + // Ensure that the resulting array size in bytes does not exceed the maximum + // allowed input size. + if (byte_size > max_input_size_) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + ("Input '" + name + "' has a byte_size (" + + std::to_string(byte_size) + + " bytes) that exceeds the maximum allowed value of " + + std::to_string(max_input_size_) + + " bytes. Use --http-max-input-size to increase the limit.") + .c_str()); } std::vector shape_vec; From f8d2913d543a9391a37365363aa3a11b945d89c2 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Mon, 2 Mar 2026 16:13:40 -0500 Subject: [PATCH 02/19] Use a global input value when checking outcomes --- src/http_server.cc | 47 ++++++++++++++++++---------------------------- src/http_server.h | 11 ++++++++--- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/src/http_server.cc b/src/http_server.cc index cc28524cc8..e5b8936fbd 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -3361,12 +3361,12 @@ HTTPAPIServer::HandleGenerate( server_.get(), req, GetResponseCompressionType(req), generate_stream_request_schema_.get(), generate_stream_response_schema_.get(), streaming, irequest_shared, - shm_manager_)); + shm_manager_, max_input_size_)); } else { generate_request.reset(new GenerateRequestClass( server_.get(), req, GetResponseCompressionType(req), generate_request_schema_.get(), generate_response_schema_.get(), - streaming, irequest_shared, shm_manager_)); + streaming, irequest_shared, shm_manager_, max_input_size_)); } generate_request->trace_ = trace; @@ -3513,13 +3513,15 @@ HTTPAPIServer::GenerateRequestClass::ConvertGenerateRequest( std::vector members; RETURN_IF_ERR(generate_request.Members(&members)); + size_t consumed_input_size{0}; + for (const auto& m : members) { auto it = schema->children_.find(m); if (it != schema->children_.end()) { switch (it->second->kind_) { case MappingSchema::Kind::EXACT_MAPPING: { // Read meta data - RETURN_IF_ERR(ExactMappingInput(m, generate_request, input_metadata)); + RETURN_IF_ERR(ExactMappingInput(m, generate_request, input_metadata, consumed_input_size)); break; } case MappingSchema::Kind::MAPPING_SCHEMA: { @@ -3549,7 +3551,7 @@ HTTPAPIServer::GenerateRequestClass::ConvertGenerateRequest( } } else if (schema->allow_unspecified_) { // Unspecified key follows EXACT_MAPPING - RETURN_IF_ERR(ExactMappingInput(m, generate_request, input_metadata)); + RETURN_IF_ERR(ExactMappingInput(m, generate_request, input_metadata, consumed_input_size)); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, @@ -3563,7 +3565,8 @@ TRITONSERVER_Error* HTTPAPIServer::GenerateRequestClass::ExactMappingInput( const std::string& name, triton::common::TritonJson::Value& generate_request, - std::map& input_metadata) + std::map& input_metadata, + size_t& consumed_input_byte_size) { auto it = input_metadata.find(name); if (it == input_metadata.end()) { @@ -3597,28 +3600,12 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( // Ensure that we do not have an integer overflow when calculating // byte_size = element_cnt * element_size. - // For element_size == 1, there is no risk of integer overflow so we can - // skip the check. - switch (element_size) { - case 1: - break; - case 2: - case 4: - case 8: - if (element_cnt > (SIZE_MAX / element_size)) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INVALID_ARG, - (std::string("input '") + name + - "' has too many elements of datatype " + value) - .c_str()); - } - break; - default: - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INVALID_ARG, - (std::string("input '") + name + "' has unsupported datatype " + - value) - .c_str()); + if (element_cnt > (SIZE_MAX / element_size)) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + (std::string("input '") + name + + "' has too many elements of datatype " + value) + .c_str()); } byte_size = element_cnt * element_size; @@ -3626,17 +3613,19 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( // Ensure that the resulting array size in bytes does not exceed the maximum // allowed input size. - if (byte_size > max_input_size_) { + if (byte_size + consumed_input_byte_size > max_input_size_) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, ("Input '" + name + "' has a byte_size (" + std::to_string(byte_size) + " bytes) that exceeds the maximum allowed value of " + - std::to_string(max_input_size_) + + std::to_string(max_input_size_ - consumed_input_byte_size) + " bytes. Use --http-max-input-size to increase the limit.") .c_str()); } + consumed_input_byte_size += byte_size; + std::vector shape_vec; { triton::common::TritonJson::Value value; diff --git a/src/http_server.h b/src/http_server.h index c5ed0eb6de..c355780464 100644 --- a/src/http_server.h +++ b/src/http_server.h @@ -388,12 +388,13 @@ class HTTPAPIServer : public HTTPServer { const MappingSchema* request_schema, const MappingSchema* response_schema, bool streaming, const std::shared_ptr& triton_request, - const std::shared_ptr& shm_manager) + const std::shared_ptr& shm_manager, + const size_t max_input_size) : InferRequestClass( server, req, response_compression_type, triton_request, shm_manager), request_schema_(request_schema), response_schema_(response_schema), - streaming_(streaming) + streaming_(streaming), max_input_size_(max_input_size) { } virtual ~GenerateRequestClass(); @@ -428,6 +429,8 @@ class HTTPAPIServer : public HTTPServer { const MappingSchema* RequestSchema() { return request_schema_; } const MappingSchema* ResponseSchema() { return response_schema_; } + size_t MaxInputSize() { return max_input_size_; } + private: struct TritonOutput { enum class Type { RESERVED, TENSOR, PARAMETER }; @@ -443,7 +446,8 @@ class HTTPAPIServer : public HTTPServer { TRITONSERVER_Error* ExactMappingInput( const std::string& name, triton::common::TritonJson::Value& value, std::map& - input_metadata); + input_metadata, + size_t& consumed_input_byte_size); // [DLIS-5551] currently always performs basic conversion, only maps schema // of EXACT_MAPPING kind. MAPPING_SCHEMA and upcoming kinds are for @@ -461,6 +465,7 @@ class HTTPAPIServer : public HTTPServer { const MappingSchema* request_schema_{nullptr}; const MappingSchema* response_schema_{nullptr}; const bool streaming_{false}; + const size_t max_input_size_{0}; // Placeholder to completing response, this class does not own // the response. TRITONSERVER_InferenceResponse* triton_response_{nullptr}; From effbd208c4518b4292e3c466457b2c15627410a8 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Wed, 4 Mar 2026 15:03:30 -0500 Subject: [PATCH 03/19] fix issues caught by review --- ...ntime.gen_qa_model_repository.docker.v2.sh | 49 +++++++++++++++++ ...nVINO.gen_qa_model_repository.docker.v2.sh | 29 ++++++++++ ...Torch.gen_qa_model_repository.docker.v2.sh | 42 +++++++++++++++ ...sorRT.gen_qa_model_repository.docker.v2.sh | 53 +++++++++++++++++++ src/http_server.cc | 11 +++- 5 files changed, 182 insertions(+), 2 deletions(-) create mode 100755 qa/common/gen.ONNXRuntime.gen_qa_model_repository.docker.v2.sh create mode 100755 qa/common/gen.OpenVINO.gen_qa_model_repository.docker.v2.sh create mode 100755 qa/common/gen.PyTorch.gen_qa_model_repository.docker.v2.sh create mode 100755 qa/common/gen.TensorRT.gen_qa_model_repository.docker.v2.sh diff --git a/qa/common/gen.ONNXRuntime.gen_qa_model_repository.docker.v2.sh b/qa/common/gen.ONNXRuntime.gen_qa_model_repository.docker.v2.sh new file mode 100755 index 0000000000..f528aaa61f --- /dev/null +++ b/qa/common/gen.ONNXRuntime.gen_qa_model_repository.docker.v2.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Make all generated files accessible outside of container +umask 0000 +nvidia-smi --query-gpu=compute_cap,compute_mode,driver_version,name,index --format=csv || true +nvidia-smi || true +set -e +set -x +export DEBIAN_FRONTEND=noninteractive +apt-get update && apt-get install -y --no-install-recommends build-essential cmake libprotobuf-dev protobuf-compiler python3 python3-dev python3-pip +ln -s /usr/bin/python3 /usr/bin/python + +pip3 install "protobuf<=3.20.1" "numpy<=1.23.5" # TODO: Remove current line DLIS-3838 +pip3 install --upgrade onnx==1.20.1 + +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --onnx --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --onnx --onnx_opset=0 --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --onnx --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_identity_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_identity_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_reshape_models.py --onnx --onnx_opset=0 --variable --models_dir=/mnt/20260303_1203/26.01/qa_reshape_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_reshape_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --onnx --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_sequence_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_sequence_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --onnx --onnx_opset=0 --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_sequence_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_sequence_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --onnx --initial-state zero --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_sequence_initial_state_implicit_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_sequence_initial_state_implicit_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --onnx --initial-state zero --onnx_opset=0 --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_sequence_initial_state_implicit_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_sequence_initial_state_implicit_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --onnx --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_sequence_implicit_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_sequence_implicit_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --onnx --onnx_opset=0 --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_sequence_implicit_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_sequence_implicit_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_dyna_sequence_models.py --onnx --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_dyna_sequence_implicit_models.py --onnx --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_dyna_sequence_implicit_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_dyna_sequence_implicit_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_ragged_models.py --onnx --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_ragged_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_ragged_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_ort_scalar_models.py --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_scalar_models +chmod -R 777 /mnt/20260303_1203/26.01/qa_scalar_models +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --ensemble --models_dir=/mnt/20260303_1203/26.01/qa_ensemble_model_repository/qa_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --ensemble --variable --models_dir=/mnt/20260303_1203/26.01/qa_ensemble_model_repository/qa_variable_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_reshape_models.py --ensemble --models_dir=/mnt/20260303_1203/26.01/qa_ensemble_model_repository/qa_reshape_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --ensemble --models_dir=/mnt/20260303_1203/26.01/qa_ensemble_model_repository/qa_identity_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --ensemble --models_dir=/mnt/20260303_1203/26.01/qa_ensemble_model_repository/qa_sequence_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --ensemble --variable --models_dir=/mnt/20260303_1203/26.01/qa_ensemble_model_repository/qa_variable_sequence_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_ensemble_model_repository diff --git a/qa/common/gen.OpenVINO.gen_qa_model_repository.docker.v2.sh b/qa/common/gen.OpenVINO.gen_qa_model_repository.docker.v2.sh new file mode 100755 index 0000000000..2509598c7d --- /dev/null +++ b/qa/common/gen.OpenVINO.gen_qa_model_repository.docker.v2.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Make all generated files accessible outside of container +umask 0000 +nvidia-smi --query-gpu=compute_cap,compute_mode,driver_version,name,index --format=csv || true +nvidia-smi || true +set -e +set -x +export DEBIAN_FRONTEND=noninteractive +apt-get update && apt-get install -y --no-install-recommends build-essential cmake libprotobuf-dev protobuf-compiler python3 python3-dev python3-pip wget gnupg2 software-properties-common + +ln -s /usr/bin/python3 /usr/bin/python + +pip3 install "numpy<=1.23.5" setuptools + +pip3 install openvino==2024.5.0 + +# Since variable shape tensors are not allowed, identity models may fail to generate. +# TODO Add variable size tensor models after DLIS-2827 adds support for variable shape tensors. +# TODO Add sequence models after DLIS-2864 adds support for sequence/control inputs. +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --openvino --models_dir=/mnt/20260303_1203/26.01/qa_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_model_repository +# python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --openvino --models_dir=/mnt/20260303_1203/26.01/qa_identity_model_repository +# chmod -R 777 /mnt/20260303_1203/26.01/qa_identity_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_reshape_models.py --openvino --models_dir=/mnt/20260303_1203/26.01/qa_reshape_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_reshape_model_repository +# python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --openvino --models_dir=/mnt/20260303_1203/26.01/qa_sequence_model_repository +# chmod -R 777 +# python3 /mnt/20260303_1203/gen_srcdir/gen_qa_dyna_sequence_models.py --openvino --models_dir=/mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository +# chmod -R 777 /mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository diff --git a/qa/common/gen.PyTorch.gen_qa_model_repository.docker.v2.sh b/qa/common/gen.PyTorch.gen_qa_model_repository.docker.v2.sh new file mode 100755 index 0000000000..bb19326190 --- /dev/null +++ b/qa/common/gen.PyTorch.gen_qa_model_repository.docker.v2.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Make all generated files accessible outside of container +umask 0000 +nvidia-smi --query-gpu=compute_cap,compute_mode,driver_version,name,index --format=csv || true +nvidia-smi || true +pip3 install onnxscript +set -e +set -x +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --libtorch --models_dir=/mnt/20260303_1203/26.01/qa_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --libtorch --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --libtorch --models_dir=/mnt/20260303_1203/26.01/qa_identity_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_identity_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_reshape_models.py --libtorch --variable --models_dir=/mnt/20260303_1203/26.01/qa_reshape_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_reshape_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --libtorch --models_dir=/mnt/20260303_1203/26.01/qa_sequence_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_sequence_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --libtorch --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_sequence_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_sequence_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --libtorch --models_dir=/mnt/20260303_1203/26.01/qa_sequence_implicit_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_sequence_implicit_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --libtorch --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_sequence_implicit_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_sequence_implicit_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_dyna_sequence_models.py --libtorch --models_dir=/mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository +if [ -z "" ] || [ "" != "igpu" ]; then + nvidia-smi --query-gpu=compute_cap | grep -qz 12.1 && echo -e '\033[33m[WARNING]\033[0m Skipping model generation for Torch TensorRT' || python3 /mnt/20260303_1203/gen_srcdir/gen_qa_torchtrt_models.py --models_dir=/mnt/20260303_1203/26.01/torchtrt_model_store + chmod -R 777 /mnt/20260303_1203/26.01/torchtrt_model_store +fi +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_ragged_models.py --libtorch --models_dir=/mnt/20260303_1203/26.01/qa_ragged_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_ragged_model_repository +# Export torchvision image models to ONNX +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_image_models.py --resnet50 --resnet152 --vgg19 --models_dir=/mnt/20260303_1203/26.01/qa_dynamic_batch_image_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_dynamic_batch_image_model_repository + +export TORCH_EXTENSIONS_DIR=/tmp/.cache/torch_extensions/ +mkdir -p ${TORCH_EXTENSIONS_DIR} +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_custom_ops_models.py --libtorch --models_dir=/mnt/20260303_1203/26.01/qa_custom_ops/libtorch_custom_ops +mkdir -p /mnt/20260303_1203/26.01/qa_custom_ops/libtorch_custom_ops/libtorch_modulo/ +cp ${TORCH_EXTENSIONS_DIR}/custom_modulo/custom_modulo.so /mnt/20260303_1203/26.01/qa_custom_ops/libtorch_custom_ops/libtorch_modulo/. +chmod -R 777 /mnt/20260303_1203/26.01/qa_custom_ops/libtorch_custom_ops diff --git a/qa/common/gen.TensorRT.gen_qa_model_repository.docker.v2.sh b/qa/common/gen.TensorRT.gen_qa_model_repository.docker.v2.sh new file mode 100755 index 0000000000..e7e09b15ff --- /dev/null +++ b/qa/common/gen.TensorRT.gen_qa_model_repository.docker.v2.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Make all generated files accessible outside of container +umask 0000 +nvidia-smi --query-gpu=compute_cap,compute_mode,driver_version,name,index --format=csv || true +nvidia-smi || true +set -e +set -x +dpkg -l | grep TensorRT +export TRT_SUPPRESS_DEPRECATION_WARNINGS=1 +# Models using shape tensor i/o +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --tensorrt-shape-io --models_dir=/mnt/20260303_1203/26.01/qa_shapetensor_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --tensorrt-shape-io --models_dir=/mnt/20260303_1203/26.01/qa_shapetensor_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_dyna_sequence_models.py --tensorrt-shape-io --models_dir=/mnt/20260303_1203/26.01/qa_shapetensor_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_shapetensor_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --tensorrt --models_dir=/mnt/20260303_1203/26.01/qa_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --tensorrt --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --tensorrt --models_dir=/mnt/20260303_1203/26.01/qa_identity_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --tensorrt-compat --models_dir=/mnt/20260303_1203/26.01/qa_identity_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_identity_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --tensorrt-big --models_dir=/mnt/20260303_1203/26.01/qa_identity_big_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_identity_big_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_reshape_models.py --tensorrt --variable --models_dir=/mnt/20260303_1203/26.01/qa_reshape_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_reshape_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --tensorrt --models_dir=/mnt/20260303_1203/26.01/qa_sequence_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_sequence_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --tensorrt --models_dir=/mnt/20260303_1203/26.01/qa_sequence_implicit_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_sequence_implicit_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --tensorrt --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_sequence_implicit_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_sequence_implicit_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_dyna_sequence_models.py --tensorrt --models_dir=/mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --tensorrt --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_sequence_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_sequence_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_dyna_sequence_implicit_models.py --tensorrt --models_dir=/mnt/20260303_1203/26.01/qa_dyna_sequence_implicit_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_dyna_sequence_implicit_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_ragged_models.py --tensorrt --models_dir=/mnt/20260303_1203/26.01/qa_ragged_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_ragged_model_repository +python3 /mnt/20260303_1203/gen_srcdir/gen_qa_trt_format_models.py --models_dir=/mnt/20260303_1203/26.01/qa_trt_format_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_trt_format_model_repository +nvidia-smi --query-gpu=compute_cap | grep -qz 11.0 && echo -e '\033[33m[WARNING]\033[0m Skipping model generation for data dependent shape' || python3 /mnt/20260303_1203/gen_srcdir/gen_qa_trt_data_dependent_shape.py --models_dir=/mnt/20260303_1203/26.01/qa_trt_data_dependent_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_trt_data_dependent_model_repository +# Make shared library for custom Hardmax plugin. +if [ -d "/usr/src/tensorrt/samples/python/onnx_custom_plugin" ]; then + cd /usr/src/tensorrt/samples/python/onnx_custom_plugin +else + git clone -b release/$( echo $TRT_VERSION | cut -d . -f -2) --depth 1 https://github.com/NVIDIA/TensorRT.git /workspace/TensorRT + cd /workspace/TensorRT/samples/python/onnx_custom_plugin +fi +rm -rf build && mkdir build && cd build && cmake .. && make -j && cp libcustomHardmaxPlugin.so /mnt/20260303_1203/26.01/qa_trt_plugin_model_repository/. +LD_PRELOAD=/mnt/20260303_1203/26.01/qa_trt_plugin_model_repository/libcustomHardmaxPlugin.so python3 /mnt/20260303_1203/gen_srcdir/gen_qa_trt_plugin_models.py --models_dir=/mnt/20260303_1203/26.01/qa_trt_plugin_model_repository +chmod -R 777 /mnt/20260303_1203/26.01/qa_trt_plugin_model_repository diff --git a/src/http_server.cc b/src/http_server.cc index e5b8936fbd..45935b97c3 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -3597,6 +3597,13 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( RETURN_IF_ERR(JsonBytesArrayByteSize(tensor_data, &byte_size)); } else { size_t element_size = TRITONSERVER_DataTypeByteSize(dtype); + if (element_size == 0) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + (std::string("input '") + name + + "' has unsupported datatype " + value) + .c_str()); + } // Ensure that we do not have an integer overflow when calculating // byte_size = element_cnt * element_size. @@ -3606,9 +3613,9 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( (std::string("input '") + name + "' has too many elements of datatype " + value) .c_str()); + + byte_size = element_cnt * element_size; } - - byte_size = element_cnt * element_size; } // Ensure that the resulting array size in bytes does not exceed the maximum From 987a66ff3c056687f906edd2101066aae034b59d Mon Sep 17 00:00:00 2001 From: J Wyman Date: Thu, 5 Mar 2026 13:18:03 -0500 Subject: [PATCH 04/19] add tests fixup error message --- qa/L0_http/generate_endpoint_test.py | 21 ++++- qa/L0_http/http_input_size_limit_test.py | 89 ++++++++++++++++++- qa/L0_http/http_test.py | 2 +- qa/L0_http/test.sh | 16 +++- ...ntime.gen_qa_model_repository.docker.v2.sh | 49 ---------- ...nVINO.gen_qa_model_repository.docker.v2.sh | 29 ------ ...Torch.gen_qa_model_repository.docker.v2.sh | 42 --------- ...sorRT.gen_qa_model_repository.docker.v2.sh | 53 ----------- src/http_server.cc | 37 ++++---- src/http_server.h | 2 +- 10 files changed, 142 insertions(+), 198 deletions(-) delete mode 100755 qa/common/gen.ONNXRuntime.gen_qa_model_repository.docker.v2.sh delete mode 100755 qa/common/gen.OpenVINO.gen_qa_model_repository.docker.v2.sh delete mode 100755 qa/common/gen.PyTorch.gen_qa_model_repository.docker.v2.sh delete mode 100755 qa/common/gen.TensorRT.gen_qa_model_repository.docker.v2.sh diff --git a/qa/L0_http/generate_endpoint_test.py b/qa/L0_http/generate_endpoint_test.py index 3eb0b6ea5f..47068f3b8b 100755 --- a/qa/L0_http/generate_endpoint_test.py +++ b/qa/L0_http/generate_endpoint_test.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,6 +25,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import base64 import sys sys.path.append("../common") @@ -274,6 +275,24 @@ def test_invalid_input_types(self): self.generate_expect_failure(self._model_name, inputs, error_msg) self.generate_stream_expect_failure(self._model_name, inputs, error_msg) + # def test_type_size_explosion(self): + # input_data = [1] * ( + # 64 * 1024 * 1024 + # ) # 64MB input, which is large but still reasonable for HTTP request body + # input_bytes = bytes(input_data) + # input_str = base64.b64encode(input_bytes).decode("utf-8") + # inputs = {"PROMPT": input_str, "STREAM": False} + # error_msg = "Request JSON size of 67108864 + 22369655 bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." + # self.generate_expect_failure(self._model_name, inputs, error_msg) + + # inputs = { + # "INPUT0": input_str[0 : (len(input_str) // 2)], + # "INPUT1": input_str[(len(input_str) // 2) :], + # "STREAM": False, + # } + # error_msg = "Request JSON size of 67108864 + 22369669 bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." + # self.generate_expect_failure(self._model_name, inputs, error_msg) + def test_duplicate_inputs(self): dupe_prompt = "input 'PROMPT' already exists in request" dupe_stream = "input 'STREAM' already exists in request" diff --git a/qa/L0_http/http_input_size_limit_test.py b/qa/L0_http/http_input_size_limit_test.py index f0ee35649b..6cecb91d2a 100755 --- a/qa/L0_http/http_input_size_limit_test.py +++ b/qa/L0_http/http_input_size_limit_test.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -29,6 +29,7 @@ sys.path.append("../common") +import base64 import gzip import io import json @@ -41,6 +42,9 @@ # Constants for size calculations # Each FP32 value is 4 bytes, so we need to divide target byte sizes by 4 to get element counts BYTES_PER_FP32 = 4 +BYTES_PER_INT64 = ( + 8 # For the type size explosion test, we use int64 which is 8 bytes per element +) MB = 2**20 # 1 MB = 1,048,576 bytes GB = 2**30 # 1 GB = 1,073,741,824 bytes DEFAULT_LIMIT_BYTES = 64 * MB # 64MB default limit @@ -58,7 +62,88 @@ class InferSizeLimitTest(tu.TestResultCollector): def _get_infer_url(self, model_name): - return "http://localhost:8000/v2/models/{}/infer".format(model_name) + return f"http://localhost:8000/v2/models/{model_name}/infer" + + def test_type_size_explosion(self): + model = "onnx_zero_1_float32" + + # Provided data is 64MB of int8, but the model expects FP32, + # which would expand to 256MB when interpreted as FP32. + bytes_input = np.ones(DEFAULT_LIMIT_BYTES, dtype=np.int8) + input_bytes = bytes_input.tobytes() + data_str = base64.b64encode(input_bytes).decode("utf-8") + headers = { + "Content-Type": "application/json", + "Inference-Header-Content-Length": f"{len(input_bytes)}", + } + shape_size = ( + DEFAULT_LIMIT_ELEMENTS // BYTES_PER_INT64 + ) # Calculate shape size based on int64 element count to match the byte size + + payload = { + "inputs": [ + { + "name": "INPUT0", + "datatype": "INT64", + "shape": [1, shape_size], + "data": data_str, + } + ] + } + + response = requests.post( + f"http://localhost:8000/v2/models/{model}/generate", + headers=headers, + json=payload, + ) + + self.assertEqual( + 400, + response.status_code, + f"Expected error code for type/size mismatch, got: {response.status_code}", + ) + error_msg = response.content.decode() + self.assertIn( + " exceeds the maximum allowed value ", + error_msg, + "Expected error message about exceeding max input size with type mismatch", + ) + + # Test multiple inputs with one that causes size explosion. + payload = { + "inputs": [ + { + "name": "INPUT0", + "datatype": "INT64", + "shape": [1, shape_size // 2], + "data": data_str[: len(data_str) // 2], + }, + { + "name": "INPUT1", + "datatype": "INT64", + "shape": [1, shape_size // 2], + "data": data_str[len(data_str) // 2 :], + }, + ] + } + + response = requests.post( + f"http://localhost:8000/v2/models/{model}/generate", + headers=headers, + json=payload, + ) + + self.assertEqual( + 400, + response.status_code, + f"Expected error code for type/size mismatch, got: {response.status_code}", + ) + error_msg = response.content.decode() + self.assertIn( + " exceeds the maximum allowed value ", + error_msg, + "Expected error message about exceeding max input size with type mismatch", + ) def test_default_limit_raw_binary(self): """Test raw binary inputs with default limit""" diff --git a/qa/L0_http/http_test.py b/qa/L0_http/http_test.py index 775e1aba36..274e40bc3c 100755 --- a/qa/L0_http/http_test.py +++ b/qa/L0_http/http_test.py @@ -368,7 +368,7 @@ def test_loading_large_invalid_model(self): error_message, ) self.assertIn( - "exceeds the maximum allowed value", + " exceeds the maximum allowed input size. ", error_message, ) except ValueError: diff --git a/qa/L0_http/test.sh b/qa/L0_http/test.sh index f5b4548c24..664cacd4eb 100755 --- a/qa/L0_http/test.sh +++ b/qa/L0_http/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -255,7 +255,7 @@ echo -n 'username:' > pswd echo "password" | openssl passwd -stdin -apr1 >> pswd nginx -c `pwd`/$NGINX_CONF -python $BASIC_AUTH_TEST +python $BASIC_AUTH_TEST >> ${CLIENT_LOG}.python.plugin.auth 2>&1 if [ $? -ne 0 ]; then cat ${CLIENT_LOG}.python.plugin.auth RET=1 @@ -668,9 +668,9 @@ fi ## Python Unit Tests TEST_RESULT_FILE='test_results.txt' PYTHON_TEST=generate_endpoint_test.py -EXPECTED_NUM_TESTS=17 +EXPECTED_NUM_TESTS=18 set +e -python $PYTHON_TEST >$CLIENT_LOG 2>&1 +python $PYTHON_TEST > $CLIENT_LOG 2>&1 if [ $? -ne 0 ]; then cat $CLIENT_LOG RET=1 @@ -802,6 +802,14 @@ if [ $? -ne 0 ]; then echo -e "\n***\n*** Default Input Size Limit Test Failed for compressed input\n***" RET=1 fi + +# Run test to verify that large inputs fail with default limit +python http_input_size_limit_test.py InferSizeLimitTest.test_type_size_explosion >> $CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Default Input Size Limit Test Failed for type size explosion\n***" + RET=1 +fi set -e kill $SERVER_PID diff --git a/qa/common/gen.ONNXRuntime.gen_qa_model_repository.docker.v2.sh b/qa/common/gen.ONNXRuntime.gen_qa_model_repository.docker.v2.sh deleted file mode 100755 index f528aaa61f..0000000000 --- a/qa/common/gen.ONNXRuntime.gen_qa_model_repository.docker.v2.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# Make all generated files accessible outside of container -umask 0000 -nvidia-smi --query-gpu=compute_cap,compute_mode,driver_version,name,index --format=csv || true -nvidia-smi || true -set -e -set -x -export DEBIAN_FRONTEND=noninteractive -apt-get update && apt-get install -y --no-install-recommends build-essential cmake libprotobuf-dev protobuf-compiler python3 python3-dev python3-pip -ln -s /usr/bin/python3 /usr/bin/python - -pip3 install "protobuf<=3.20.1" "numpy<=1.23.5" # TODO: Remove current line DLIS-3838 -pip3 install --upgrade onnx==1.20.1 - -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --onnx --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --onnx --onnx_opset=0 --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --onnx --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_identity_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_identity_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_reshape_models.py --onnx --onnx_opset=0 --variable --models_dir=/mnt/20260303_1203/26.01/qa_reshape_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_reshape_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --onnx --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_sequence_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_sequence_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --onnx --onnx_opset=0 --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_sequence_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_sequence_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --onnx --initial-state zero --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_sequence_initial_state_implicit_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_sequence_initial_state_implicit_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --onnx --initial-state zero --onnx_opset=0 --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_sequence_initial_state_implicit_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_sequence_initial_state_implicit_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --onnx --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_sequence_implicit_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_sequence_implicit_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --onnx --onnx_opset=0 --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_sequence_implicit_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_sequence_implicit_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_dyna_sequence_models.py --onnx --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_dyna_sequence_implicit_models.py --onnx --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_dyna_sequence_implicit_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_dyna_sequence_implicit_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_ragged_models.py --onnx --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_ragged_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_ragged_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_ort_scalar_models.py --onnx_opset=0 --models_dir=/mnt/20260303_1203/26.01/qa_scalar_models -chmod -R 777 /mnt/20260303_1203/26.01/qa_scalar_models -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --ensemble --models_dir=/mnt/20260303_1203/26.01/qa_ensemble_model_repository/qa_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --ensemble --variable --models_dir=/mnt/20260303_1203/26.01/qa_ensemble_model_repository/qa_variable_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_reshape_models.py --ensemble --models_dir=/mnt/20260303_1203/26.01/qa_ensemble_model_repository/qa_reshape_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --ensemble --models_dir=/mnt/20260303_1203/26.01/qa_ensemble_model_repository/qa_identity_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --ensemble --models_dir=/mnt/20260303_1203/26.01/qa_ensemble_model_repository/qa_sequence_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --ensemble --variable --models_dir=/mnt/20260303_1203/26.01/qa_ensemble_model_repository/qa_variable_sequence_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_ensemble_model_repository diff --git a/qa/common/gen.OpenVINO.gen_qa_model_repository.docker.v2.sh b/qa/common/gen.OpenVINO.gen_qa_model_repository.docker.v2.sh deleted file mode 100755 index 2509598c7d..0000000000 --- a/qa/common/gen.OpenVINO.gen_qa_model_repository.docker.v2.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Make all generated files accessible outside of container -umask 0000 -nvidia-smi --query-gpu=compute_cap,compute_mode,driver_version,name,index --format=csv || true -nvidia-smi || true -set -e -set -x -export DEBIAN_FRONTEND=noninteractive -apt-get update && apt-get install -y --no-install-recommends build-essential cmake libprotobuf-dev protobuf-compiler python3 python3-dev python3-pip wget gnupg2 software-properties-common - -ln -s /usr/bin/python3 /usr/bin/python - -pip3 install "numpy<=1.23.5" setuptools - -pip3 install openvino==2024.5.0 - -# Since variable shape tensors are not allowed, identity models may fail to generate. -# TODO Add variable size tensor models after DLIS-2827 adds support for variable shape tensors. -# TODO Add sequence models after DLIS-2864 adds support for sequence/control inputs. -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --openvino --models_dir=/mnt/20260303_1203/26.01/qa_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_model_repository -# python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --openvino --models_dir=/mnt/20260303_1203/26.01/qa_identity_model_repository -# chmod -R 777 /mnt/20260303_1203/26.01/qa_identity_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_reshape_models.py --openvino --models_dir=/mnt/20260303_1203/26.01/qa_reshape_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_reshape_model_repository -# python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --openvino --models_dir=/mnt/20260303_1203/26.01/qa_sequence_model_repository -# chmod -R 777 -# python3 /mnt/20260303_1203/gen_srcdir/gen_qa_dyna_sequence_models.py --openvino --models_dir=/mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository -# chmod -R 777 /mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository diff --git a/qa/common/gen.PyTorch.gen_qa_model_repository.docker.v2.sh b/qa/common/gen.PyTorch.gen_qa_model_repository.docker.v2.sh deleted file mode 100755 index bb19326190..0000000000 --- a/qa/common/gen.PyTorch.gen_qa_model_repository.docker.v2.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# Make all generated files accessible outside of container -umask 0000 -nvidia-smi --query-gpu=compute_cap,compute_mode,driver_version,name,index --format=csv || true -nvidia-smi || true -pip3 install onnxscript -set -e -set -x -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --libtorch --models_dir=/mnt/20260303_1203/26.01/qa_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --libtorch --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --libtorch --models_dir=/mnt/20260303_1203/26.01/qa_identity_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_identity_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_reshape_models.py --libtorch --variable --models_dir=/mnt/20260303_1203/26.01/qa_reshape_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_reshape_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --libtorch --models_dir=/mnt/20260303_1203/26.01/qa_sequence_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_sequence_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --libtorch --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_sequence_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_sequence_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --libtorch --models_dir=/mnt/20260303_1203/26.01/qa_sequence_implicit_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_sequence_implicit_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --libtorch --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_sequence_implicit_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_sequence_implicit_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_dyna_sequence_models.py --libtorch --models_dir=/mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository -if [ -z "" ] || [ "" != "igpu" ]; then - nvidia-smi --query-gpu=compute_cap | grep -qz 12.1 && echo -e '\033[33m[WARNING]\033[0m Skipping model generation for Torch TensorRT' || python3 /mnt/20260303_1203/gen_srcdir/gen_qa_torchtrt_models.py --models_dir=/mnt/20260303_1203/26.01/torchtrt_model_store - chmod -R 777 /mnt/20260303_1203/26.01/torchtrt_model_store -fi -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_ragged_models.py --libtorch --models_dir=/mnt/20260303_1203/26.01/qa_ragged_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_ragged_model_repository -# Export torchvision image models to ONNX -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_image_models.py --resnet50 --resnet152 --vgg19 --models_dir=/mnt/20260303_1203/26.01/qa_dynamic_batch_image_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_dynamic_batch_image_model_repository - -export TORCH_EXTENSIONS_DIR=/tmp/.cache/torch_extensions/ -mkdir -p ${TORCH_EXTENSIONS_DIR} -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_custom_ops_models.py --libtorch --models_dir=/mnt/20260303_1203/26.01/qa_custom_ops/libtorch_custom_ops -mkdir -p /mnt/20260303_1203/26.01/qa_custom_ops/libtorch_custom_ops/libtorch_modulo/ -cp ${TORCH_EXTENSIONS_DIR}/custom_modulo/custom_modulo.so /mnt/20260303_1203/26.01/qa_custom_ops/libtorch_custom_ops/libtorch_modulo/. -chmod -R 777 /mnt/20260303_1203/26.01/qa_custom_ops/libtorch_custom_ops diff --git a/qa/common/gen.TensorRT.gen_qa_model_repository.docker.v2.sh b/qa/common/gen.TensorRT.gen_qa_model_repository.docker.v2.sh deleted file mode 100755 index e7e09b15ff..0000000000 --- a/qa/common/gen.TensorRT.gen_qa_model_repository.docker.v2.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Make all generated files accessible outside of container -umask 0000 -nvidia-smi --query-gpu=compute_cap,compute_mode,driver_version,name,index --format=csv || true -nvidia-smi || true -set -e -set -x -dpkg -l | grep TensorRT -export TRT_SUPPRESS_DEPRECATION_WARNINGS=1 -# Models using shape tensor i/o -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --tensorrt-shape-io --models_dir=/mnt/20260303_1203/26.01/qa_shapetensor_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --tensorrt-shape-io --models_dir=/mnt/20260303_1203/26.01/qa_shapetensor_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_dyna_sequence_models.py --tensorrt-shape-io --models_dir=/mnt/20260303_1203/26.01/qa_shapetensor_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_shapetensor_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --tensorrt --models_dir=/mnt/20260303_1203/26.01/qa_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_models.py --tensorrt --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --tensorrt --models_dir=/mnt/20260303_1203/26.01/qa_identity_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --tensorrt-compat --models_dir=/mnt/20260303_1203/26.01/qa_identity_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_identity_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_identity_models.py --tensorrt-big --models_dir=/mnt/20260303_1203/26.01/qa_identity_big_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_identity_big_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_reshape_models.py --tensorrt --variable --models_dir=/mnt/20260303_1203/26.01/qa_reshape_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_reshape_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --tensorrt --models_dir=/mnt/20260303_1203/26.01/qa_sequence_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_sequence_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --tensorrt --models_dir=/mnt/20260303_1203/26.01/qa_sequence_implicit_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_sequence_implicit_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_implicit_models.py --tensorrt --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_sequence_implicit_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_sequence_implicit_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_dyna_sequence_models.py --tensorrt --models_dir=/mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_dyna_sequence_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_sequence_models.py --tensorrt --variable --models_dir=/mnt/20260303_1203/26.01/qa_variable_sequence_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_variable_sequence_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_dyna_sequence_implicit_models.py --tensorrt --models_dir=/mnt/20260303_1203/26.01/qa_dyna_sequence_implicit_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_dyna_sequence_implicit_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_ragged_models.py --tensorrt --models_dir=/mnt/20260303_1203/26.01/qa_ragged_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_ragged_model_repository -python3 /mnt/20260303_1203/gen_srcdir/gen_qa_trt_format_models.py --models_dir=/mnt/20260303_1203/26.01/qa_trt_format_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_trt_format_model_repository -nvidia-smi --query-gpu=compute_cap | grep -qz 11.0 && echo -e '\033[33m[WARNING]\033[0m Skipping model generation for data dependent shape' || python3 /mnt/20260303_1203/gen_srcdir/gen_qa_trt_data_dependent_shape.py --models_dir=/mnt/20260303_1203/26.01/qa_trt_data_dependent_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_trt_data_dependent_model_repository -# Make shared library for custom Hardmax plugin. -if [ -d "/usr/src/tensorrt/samples/python/onnx_custom_plugin" ]; then - cd /usr/src/tensorrt/samples/python/onnx_custom_plugin -else - git clone -b release/$( echo $TRT_VERSION | cut -d . -f -2) --depth 1 https://github.com/NVIDIA/TensorRT.git /workspace/TensorRT - cd /workspace/TensorRT/samples/python/onnx_custom_plugin -fi -rm -rf build && mkdir build && cd build && cmake .. && make -j && cp libcustomHardmaxPlugin.so /mnt/20260303_1203/26.01/qa_trt_plugin_model_repository/. -LD_PRELOAD=/mnt/20260303_1203/26.01/qa_trt_plugin_model_repository/libcustomHardmaxPlugin.so python3 /mnt/20260303_1203/gen_srcdir/gen_qa_trt_plugin_models.py --models_dir=/mnt/20260303_1203/26.01/qa_trt_plugin_model_repository -chmod -R 777 /mnt/20260303_1203/26.01/qa_trt_plugin_model_repository diff --git a/src/http_server.cc b/src/http_server.cc index 45935b97c3..f0250bb4ef 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -3084,12 +3084,13 @@ HTTPAPIServer::EVBufferToJson( const size_t length, int n) { if (length > max_input_size_) { + auto overrun = length - max_input_size_; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, - ("Request JSON size of " + std::to_string(length) + - " bytes exceeds the maximum allowed value of " + - std::to_string(max_input_size_) + - " bytes. Use --http-max-input-size to increase the limit.") + ("Request JSON size of " + std::to_string(length) + " + " + + std::to_string(overrun) + + " bytes exceeds the maximum allowed input size. " + "Use --http-max-input-size to increase the limit.") .c_str()); } @@ -3521,7 +3522,8 @@ HTTPAPIServer::GenerateRequestClass::ConvertGenerateRequest( switch (it->second->kind_) { case MappingSchema::Kind::EXACT_MAPPING: { // Read meta data - RETURN_IF_ERR(ExactMappingInput(m, generate_request, input_metadata, consumed_input_size)); + RETURN_IF_ERR(ExactMappingInput( + m, generate_request, input_metadata, consumed_input_size)); break; } case MappingSchema::Kind::MAPPING_SCHEMA: { @@ -3551,7 +3553,8 @@ HTTPAPIServer::GenerateRequestClass::ConvertGenerateRequest( } } else if (schema->allow_unspecified_) { // Unspecified key follows EXACT_MAPPING - RETURN_IF_ERR(ExactMappingInput(m, generate_request, input_metadata, consumed_input_size)); + RETURN_IF_ERR(ExactMappingInput( + m, generate_request, input_metadata, consumed_input_size)); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, @@ -3600,8 +3603,8 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( if (element_size == 0) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, - (std::string("input '") + name + - "' has unsupported datatype " + value) + (std::string("input '") + name + "' has unsupported datatype " + + value) .c_str()); } @@ -3611,23 +3614,25 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, (std::string("input '") + name + - "' has too many elements of datatype " + value) + "' has too many elements of datatype " + value) .c_str()); - + byte_size = element_cnt * element_size; } } // Ensure that the resulting array size in bytes does not exceed the maximum // allowed input size. - if (byte_size + consumed_input_byte_size > max_input_size_) { + if (byte_size + consumed_input_byte_size > max_input_size_ || + byte_size + consumed_input_byte_size < consumed_input_byte_size) { + auto overrun = byte_size + consumed_input_byte_size - max_input_size_; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, - ("Input '" + name + "' has a byte_size (" + - std::to_string(byte_size) + - " bytes) that exceeds the maximum allowed value of " + - std::to_string(max_input_size_ - consumed_input_byte_size) + - " bytes. Use --http-max-input-size to increase the limit.") + ("Input '" + name + "' has size of " + + std::to_string(max_input_size_ - consumed_input_byte_size) + " + " + + std::to_string(overrun) + + " bytes exceeds the maximum allowed input size. " + "Use --http-max-input-size to increase the limit.") .c_str()); } diff --git a/src/http_server.h b/src/http_server.h index c355780464..a20e1ab44a 100644 --- a/src/http_server.h +++ b/src/http_server.h @@ -1,4 +1,4 @@ -// Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions From e5c03ba2a3700eb86f0f6be3628da05937250d70 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Tue, 10 Mar 2026 12:23:56 -0400 Subject: [PATCH 05/19] Potential fix for code scanning alert no. 1030: Unused import Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- qa/L0_http/generate_endpoint_test.py | 34 ++++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/qa/L0_http/generate_endpoint_test.py b/qa/L0_http/generate_endpoint_test.py index 47068f3b8b..5320277d5f 100755 --- a/qa/L0_http/generate_endpoint_test.py +++ b/qa/L0_http/generate_endpoint_test.py @@ -275,23 +275,23 @@ def test_invalid_input_types(self): self.generate_expect_failure(self._model_name, inputs, error_msg) self.generate_stream_expect_failure(self._model_name, inputs, error_msg) - # def test_type_size_explosion(self): - # input_data = [1] * ( - # 64 * 1024 * 1024 - # ) # 64MB input, which is large but still reasonable for HTTP request body - # input_bytes = bytes(input_data) - # input_str = base64.b64encode(input_bytes).decode("utf-8") - # inputs = {"PROMPT": input_str, "STREAM": False} - # error_msg = "Request JSON size of 67108864 + 22369655 bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." - # self.generate_expect_failure(self._model_name, inputs, error_msg) - - # inputs = { - # "INPUT0": input_str[0 : (len(input_str) // 2)], - # "INPUT1": input_str[(len(input_str) // 2) :], - # "STREAM": False, - # } - # error_msg = "Request JSON size of 67108864 + 22369669 bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." - # self.generate_expect_failure(self._model_name, inputs, error_msg) + def test_type_size_explosion(self): + input_data = [1] * ( + 64 * 1024 * 1024 + ) # 64MB input, which is large but still reasonable for HTTP request body + input_bytes = bytes(input_data) + input_str = base64.b64encode(input_bytes).decode("utf-8") + inputs = {"PROMPT": input_str, "STREAM": False} + error_msg = "Request JSON size of 67108864 + 22369655 bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." + self.generate_expect_failure(self._model_name, inputs, error_msg) + + inputs = { + "INPUT0": input_str[0 : (len(input_str) // 2)], + "INPUT1": input_str[(len(input_str) // 2) :], + "STREAM": False, + } + error_msg = "Request JSON size of 67108864 + 22369669 bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." + self.generate_expect_failure(self._model_name, inputs, error_msg) def test_duplicate_inputs(self): dupe_prompt = "input 'PROMPT' already exists in request" From e0d8c442f13af92cdefc75a0afae6677f258e3c0 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Wed, 11 Mar 2026 20:52:31 -0400 Subject: [PATCH 06/19] adjust message matches --- qa/L0_http/http_input_size_limit_test.py | 48 +++++++++++++++++++----- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/qa/L0_http/http_input_size_limit_test.py b/qa/L0_http/http_input_size_limit_test.py index 6cecb91d2a..e2adb6f7fa 100755 --- a/qa/L0_http/http_input_size_limit_test.py +++ b/qa/L0_http/http_input_size_limit_test.py @@ -104,9 +104,16 @@ def test_type_size_explosion(self): ) error_msg = response.content.decode() self.assertIn( - " exceeds the maximum allowed value ", + "Request JSON size of ", + error_msg, + ) + self.assertIn( + " bytes exceeds the maximum allowed input size", + error_msg, + ) + self.assertIn( + "Use --http-max-input-size to increase the limit", error_msg, - "Expected error message about exceeding max input size with type mismatch", ) # Test multiple inputs with one that causes size explosion. @@ -140,9 +147,16 @@ def test_type_size_explosion(self): ) error_msg = response.content.decode() self.assertIn( - " exceeds the maximum allowed value ", + "Request JSON size of ", + error_msg, + ) + self.assertIn( + " bytes exceeds the maximum allowed input size", + error_msg, + ) + self.assertIn( + "Use --http-max-input-size to increase the limit", error_msg, - "Expected error message about exceeding max input size with type mismatch", ) def test_default_limit_raw_binary(self): @@ -250,9 +264,16 @@ def test_default_limit_json(self): # Verify error message contains size limit info error_msg = response.content.decode() self.assertIn( - "exceeds the maximum allowed value", + "Request JSON size of ", + error_msg, + ) + self.assertIn( + " bytes exceeds the maximum allowed input size", + error_msg, + ) + self.assertIn( + "Use --http-max-input-size to increase the limit", error_msg, - "Expected error message about exceeding max input size", ) # Test case 2: Input just under the 64MB limit (should succeed) @@ -405,9 +426,16 @@ def test_large_input_json(self): # Verify error message contains size limit info error_msg = response.content.decode() self.assertIn( - "exceeds the maximum allowed value", + "Request JSON size of ", + error_msg, + ) + self.assertIn( + " bytes exceeds the maximum allowed input size", + error_msg, + ) + self.assertIn( + "Use --http-max-input-size to increase the limit", error_msg, - "Expected error message about exceeding max input size", ) # Test case 2: Input just under the 128MB configured limit (should succeed) @@ -490,11 +518,11 @@ def test_large_string_in_json(self): # Verify error message error_msg = response.content.decode() self.assertIn( - "Request JSON size", + "Request JSON size of ", error_msg, ) self.assertIn( - "exceeds the maximum allowed value", + " bytes exceeds the maximum allowed input size", error_msg, ) self.assertIn( From effb1a18929153fb19e714f82d58894577321744 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Wed, 11 Mar 2026 21:24:56 -0400 Subject: [PATCH 07/19] remove unused import --- qa/L0_http/generate_endpoint_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qa/L0_http/generate_endpoint_test.py b/qa/L0_http/generate_endpoint_test.py index 5320277d5f..3391ab1089 100755 --- a/qa/L0_http/generate_endpoint_test.py +++ b/qa/L0_http/generate_endpoint_test.py @@ -25,7 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import base64 import sys sys.path.append("../common") From a7994f5de83b4e25570412fb8126391bd0896b24 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Wed, 11 Mar 2026 21:28:15 -0400 Subject: [PATCH 08/19] change casing of error message to match existing messages. --- src/http_server.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http_server.cc b/src/http_server.cc index f0250bb4ef..48284e53c8 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -3628,7 +3628,7 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( auto overrun = byte_size + consumed_input_byte_size - max_input_size_; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, - ("Input '" + name + "' has size of " + + (std::string("input '") + name + "' has size of " + std::to_string(max_input_size_ - consumed_input_byte_size) + " + " + std::to_string(overrun) + " bytes exceeds the maximum allowed input size. " From dcac291826bcce16088f19f0281991c3844bbb0d Mon Sep 17 00:00:00 2001 From: J Wyman Date: Thu, 19 Mar 2026 16:01:35 -0400 Subject: [PATCH 09/19] catch & handle null pointer conditions --- src/http_server.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/http_server.cc b/src/http_server.cc index 48284e53c8..0f1e9a86b6 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -478,6 +478,11 @@ ReadDataFromJsonHelper( triton::common::TritonJson::Value& tensor_data, int* counter, int64_t expected_cnt, int current_depth = 0) { + if (!base) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INTERNAL, + "output buffer `base` pointer cannot be null"); + } // FIXME should move 'switch' statement outside the recursive function and // pass in a read data callback once data type is confirmed. // Currently 'switch' is performed on each element even through all elements @@ -501,6 +506,10 @@ ReadDataFromJsonHelper( base, dtype, el, counter, expected_cnt, current_depth + 1)); } } else { + if (!counter) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INTERNAL, "`counter` pointer cannot be null"); + } // Check if writing to 'serialized' is overrunning the expected byte_size if (*counter < 0 || static_cast(*counter) >= expected_cnt) { return TRITONSERVER_ErrorNew( From edb8aef67c5c56f95c2cd26201bd8ab2290c0712 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Fri, 20 Mar 2026 15:52:26 -0400 Subject: [PATCH 10/19] avoid adding zero sized buffer to TRITONSERVER_InferenceRequestAddInput --- qa/L0_http/generate_endpoint_test.py | 1 + qa/L0_http/http_test.py | 2 +- qa/L0_http/test.sh | 2 +- src/http_server.cc | 57 +++++++++++++++++++--------- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/qa/L0_http/generate_endpoint_test.py b/qa/L0_http/generate_endpoint_test.py index 3391ab1089..d1e684aad7 100755 --- a/qa/L0_http/generate_endpoint_test.py +++ b/qa/L0_http/generate_endpoint_test.py @@ -34,6 +34,7 @@ import time import unittest +import base64 import requests import sseclient import test_util as tu diff --git a/qa/L0_http/http_test.py b/qa/L0_http/http_test.py index 274e40bc3c..48ef1728f4 100755 --- a/qa/L0_http/http_test.py +++ b/qa/L0_http/http_test.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions diff --git a/qa/L0_http/test.sh b/qa/L0_http/test.sh index 664cacd4eb..f20a41e160 100755 --- a/qa/L0_http/test.sh +++ b/qa/L0_http/test.sh @@ -655,7 +655,7 @@ wait $SERVER_PID # https://github.com/mpetazzoni/sseclient pip install sseclient-py -SERVER_ARGS="--model-repository=`pwd`/../python_models/generate_models" +SERVER_ARGS="--model-repository=`pwd`/../python_models/generate_models --log-verbose=1" SERVER_LOG="./inference_server_generate_endpoint_test.log" CLIENT_LOG="./generate_endpoint_test.log" run_server diff --git a/src/http_server.cc b/src/http_server.cc index 0f1e9a86b6..a8a587d24a 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -481,7 +481,7 @@ ReadDataFromJsonHelper( if (!base) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - "output buffer `base` pointer cannot be null"); + "Failed to parse 'data' field: output buffer unavailable"); } // FIXME should move 'switch' statement outside the recursive function and // pass in a read data callback once data type is confirmed. @@ -508,13 +508,14 @@ ReadDataFromJsonHelper( } else { if (!counter) { return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INTERNAL, "`counter` pointer cannot be null"); + TRITONSERVER_ERROR_INTERNAL, + "Failed to parse 'data' field: invalid counter provided"); } // Check if writing to 'serialized' is overrunning the expected byte_size if (*counter < 0 || static_cast(*counter) >= expected_cnt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - "Shape does not match true shape of 'data' field"); + "Failed to parse 'data' field: shape does not match true shape"); } switch (dtype) { case TRITONSERVER_TYPE_BOOL: { @@ -618,7 +619,8 @@ ReadDataFromJsonHelper( if (len > INT64_MAX) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - "Tensor size is too large to be processed"); + "Failed to parse 'data' field: tensor size is too large to be " + "processed"); } // Quick sanity check to ensure we don't write beyond `expected_cnt`. int64_t actual_cnt = static_cast(*counter) + @@ -627,7 +629,7 @@ ReadDataFromJsonHelper( if (actual_cnt < 0 || actual_cnt > expected_cnt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - "Shape does not match true shape of 'data' field"); + "Failed to parse 'data' field: shape does not match true shape"); } memcpy( base + *counter, reinterpret_cast(&len), sizeof(uint32_t)); @@ -677,6 +679,11 @@ ReadDataFromJson( .c_str()); default: + if (!base) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INTERNAL, + "Failed to parse 'data' field: output buffer unavailable"); + } RETURN_MSG_IF_ERR( ReadDataFromJsonHelper( base, dtype, tensor_data, &counter, expected_cnt), @@ -688,8 +695,7 @@ ReadDataFromJson( if (counter != expected_cnt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - "Unable to parse 'data': Shape does not match true shape of 'data' " - "field"); + "Failed to parse 'data' field: shape does not match true shape"); } return nullptr; @@ -704,7 +710,8 @@ WriteDataToJsonCheck( return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( - "output tensor shape does not match size of output for '" + + "Failed to write 'data' field: output tensor shape does not match " + "size of output for '" + output_name + "'") .c_str()); } @@ -725,7 +732,8 @@ WriteDataToJson( return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( - "output tensor shape does not match size of output for '" + + "Failed to write 'data' field: output tensor shape does not " + "match size of output for '" + output_name + "'") .c_str()); } @@ -2694,12 +2702,12 @@ HTTPAPIServer::ParseJsonTritonIO( infer_req->serialized_data_.emplace_back(); std::vector& serialized = infer_req->serialized_data_.back(); serialized.resize(byte_size); - + char* serialized_base = &serialized[0]; RETURN_IF_ERR(ReadDataFromJson( - input_name, tensor_data, &serialized[0], dtype, + input_name, tensor_data, serialized_base, dtype, dtype == TRITONSERVER_TYPE_BYTES ? byte_size : element_cnt)); RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData( - irequest, input_name, &serialized[0], serialized.size(), + irequest, input_name, serialized_base, serialized.size(), TRITONSERVER_MEMORY_CPU, 0 /* memory_type_id */)); } } @@ -3602,9 +3610,9 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( .c_str()); } + size_t byte_size{0}; size_t element_cnt = tensor_data.IsArray() ? tensor_data.ArraySize() : 1; - size_t byte_size = 0; if (dtype == TRITONSERVER_TYPE_BYTES) { RETURN_IF_ERR(JsonBytesArrayByteSize(tensor_data, &byte_size)); } else { @@ -3625,9 +3633,17 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( (std::string("input '") + name + "' has too many elements of datatype " + value) .c_str()); - - byte_size = element_cnt * element_size; } + + byte_size = element_cnt * element_size; + } + + // For zero-size input, we can skip the rest of the validation and just add + // it as an empty input. + if (byte_size == 0) { + RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddInput( + triton_request_.get(), name.c_str(), dtype, nullptr, 0)); + return nullptr; } // Ensure that the resulting array size in bytes does not exceed the maximum @@ -3658,11 +3674,13 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( name) .c_str()); } + for (size_t i = 0; i < value.ArraySize(); ++i) { int64_t d = 0; RETURN_IF_ERR(value.IndexAsInt(i, &d)); shape_vec.push_back(d); } + // Because generate request don't carry too much shape information, using // a two-pass process to pad the request value to match input shape. // 1. iterate shape for fixed dimension to distribute 'element_cnt'. @@ -3680,12 +3698,14 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( element_cnt /= *rit; } } + for (auto rit = shape_vec.rbegin(); rit != shape_vec.rend(); ++rit) { if (*rit == -1) { *rit = element_cnt; element_cnt = 1; } } + if (element_cnt != 1) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, @@ -3700,15 +3720,18 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( serialized_data_.emplace_back(); std::vector& serialized = serialized_data_.back(); serialized.resize(byte_size); + char* serialized_base = &serialized[0]; + RETURN_IF_ERR(ReadDataFromJson( - name.c_str(), tensor_data, &serialized[0], dtype, + name.c_str(), tensor_data, serialized_base, dtype, dtype == TRITONSERVER_TYPE_BYTES ? byte_size : element_cnt)); RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddInput( triton_request_.get(), name.c_str(), dtype, &shape_vec[0], shape_vec.size())); + RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData( - triton_request_.get(), name.c_str(), &serialized[0], serialized.size(), + triton_request_.get(), name.c_str(), serialized_base, serialized.size(), TRITONSERVER_MEMORY_CPU, 0 /* memory_type_id */)); } return nullptr; // success From 98eabf98833353fd348588048c46efb550ea6d04 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Mon, 23 Mar 2026 12:03:54 -0400 Subject: [PATCH 11/19] fix test --- qa/L0_http/generate_endpoint_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qa/L0_http/generate_endpoint_test.py b/qa/L0_http/generate_endpoint_test.py index d1e684aad7..a43746bb24 100755 --- a/qa/L0_http/generate_endpoint_test.py +++ b/qa/L0_http/generate_endpoint_test.py @@ -29,12 +29,12 @@ sys.path.append("../common") +import base64 import json import threading import time import unittest -import base64 import requests import sseclient import test_util as tu @@ -282,7 +282,7 @@ def test_type_size_explosion(self): input_bytes = bytes(input_data) input_str = base64.b64encode(input_bytes).decode("utf-8") inputs = {"PROMPT": input_str, "STREAM": False} - error_msg = "Request JSON size of 67108864 + 22369655 bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." + error_msg = "Request JSON size of 89478519 + 22369655 bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." self.generate_expect_failure(self._model_name, inputs, error_msg) inputs = { @@ -290,7 +290,7 @@ def test_type_size_explosion(self): "INPUT1": input_str[(len(input_str) // 2) :], "STREAM": False, } - error_msg = "Request JSON size of 67108864 + 22369669 bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." + error_msg = "Request JSON size of 89478533 + 22369669 bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." self.generate_expect_failure(self._model_name, inputs, error_msg) def test_duplicate_inputs(self): From 3d21dc1f98c858b615975ba0a3dabf270c57b1b6 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Mon, 23 Mar 2026 14:57:31 -0400 Subject: [PATCH 12/19] print error when error --- qa/L0_http/generate_endpoint_test.py | 4 +-- qa/L0_http/test.sh | 46 ++++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/qa/L0_http/generate_endpoint_test.py b/qa/L0_http/generate_endpoint_test.py index a43746bb24..9be9841c97 100755 --- a/qa/L0_http/generate_endpoint_test.py +++ b/qa/L0_http/generate_endpoint_test.py @@ -282,7 +282,7 @@ def test_type_size_explosion(self): input_bytes = bytes(input_data) input_str = base64.b64encode(input_bytes).decode("utf-8") inputs = {"PROMPT": input_str, "STREAM": False} - error_msg = "Request JSON size of 89478519 + 22369655 bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." + error_msg = " bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." self.generate_expect_failure(self._model_name, inputs, error_msg) inputs = { @@ -290,7 +290,7 @@ def test_type_size_explosion(self): "INPUT1": input_str[(len(input_str) // 2) :], "STREAM": False, } - error_msg = "Request JSON size of 89478533 + 22369669 bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." + error_msg = " bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." self.generate_expect_failure(self._model_name, inputs, error_msg) def test_duplicate_inputs(self): diff --git a/qa/L0_http/test.sh b/qa/L0_http/test.sh index f20a41e160..4845846298 100755 --- a/qa/L0_http/test.sh +++ b/qa/L0_http/test.sh @@ -380,9 +380,15 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "1" ]; then + echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then + echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi @@ -397,9 +403,15 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "0" ]; then + echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] found in output when not expected" + cat ./curl.out + echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then + echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi @@ -414,9 +426,15 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "0" ]; then + echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] found in output when not expected" + cat ./curl.out + echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then + echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi @@ -431,9 +449,15 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "0" ]; then + echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] found in output when not expected" + cat ./curl.out + echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "0" ]; then + echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] found in output when not expected" + cat ./curl.out + echo "" RET=1 fi @@ -449,9 +473,15 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "1" ]; then + echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then + echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi @@ -465,7 +495,10 @@ if [ "$code" == "200" ]; then echo -e "\n***\n*** Test Failed\n***" RET=1 fi -if [ `grep -c "\{\"error\":\"Unable to parse 'data': Shape does not match true shape of 'data' field\"\}" ./curl.out` != "1" ]; then +if [ `grep -c "\{\"error\":\"Failed to parse 'data' field: shape does not match true shape\"\}" ./curl.out` != "1" ]; then + echo -e "\{\"error\":\"Failed to parse 'data' field: shape does not match true shape\"\} not found in output when expected" + cat ./curl.out + echo "" RET=1 fi @@ -478,7 +511,10 @@ if [ "$code" == "200" ]; then echo -e "\n***\n*** Test Failed\n***" RET=1 fi -if [ `grep -c "\{\"error\":\"Unable to parse 'data': Shape does not match true shape of 'data' field\"\}" ./curl.out` != "1" ]; then +if [ `grep -c "\{\"error\":\"Unable to parse 'data': shape does not match true shape\"\}" ./curl.out` != "1" ]; then + echo -e "\{\"error\":\"Unable to parse 'data': shape does not match true shape\"\} not found in output when expected" + cat ./curl.out + echo "" RET=1 fi @@ -493,9 +529,15 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "1" ]; then + echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then + echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi From defd24a0f597ba621dad666fc80d635c71056179 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Mon, 23 Mar 2026 16:20:57 -0400 Subject: [PATCH 13/19] revert some error message changes --- src/http_server.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/http_server.cc b/src/http_server.cc index a8a587d24a..ef3931561f 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -509,13 +509,13 @@ ReadDataFromJsonHelper( if (!counter) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - "Failed to parse 'data' field: invalid counter provided"); + "Invalid counter provided"); } // Check if writing to 'serialized' is overrunning the expected byte_size if (*counter < 0 || static_cast(*counter) >= expected_cnt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - "Failed to parse 'data' field: shape does not match true shape"); + "Shape does not match true shape of 'data' field"); } switch (dtype) { case TRITONSERVER_TYPE_BOOL: { @@ -619,8 +619,7 @@ ReadDataFromJsonHelper( if (len > INT64_MAX) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - "Failed to parse 'data' field: tensor size is too large to be " - "processed"); + "Tensor size is too large to be processed"); } // Quick sanity check to ensure we don't write beyond `expected_cnt`. int64_t actual_cnt = static_cast(*counter) + @@ -629,7 +628,7 @@ ReadDataFromJsonHelper( if (actual_cnt < 0 || actual_cnt > expected_cnt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - "Failed to parse 'data' field: shape does not match true shape"); + "Shape does not match true shape of 'data' field"); } memcpy( base + *counter, reinterpret_cast(&len), sizeof(uint32_t)); From 5efc58b43f44f65835f94751e37ee452081d3a5a Mon Sep 17 00:00:00 2001 From: J Wyman Date: Mon, 23 Mar 2026 17:24:26 -0400 Subject: [PATCH 14/19] fix test match --- qa/L0_http/test.sh | 4 ++-- src/http_server.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/qa/L0_http/test.sh b/qa/L0_http/test.sh index 4845846298..5e7ec20676 100755 --- a/qa/L0_http/test.sh +++ b/qa/L0_http/test.sh @@ -511,8 +511,8 @@ if [ "$code" == "200" ]; then echo -e "\n***\n*** Test Failed\n***" RET=1 fi -if [ `grep -c "\{\"error\":\"Unable to parse 'data': shape does not match true shape\"\}" ./curl.out` != "1" ]; then - echo -e "\{\"error\":\"Unable to parse 'data': shape does not match true shape\"\} not found in output when expected" +if [ `grep -c "\{\"error\":\"Unable to parse 'data': Shape does not match true shape of 'data' field\"\}" ./curl.out` != "1" ]; then + echo -e "\{\"error\":\"Unable to parse 'data': Shape does not match true shape of 'data' field\"\} not found in output when expected" cat ./curl.out echo "" RET=1 diff --git a/src/http_server.cc b/src/http_server.cc index ef3931561f..8595409d9f 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -628,7 +628,7 @@ ReadDataFromJsonHelper( if (actual_cnt < 0 || actual_cnt > expected_cnt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - "Shape does not match true shape of 'data' field"); + "shape does not match true shape of 'data' field"); } memcpy( base + *counter, reinterpret_cast(&len), sizeof(uint32_t)); From 8428658bdd4399b34aed6cb342617167e157fc34 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Mon, 23 Mar 2026 17:42:13 -0400 Subject: [PATCH 15/19] make pre-commit happy --- src/http_server.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/http_server.cc b/src/http_server.cc index 8595409d9f..edbb6d5ef5 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -508,8 +508,7 @@ ReadDataFromJsonHelper( } else { if (!counter) { return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INTERNAL, - "Invalid counter provided"); + TRITONSERVER_ERROR_INTERNAL, "Invalid counter provided"); } // Check if writing to 'serialized' is overrunning the expected byte_size if (*counter < 0 || static_cast(*counter) >= expected_cnt) { From 4b99ac09a3fad327c4f7ef8544b38d4a7111f16f Mon Sep 17 00:00:00 2001 From: J Wyman Date: Wed, 25 Mar 2026 12:45:01 -0400 Subject: [PATCH 16/19] react to @mudit-eng comments --- src/http_server.cc | 13 +++++-------- src/http_server.h | 2 -- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/http_server.cc b/src/http_server.cc index edbb6d5ef5..4afb93bf69 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -2700,12 +2700,12 @@ HTTPAPIServer::ParseJsonTritonIO( infer_req->serialized_data_.emplace_back(); std::vector& serialized = infer_req->serialized_data_.back(); serialized.resize(byte_size); - char* serialized_base = &serialized[0]; + RETURN_IF_ERR(ReadDataFromJson( - input_name, tensor_data, serialized_base, dtype, + input_name, tensor_data, &serialized[0], dtype, dtype == TRITONSERVER_TYPE_BYTES ? byte_size : element_cnt)); RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData( - irequest, input_name, serialized_base, serialized.size(), + irequest, input_name, &serialized[0], serialized.size(), TRITONSERVER_MEMORY_CPU, 0 /* memory_type_id */)); } } @@ -3718,18 +3718,15 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( serialized_data_.emplace_back(); std::vector& serialized = serialized_data_.back(); serialized.resize(byte_size); - char* serialized_base = &serialized[0]; RETURN_IF_ERR(ReadDataFromJson( - name.c_str(), tensor_data, serialized_base, dtype, + name.c_str(), tensor_data, &serialized[0], dtype, dtype == TRITONSERVER_TYPE_BYTES ? byte_size : element_cnt)); - RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddInput( triton_request_.get(), name.c_str(), dtype, &shape_vec[0], shape_vec.size())); - RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData( - triton_request_.get(), name.c_str(), serialized_base, serialized.size(), + triton_request_.get(), name.c_str(), &serialized[0], serialized.size(), TRITONSERVER_MEMORY_CPU, 0 /* memory_type_id */)); } return nullptr; // success diff --git a/src/http_server.h b/src/http_server.h index a20e1ab44a..2187ec1700 100644 --- a/src/http_server.h +++ b/src/http_server.h @@ -429,8 +429,6 @@ class HTTPAPIServer : public HTTPServer { const MappingSchema* RequestSchema() { return request_schema_; } const MappingSchema* ResponseSchema() { return response_schema_; } - size_t MaxInputSize() { return max_input_size_; } - private: struct TritonOutput { enum class Type { RESERVED, TENSOR, PARAMETER }; From 148a4ddf0ac5954fdb8e9b3a65f7c3a578c18a66 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Fri, 3 Apr 2026 16:19:16 -0400 Subject: [PATCH 17/19] fixup error messages update tests to match message changes --- qa/L0_http/generate_endpoint_test.py | 4 ++-- qa/L0_http/http_input_size_limit_test.py | 22 +++++++++++----------- src/http_server.cc | 21 ++++++++++----------- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/qa/L0_http/generate_endpoint_test.py b/qa/L0_http/generate_endpoint_test.py index 9be9841c97..5089a03997 100755 --- a/qa/L0_http/generate_endpoint_test.py +++ b/qa/L0_http/generate_endpoint_test.py @@ -282,7 +282,7 @@ def test_type_size_explosion(self): input_bytes = bytes(input_data) input_str = base64.b64encode(input_bytes).decode("utf-8") inputs = {"PROMPT": input_str, "STREAM": False} - error_msg = " bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." + error_msg = " bytes exceeds the maximum allowed input size of " self.generate_expect_failure(self._model_name, inputs, error_msg) inputs = { @@ -290,7 +290,7 @@ def test_type_size_explosion(self): "INPUT1": input_str[(len(input_str) // 2) :], "STREAM": False, } - error_msg = " bytes exceeds the maximum allowed input size. Use --http-max-input-size to increase the limit." + error_msg = " bytes exceeds the maximum allowed input size of " self.generate_expect_failure(self._model_name, inputs, error_msg) def test_duplicate_inputs(self): diff --git a/qa/L0_http/http_input_size_limit_test.py b/qa/L0_http/http_input_size_limit_test.py index e2adb6f7fa..f78e3cd804 100755 --- a/qa/L0_http/http_input_size_limit_test.py +++ b/qa/L0_http/http_input_size_limit_test.py @@ -108,11 +108,11 @@ def test_type_size_explosion(self): error_msg, ) self.assertIn( - " bytes exceeds the maximum allowed input size", + " bytes exceeds the maximum allowed input size of ", error_msg, ) self.assertIn( - "Use --http-max-input-size to increase the limit", + "Use --http-max-input-size to increase the limit.", error_msg, ) @@ -147,15 +147,15 @@ def test_type_size_explosion(self): ) error_msg = response.content.decode() self.assertIn( - "Request JSON size of ", + "request JSON size of ", error_msg, ) self.assertIn( - " bytes exceeds the maximum allowed input size", + " bytes exceeds the maximum allowed input size of ", error_msg, ) self.assertIn( - "Use --http-max-input-size to increase the limit", + "Use --http-max-input-size to increase the limit.", error_msg, ) @@ -268,7 +268,7 @@ def test_default_limit_json(self): error_msg, ) self.assertIn( - " bytes exceeds the maximum allowed input size", + " bytes exceeds the maximum allowed input size of ", error_msg, ) self.assertIn( @@ -426,15 +426,15 @@ def test_large_input_json(self): # Verify error message contains size limit info error_msg = response.content.decode() self.assertIn( - "Request JSON size of ", + "request JSON size of ", error_msg, ) self.assertIn( - " bytes exceeds the maximum allowed input size", + " bytes exceeds the maximum allowed input size of ", error_msg, ) self.assertIn( - "Use --http-max-input-size to increase the limit", + "Use --http-max-input-size to increase the limit.", error_msg, ) @@ -522,11 +522,11 @@ def test_large_string_in_json(self): error_msg, ) self.assertIn( - " bytes exceeds the maximum allowed input size", + " bytes exceeds the maximum allowed input size of ", error_msg, ) self.assertIn( - "Use --http-max-input-size to increase the limit", + "Use --http-max-input-size to increase the limit.", error_msg, ) diff --git a/src/http_server.cc b/src/http_server.cc index 4afb93bf69..0b105c6396 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -3099,13 +3099,12 @@ HTTPAPIServer::EVBufferToJson( const size_t length, int n) { if (length > max_input_size_) { - auto overrun = length - max_input_size_; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, - ("Request JSON size of " + std::to_string(length) + " + " + - std::to_string(overrun) + - " bytes exceeds the maximum allowed input size. " - "Use --http-max-input-size to increase the limit.") + ("request JSON size of " + std::to_string(length) + + " bytes exceeds the maximum allowed input size of " + + std::to_string(max_input_size_) + + " bytes. Use --http-max-input-size to increase the limit.") .c_str()); } @@ -3648,14 +3647,14 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( // allowed input size. if (byte_size + consumed_input_byte_size > max_input_size_ || byte_size + consumed_input_byte_size < consumed_input_byte_size) { - auto overrun = byte_size + consumed_input_byte_size - max_input_size_; return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, - (std::string("input '") + name + "' has size of " + - std::to_string(max_input_size_ - consumed_input_byte_size) + " + " + - std::to_string(overrun) + - " bytes exceeds the maximum allowed input size. " - "Use --http-max-input-size to increase the limit.") + (std::string("request size of ") + + std::to_string(consumed_input_byte_size) + " bytes with input '" + + name + "' of size " + std::to_string(byte_size) + + " bytes exceeds the maximum allowed input size of " + + std::to_string(max_input_size_) + + ". Use --http-max-input-size to increase the limit.") .c_str()); } From a329b1039f5f1e48b80eb46cbbde910b14dbf7bd Mon Sep 17 00:00:00 2001 From: J Wyman Date: Fri, 3 Apr 2026 17:06:48 -0400 Subject: [PATCH 18/19] rename tests --- qa/L0_http/generate_endpoint_test.py | 15 ++++++++++++++- qa/L0_http/http_input_size_limit_test.py | 16 ++++++++++++++-- qa/L0_http/test.sh | 2 +- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/qa/L0_http/generate_endpoint_test.py b/qa/L0_http/generate_endpoint_test.py index 5089a03997..702bb38f8c 100755 --- a/qa/L0_http/generate_endpoint_test.py +++ b/qa/L0_http/generate_endpoint_test.py @@ -275,7 +275,20 @@ def test_invalid_input_types(self): self.generate_expect_failure(self._model_name, inputs, error_msg) self.generate_stream_expect_failure(self._model_name, inputs, error_msg) - def test_type_size_explosion(self): + def test_json_dtype_size_expansion_exceeds_limit_error(self): + ''' + Test that when the client sends a JSON input of byte[], that when it + expands to dtype[], it exceeds the maximum allowed input size and + returns an appropriate error message. The test sends a large base64 + encoded string as input, which simulates a byte[] input that would + expand to a much larger dtype[] input on the server side when + `sizeof(dtype) > 1`. + The test checks that the error message indicates that the input size + exceeds the limit. + This is important to prevent clients from sending inputs that could + cause excessive memory usage on the server. + ''' + input_data = [1] * ( 64 * 1024 * 1024 ) # 64MB input, which is large but still reasonable for HTTP request body diff --git a/qa/L0_http/http_input_size_limit_test.py b/qa/L0_http/http_input_size_limit_test.py index f78e3cd804..fff4262220 100755 --- a/qa/L0_http/http_input_size_limit_test.py +++ b/qa/L0_http/http_input_size_limit_test.py @@ -64,7 +64,19 @@ class InferSizeLimitTest(tu.TestResultCollector): def _get_infer_url(self, model_name): return f"http://localhost:8000/v2/models/{model_name}/infer" - def test_type_size_explosion(self): + def test_json_dtype_size_expansion_exceeds_limit_error(self): + ''' + Test that when the client sends a JSON input of byte[], that when it + expands to dtype[], it exceeds the maximum allowed input size and + returns an appropriate error message. The test sends a large base64 + encoded string as input, which simulates a byte[] input that would + expand to a much larger dtype[] input on the server side when + `sizeof(dtype) > 1`. + The test checks that the error message indicates that the input size + exceeds the limit. + This is important to prevent clients from sending inputs that could + cause excessive memory usage on the server. + ''' model = "onnx_zero_1_float32" # Provided data is 64MB of int8, but the model expects FP32, @@ -272,7 +284,7 @@ def test_default_limit_json(self): error_msg, ) self.assertIn( - "Use --http-max-input-size to increase the limit", + "Use --http-max-input-size to increase the limit.", error_msg, ) diff --git a/qa/L0_http/test.sh b/qa/L0_http/test.sh index 5e7ec20676..8d6944dbdd 100755 --- a/qa/L0_http/test.sh +++ b/qa/L0_http/test.sh @@ -846,7 +846,7 @@ if [ $? -ne 0 ]; then fi # Run test to verify that large inputs fail with default limit -python http_input_size_limit_test.py InferSizeLimitTest.test_type_size_explosion >> $CLIENT_LOG 2>&1 +python http_input_size_limit_test.py InferSizeLimitTest.test_json_dtype_size_expansion_exceeds_limit_error >> $CLIENT_LOG 2>&1 if [ $? -ne 0 ]; then cat $CLIENT_LOG echo -e "\n***\n*** Default Input Size Limit Test Failed for type size explosion\n***" From be97bed74bb85424fcfc69fb62cb4ca6e7385a9e Mon Sep 17 00:00:00 2001 From: J Wyman Date: Mon, 6 Apr 2026 18:14:52 -0400 Subject: [PATCH 19/19] make pre-commit happy --- qa/L0_http/generate_endpoint_test.py | 4 ++-- qa/L0_http/http_input_size_limit_test.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/qa/L0_http/generate_endpoint_test.py b/qa/L0_http/generate_endpoint_test.py index 702bb38f8c..df9317b4dd 100755 --- a/qa/L0_http/generate_endpoint_test.py +++ b/qa/L0_http/generate_endpoint_test.py @@ -276,7 +276,7 @@ def test_invalid_input_types(self): self.generate_stream_expect_failure(self._model_name, inputs, error_msg) def test_json_dtype_size_expansion_exceeds_limit_error(self): - ''' + """ Test that when the client sends a JSON input of byte[], that when it expands to dtype[], it exceeds the maximum allowed input size and returns an appropriate error message. The test sends a large base64 @@ -287,7 +287,7 @@ def test_json_dtype_size_expansion_exceeds_limit_error(self): exceeds the limit. This is important to prevent clients from sending inputs that could cause excessive memory usage on the server. - ''' + """ input_data = [1] * ( 64 * 1024 * 1024 diff --git a/qa/L0_http/http_input_size_limit_test.py b/qa/L0_http/http_input_size_limit_test.py index fff4262220..73b610eff6 100755 --- a/qa/L0_http/http_input_size_limit_test.py +++ b/qa/L0_http/http_input_size_limit_test.py @@ -65,7 +65,7 @@ def _get_infer_url(self, model_name): return f"http://localhost:8000/v2/models/{model_name}/infer" def test_json_dtype_size_expansion_exceeds_limit_error(self): - ''' + """ Test that when the client sends a JSON input of byte[], that when it expands to dtype[], it exceeds the maximum allowed input size and returns an appropriate error message. The test sends a large base64 @@ -76,7 +76,7 @@ def test_json_dtype_size_expansion_exceeds_limit_error(self): exceeds the limit. This is important to prevent clients from sending inputs that could cause excessive memory usage on the server. - ''' + """ model = "onnx_zero_1_float32" # Provided data is 64MB of int8, but the model expects FP32, @@ -115,6 +115,9 @@ def test_json_dtype_size_expansion_exceeds_limit_error(self): f"Expected error code for type/size mismatch, got: {response.status_code}", ) error_msg = response.content.decode() + print( + f"Error message: {error_msg}", flush=True + ) # Print the error message for debugging self.assertIn( "Request JSON size of ", error_msg, @@ -158,6 +161,9 @@ def test_json_dtype_size_expansion_exceeds_limit_error(self): f"Expected error code for type/size mismatch, got: {response.status_code}", ) error_msg = response.content.decode() + print( + f"Error message: {error_msg}", flush=True + ) # Print the error message for debugging self.assertIn( "request JSON size of ", error_msg,