diff --git a/qa/L0_http/generate_endpoint_test.py b/qa/L0_http/generate_endpoint_test.py index 3eb0b6ea5f..df9317b4dd 100755 --- a/qa/L0_http/generate_endpoint_test.py +++ b/qa/L0_http/generate_endpoint_test.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -29,6 +29,7 @@ sys.path.append("../common") +import base64 import json import threading import time @@ -274,6 +275,37 @@ def test_invalid_input_types(self): self.generate_expect_failure(self._model_name, inputs, error_msg) self.generate_stream_expect_failure(self._model_name, inputs, error_msg) + def test_json_dtype_size_expansion_exceeds_limit_error(self): + """ + Test that when the client sends a JSON input of byte[], that when it + expands to dtype[], it exceeds the maximum allowed input size and + returns an appropriate error message. The test sends a large base64 + encoded string as input, which simulates a byte[] input that would + expand to a much larger dtype[] input on the server side when + `sizeof(dtype) > 1`. + The test checks that the error message indicates that the input size + exceeds the limit. + This is important to prevent clients from sending inputs that could + cause excessive memory usage on the server. + """ + + input_data = [1] * ( + 64 * 1024 * 1024 + ) # 64MB input, which is large but still reasonable for HTTP request body + input_bytes = bytes(input_data) + input_str = base64.b64encode(input_bytes).decode("utf-8") + inputs = {"PROMPT": input_str, "STREAM": False} + error_msg = " bytes exceeds the maximum allowed input size of " + self.generate_expect_failure(self._model_name, inputs, error_msg) + + inputs = { + "INPUT0": input_str[0 : (len(input_str) // 2)], + "INPUT1": input_str[(len(input_str) // 2) :], + "STREAM": False, + } + error_msg = " bytes exceeds the maximum allowed input size of " + self.generate_expect_failure(self._model_name, inputs, error_msg) + def test_duplicate_inputs(self): dupe_prompt = "input 'PROMPT' already exists in request" dupe_stream = "input 'STREAM' already exists in request" diff --git a/qa/L0_http/http_input_size_limit_test.py b/qa/L0_http/http_input_size_limit_test.py index f0ee35649b..73b610eff6 100755 --- a/qa/L0_http/http_input_size_limit_test.py +++ b/qa/L0_http/http_input_size_limit_test.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -29,6 +29,7 @@ sys.path.append("../common") +import base64 import gzip import io import json @@ -41,6 +42,9 @@ # Constants for size calculations # Each FP32 value is 4 bytes, so we need to divide target byte sizes by 4 to get element counts BYTES_PER_FP32 = 4 +BYTES_PER_INT64 = ( + 8 # For the type size explosion test, we use int64 which is 8 bytes per element +) MB = 2**20 # 1 MB = 1,048,576 bytes GB = 2**30 # 1 GB = 1,073,741,824 bytes DEFAULT_LIMIT_BYTES = 64 * MB # 64MB default limit @@ -58,7 +62,120 @@ class InferSizeLimitTest(tu.TestResultCollector): def _get_infer_url(self, model_name): - return "http://localhost:8000/v2/models/{}/infer".format(model_name) + return f"http://localhost:8000/v2/models/{model_name}/infer" + + def test_json_dtype_size_expansion_exceeds_limit_error(self): + """ + Test that when the client sends a JSON input of byte[], that when it + expands to dtype[], it exceeds the maximum allowed input size and + returns an appropriate error message. The test sends a large base64 + encoded string as input, which simulates a byte[] input that would + expand to a much larger dtype[] input on the server side when + `sizeof(dtype) > 1`. + The test checks that the error message indicates that the input size + exceeds the limit. + This is important to prevent clients from sending inputs that could + cause excessive memory usage on the server. + """ + model = "onnx_zero_1_float32" + + # Provided data is 64MB of int8, but the model expects FP32, + # which would expand to 256MB when interpreted as FP32. + bytes_input = np.ones(DEFAULT_LIMIT_BYTES, dtype=np.int8) + input_bytes = bytes_input.tobytes() + data_str = base64.b64encode(input_bytes).decode("utf-8") + headers = { + "Content-Type": "application/json", + "Inference-Header-Content-Length": f"{len(input_bytes)}", + } + shape_size = ( + DEFAULT_LIMIT_ELEMENTS // BYTES_PER_INT64 + ) # Calculate shape size based on int64 element count to match the byte size + + payload = { + "inputs": [ + { + "name": "INPUT0", + "datatype": "INT64", + "shape": [1, shape_size], + "data": data_str, + } + ] + } + + response = requests.post( + f"http://localhost:8000/v2/models/{model}/generate", + headers=headers, + json=payload, + ) + + self.assertEqual( + 400, + response.status_code, + f"Expected error code for type/size mismatch, got: {response.status_code}", + ) + error_msg = response.content.decode() + print( + f"Error message: {error_msg}", flush=True + ) # Print the error message for debugging + self.assertIn( + "Request JSON size of ", + error_msg, + ) + self.assertIn( + " bytes exceeds the maximum allowed input size of ", + error_msg, + ) + self.assertIn( + "Use --http-max-input-size to increase the limit.", + error_msg, + ) + + # Test multiple inputs with one that causes size explosion. + payload = { + "inputs": [ + { + "name": "INPUT0", + "datatype": "INT64", + "shape": [1, shape_size // 2], + "data": data_str[: len(data_str) // 2], + }, + { + "name": "INPUT1", + "datatype": "INT64", + "shape": [1, shape_size // 2], + "data": data_str[len(data_str) // 2 :], + }, + ] + } + + response = requests.post( + f"http://localhost:8000/v2/models/{model}/generate", + headers=headers, + json=payload, + ) + + self.assertEqual( + 400, + response.status_code, + f"Expected error code for type/size mismatch, got: {response.status_code}", + ) + error_msg = response.content.decode() + print( + f"Error message: {error_msg}", flush=True + ) # Print the error message for debugging + self.assertIn( + "request JSON size of ", + error_msg, + ) + self.assertIn( + " bytes exceeds the maximum allowed input size of ", + error_msg, + ) + self.assertIn( + "Use --http-max-input-size to increase the limit.", + error_msg, + ) def test_default_limit_raw_binary(self): """Test raw binary inputs with default limit""" @@ -165,9 +282,16 @@ def test_default_limit_json(self): # Verify error message contains size limit info error_msg = response.content.decode() self.assertIn( - "exceeds the maximum allowed value", + "Request JSON size of ", + error_msg, + ) + self.assertIn( + " bytes exceeds the maximum allowed input size of ", + error_msg, + ) + self.assertIn( + "Use --http-max-input-size to increase the limit.", error_msg, - "Expected error message about exceeding max input size", ) # Test case 2: Input just under the 64MB limit (should succeed) @@ -320,9 +444,16 @@ def test_large_input_json(self): # Verify error message contains size limit info error_msg = response.content.decode() self.assertIn( - "exceeds the maximum allowed value", + "request JSON size of ", + error_msg, + ) + self.assertIn( + " bytes exceeds the maximum allowed input size of ", + error_msg, + ) + self.assertIn( + "Use --http-max-input-size to increase the limit.", error_msg, - "Expected error message about exceeding max input size", ) # Test case 2: Input just under the 128MB configured limit (should succeed) @@ -405,15 +536,15 @@ def test_large_string_in_json(self): # Verify error message error_msg = response.content.decode() self.assertIn( - "Request JSON size", + "Request JSON size of ", error_msg, ) self.assertIn( - "exceeds the maximum allowed value", + " bytes exceeds the maximum allowed input size of ", error_msg, ) self.assertIn( - "Use --http-max-input-size to increase the limit", + "Use --http-max-input-size to increase the limit.", error_msg, ) diff --git a/qa/L0_http/http_test.py b/qa/L0_http/http_test.py index 775e1aba36..48ef1728f4 100755 --- a/qa/L0_http/http_test.py +++ b/qa/L0_http/http_test.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -368,7 +368,7 @@ def test_loading_large_invalid_model(self): error_message, ) self.assertIn( - "exceeds the maximum allowed value", + " exceeds the maximum allowed input size. ", error_message, ) except ValueError: diff --git a/qa/L0_http/test.sh b/qa/L0_http/test.sh index f5b4548c24..8d6944dbdd 100755 --- a/qa/L0_http/test.sh +++ b/qa/L0_http/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -255,7 +255,7 @@ echo -n 'username:' > pswd echo "password" | openssl passwd -stdin -apr1 >> pswd nginx -c `pwd`/$NGINX_CONF -python $BASIC_AUTH_TEST +python $BASIC_AUTH_TEST >> ${CLIENT_LOG}.python.plugin.auth 2>&1 if [ $? -ne 0 ]; then cat ${CLIENT_LOG}.python.plugin.auth RET=1 @@ -380,9 +380,15 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "1" ]; then + echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then + echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi @@ -397,9 +403,15 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "0" ]; then + echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] found in output when not expected" + cat ./curl.out + echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then + echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi @@ -414,9 +426,15 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "0" ]; then + echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] found in output when not expected" + cat ./curl.out + echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then + echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi @@ -431,9 +449,15 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "0" ]; then + echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] found in output when not expected" + cat ./curl.out + echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "0" ]; then + echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] found in output when not expected" + cat ./curl.out + echo "" RET=1 fi @@ -449,9 +473,15 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "1" ]; then + echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then + echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi @@ -465,7 +495,10 @@ if [ "$code" == "200" ]; then echo -e "\n***\n*** Test Failed\n***" RET=1 fi -if [ `grep -c "\{\"error\":\"Unable to parse 'data': Shape does not match true shape of 'data' field\"\}" ./curl.out` != "1" ]; then +if [ `grep -c "\{\"error\":\"Failed to parse 'data' field: shape does not match true shape\"\}" ./curl.out` != "1" ]; then + echo -e "\{\"error\":\"Failed to parse 'data' field: shape does not match true shape\"\} not found in output when expected" + cat ./curl.out + echo "" RET=1 fi @@ -479,6 +512,9 @@ if [ "$code" == "200" ]; then RET=1 fi if [ `grep -c "\{\"error\":\"Unable to parse 'data': Shape does not match true shape of 'data' field\"\}" ./curl.out` != "1" ]; then + echo -e "\{\"error\":\"Unable to parse 'data': Shape does not match true shape of 'data' field\"\} not found in output when expected" + cat ./curl.out + echo "" RET=1 fi @@ -493,9 +529,15 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "1" ]; then + echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then + echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" + cat ./curl.out + echo "" RET=1 fi @@ -655,7 +697,7 @@ wait $SERVER_PID # https://github.com/mpetazzoni/sseclient pip install sseclient-py -SERVER_ARGS="--model-repository=`pwd`/../python_models/generate_models" +SERVER_ARGS="--model-repository=`pwd`/../python_models/generate_models --log-verbose=1" SERVER_LOG="./inference_server_generate_endpoint_test.log" CLIENT_LOG="./generate_endpoint_test.log" run_server @@ -668,9 +710,9 @@ fi ## Python Unit Tests TEST_RESULT_FILE='test_results.txt' PYTHON_TEST=generate_endpoint_test.py -EXPECTED_NUM_TESTS=17 +EXPECTED_NUM_TESTS=18 set +e -python $PYTHON_TEST >$CLIENT_LOG 2>&1 +python $PYTHON_TEST > $CLIENT_LOG 2>&1 if [ $? -ne 0 ]; then cat $CLIENT_LOG RET=1 @@ -802,6 +844,14 @@ if [ $? -ne 0 ]; then echo -e "\n***\n*** Default Input Size Limit Test Failed for compressed input\n***" RET=1 fi + +# Run test to verify that large inputs fail with default limit +python http_input_size_limit_test.py InferSizeLimitTest.test_json_dtype_size_expansion_exceeds_limit_error >> $CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Default Input Size Limit Test Failed for type size explosion\n***" + RET=1 +fi set -e kill $SERVER_PID diff --git a/src/http_server.cc b/src/http_server.cc index cce57ee254..0b105c6396 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -478,6 +478,11 @@ ReadDataFromJsonHelper( triton::common::TritonJson::Value& tensor_data, int* counter, int64_t expected_cnt, int current_depth = 0) { + if (!base) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INTERNAL, + "Failed to parse 'data' field: output buffer unavailable"); + } // FIXME should move 'switch' statement outside the recursive function and // pass in a read data callback once data type is confirmed. // Currently 'switch' is performed on each element even through all elements @@ -501,6 +506,10 @@ ReadDataFromJsonHelper( base, dtype, el, counter, expected_cnt, current_depth + 1)); } } else { + if (!counter) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INTERNAL, "Invalid counter provided"); + } // Check if writing to 'serialized' is overrunning the expected byte_size if (*counter < 0 || static_cast(*counter) >= expected_cnt) { return TRITONSERVER_ErrorNew( @@ -618,7 +627,7 @@ ReadDataFromJsonHelper( if (actual_cnt < 0 || actual_cnt > expected_cnt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - "Shape does not match true shape of 'data' field"); + "shape does not match true shape of 'data' field"); } memcpy( base + *counter, reinterpret_cast(&len), sizeof(uint32_t)); @@ -668,6 +677,11 @@ ReadDataFromJson( .c_str()); default: + if (!base) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INTERNAL, + "Failed to parse 'data' field: output buffer unavailable"); + } RETURN_MSG_IF_ERR( ReadDataFromJsonHelper( base, dtype, tensor_data, &counter, expected_cnt), @@ -679,8 +693,7 @@ ReadDataFromJson( if (counter != expected_cnt) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - "Unable to parse 'data': Shape does not match true shape of 'data' " - "field"); + "Failed to parse 'data' field: shape does not match true shape"); } return nullptr; @@ -695,7 +708,8 @@ WriteDataToJsonCheck( return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( - "output tensor shape does not match size of output for '" + + "Failed to write 'data' field: output tensor shape does not match " + "size of output for '" + output_name + "'") .c_str()); } @@ -716,7 +730,8 @@ WriteDataToJson( return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, std::string( - "output tensor shape does not match size of output for '" + + "Failed to write 'data' field: output tensor shape does not " + "match size of output for '" + output_name + "'") .c_str()); } @@ -3086,8 +3101,8 @@ HTTPAPIServer::EVBufferToJson( if (length > max_input_size_) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, - ("Request JSON size of " + std::to_string(length) + - " bytes exceeds the maximum allowed value of " + + ("request JSON size of " + std::to_string(length) + + " bytes exceeds the maximum allowed input size of " + std::to_string(max_input_size_) + " bytes. Use --http-max-input-size to increase the limit.") .c_str()); @@ -3361,12 +3376,12 @@ HTTPAPIServer::HandleGenerate( server_.get(), req, GetResponseCompressionType(req), generate_stream_request_schema_.get(), generate_stream_response_schema_.get(), streaming, irequest_shared, - shm_manager_)); + shm_manager_, max_input_size_)); } else { generate_request.reset(new GenerateRequestClass( server_.get(), req, GetResponseCompressionType(req), generate_request_schema_.get(), generate_response_schema_.get(), - streaming, irequest_shared, shm_manager_)); + streaming, irequest_shared, shm_manager_, max_input_size_)); } generate_request->trace_ = trace; @@ -3513,13 +3528,16 @@ HTTPAPIServer::GenerateRequestClass::ConvertGenerateRequest( std::vector members; RETURN_IF_ERR(generate_request.Members(&members)); + size_t consumed_input_size{0}; + for (const auto& m : members) { auto it = schema->children_.find(m); if (it != schema->children_.end()) { switch (it->second->kind_) { case MappingSchema::Kind::EXACT_MAPPING: { // Read meta data - RETURN_IF_ERR(ExactMappingInput(m, generate_request, input_metadata)); + RETURN_IF_ERR(ExactMappingInput( + m, generate_request, input_metadata, consumed_input_size)); break; } case MappingSchema::Kind::MAPPING_SCHEMA: { @@ -3549,7 +3567,8 @@ HTTPAPIServer::GenerateRequestClass::ConvertGenerateRequest( } } else if (schema->allow_unspecified_) { // Unspecified key follows EXACT_MAPPING - RETURN_IF_ERR(ExactMappingInput(m, generate_request, input_metadata)); + RETURN_IF_ERR(ExactMappingInput( + m, generate_request, input_metadata, consumed_input_size)); } else { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_UNSUPPORTED, @@ -3563,7 +3582,8 @@ TRITONSERVER_Error* HTTPAPIServer::GenerateRequestClass::ExactMappingInput( const std::string& name, triton::common::TritonJson::Value& generate_request, - std::map& input_metadata) + std::map& input_metadata, + size_t& consumed_input_byte_size) { auto it = input_metadata.find(name); if (it == input_metadata.end()) { @@ -3587,15 +3607,59 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( .c_str()); } + size_t byte_size{0}; size_t element_cnt = tensor_data.IsArray() ? tensor_data.ArraySize() : 1; - size_t byte_size = 0; if (dtype == TRITONSERVER_TYPE_BYTES) { RETURN_IF_ERR(JsonBytesArrayByteSize(tensor_data, &byte_size)); } else { - byte_size = element_cnt * TRITONSERVER_DataTypeByteSize(dtype); + size_t element_size = TRITONSERVER_DataTypeByteSize(dtype); + if (element_size == 0) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + (std::string("input '") + name + "' has unsupported datatype " + + value) + .c_str()); + } + + // Ensure that we do not have an integer overflow when calculating + // byte_size = element_cnt * element_size. + if (element_cnt > (SIZE_MAX / element_size)) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + (std::string("input '") + name + + "' has too many elements of datatype " + value) + .c_str()); + } + + byte_size = element_cnt * element_size; + } + + // For zero-size input, we can skip the rest of the validation and just add + // it as an empty input. + if (byte_size == 0) { + RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddInput( + triton_request_.get(), name.c_str(), dtype, nullptr, 0)); + return nullptr; } + // Ensure that the resulting array size in bytes does not exceed the maximum + // allowed input size. + if (byte_size + consumed_input_byte_size > max_input_size_ || + byte_size + consumed_input_byte_size < consumed_input_byte_size) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, + (std::string("request size of ") + + std::to_string(consumed_input_byte_size) + " bytes with input '" + + name + "' of size " + std::to_string(byte_size) + + " bytes exceeds the maximum allowed input size of " + + std::to_string(max_input_size_) + + ". Use --http-max-input-size to increase the limit.") + .c_str()); + } + + consumed_input_byte_size += byte_size; + std::vector shape_vec; { triton::common::TritonJson::Value value; @@ -3607,11 +3671,13 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( name) .c_str()); } + for (size_t i = 0; i < value.ArraySize(); ++i) { int64_t d = 0; RETURN_IF_ERR(value.IndexAsInt(i, &d)); shape_vec.push_back(d); } + // Because generate request don't carry too much shape information, using // a two-pass process to pad the request value to match input shape. // 1. iterate shape for fixed dimension to distribute 'element_cnt'. @@ -3629,12 +3695,14 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( element_cnt /= *rit; } } + for (auto rit = shape_vec.rbegin(); rit != shape_vec.rend(); ++rit) { if (*rit == -1) { *rit = element_cnt; element_cnt = 1; } } + if (element_cnt != 1) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, @@ -3649,10 +3717,10 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingInput( serialized_data_.emplace_back(); std::vector& serialized = serialized_data_.back(); serialized.resize(byte_size); + RETURN_IF_ERR(ReadDataFromJson( name.c_str(), tensor_data, &serialized[0], dtype, dtype == TRITONSERVER_TYPE_BYTES ? byte_size : element_cnt)); - RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddInput( triton_request_.get(), name.c_str(), dtype, &shape_vec[0], shape_vec.size())); diff --git a/src/http_server.h b/src/http_server.h index c5ed0eb6de..2187ec1700 100644 --- a/src/http_server.h +++ b/src/http_server.h @@ -1,4 +1,4 @@ -// Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -388,12 +388,13 @@ class HTTPAPIServer : public HTTPServer { const MappingSchema* request_schema, const MappingSchema* response_schema, bool streaming, const std::shared_ptr& triton_request, - const std::shared_ptr& shm_manager) + const std::shared_ptr& shm_manager, + const size_t max_input_size) : InferRequestClass( server, req, response_compression_type, triton_request, shm_manager), request_schema_(request_schema), response_schema_(response_schema), - streaming_(streaming) + streaming_(streaming), max_input_size_(max_input_size) { } virtual ~GenerateRequestClass(); @@ -443,7 +444,8 @@ class HTTPAPIServer : public HTTPServer { TRITONSERVER_Error* ExactMappingInput( const std::string& name, triton::common::TritonJson::Value& value, std::map& - input_metadata); + input_metadata, + size_t& consumed_input_byte_size); // [DLIS-5551] currently always performs basic conversion, only maps schema // of EXACT_MAPPING kind. MAPPING_SCHEMA and upcoming kinds are for @@ -461,6 +463,7 @@ class HTTPAPIServer : public HTTPServer { const MappingSchema* request_schema_{nullptr}; const MappingSchema* response_schema_{nullptr}; const bool streaming_{false}; + const size_t max_input_size_{0}; // Placeholder to completing response, this class does not own // the response. TRITONSERVER_InferenceResponse* triton_response_{nullptr};