diff --git a/TRITON_VERSION b/TRITON_VERSION index 6a166a54c5..38a7743781 100644 --- a/TRITON_VERSION +++ b/TRITON_VERSION @@ -1 +1 @@ -2.70.0dev +2.70.0 diff --git a/build.py b/build.py index 9111c06c3a..e12a074aa3 100755 --- a/build.py +++ b/build.py @@ -75,8 +75,8 @@ "triton_container_version": "26.06dev", "upstream_container_version": "26.05", "ort_version": "1.24.4", - "ort_openvino_version": "2026.1.0", - "standalone_openvino_version": "2026.1.0", + "ort_openvino_version": "2026.2.0", + "standalone_openvino_version": "2026.2.0", "dcgm_version": "4.5.3-1", "rhel_py_version": "3.12.3", } diff --git a/qa/common/gen_common.py b/qa/common/gen_common.py index db0869ef38..00d23e1e54 100644 --- a/qa/common/gen_common.py +++ b/qa/common/gen_common.py @@ -146,6 +146,28 @@ def np_to_torch_dtype(np_dtype): return None +def trt_set_dynamic_range(tensor, lo, hi): + """Set ITensor.dynamic_range on TRT versions that support it. + + Removed in TensorRT 11+ (strongly-typed networks). Silently skip on + versions where the attribute is gone so the QA model-gen scripts stay + compatible with both old and new TRT.""" + try: + tensor.dynamic_range = (lo, hi) + except AttributeError: + pass # ITensor.dynamic_range removed in TensorRT 11+ (strongly-typed) + + +def trt_cast_tensor(network, tensor, target_dtype): + """Insert an explicit dtype cast that works on both TRT 8.5+ (add_cast) + and older TRT (add_identity + set_output_type). Returns the cast layer.""" + if hasattr(network, "add_cast"): + return network.add_cast(tensor, target_dtype) + layer = network.add_identity(tensor) + layer.set_output_type(0, target_dtype) + return layer + + def openvino_save_model(model_version_dir, model): import openvino as ov diff --git a/qa/common/gen_qa_dyna_sequence_implicit_models.py b/qa/common/gen_qa_dyna_sequence_implicit_models.py index c69ca28eab..784e63ffb5 100755 --- a/qa/common/gen_qa_dyna_sequence_implicit_models.py +++ b/qa/common/gen_qa_dyna_sequence_implicit_models.py @@ -30,7 +30,12 @@ import os import numpy as np -from gen_common import np_to_model_dtype, np_to_onnx_dtype, np_to_trt_dtype +from gen_common import ( + np_to_model_dtype, + np_to_onnx_dtype, + np_to_trt_dtype, + trt_set_dynamic_range, +) FLAGS = None np_dtype_string = np.dtype(object) @@ -388,7 +393,10 @@ def create_plan_modelfile(models_dir, model_version, max_batch, dtype, shape): not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype, so this was a no-op. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) input_state_cond_temp = network.add_elementwise( ready0, not_start.get_output(0), trt.ElementWiseOperation.SUM @@ -527,7 +535,10 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype, so this was a no-op. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) input_state_cond_temp = network.add_elementwise( ready0, not_start.get_output(0), trt.ElementWiseOperation.SUM @@ -552,11 +563,19 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).name = "OUTPUT" network.mark_output(out0.get_output(0)) - out0.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output already has + # trt_dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ out0_state.get_output(0).name = "OUTPUT_STATE" network.mark_output(out0_state.get_output(0)) - out0_state.get_output(0).dtype = trt_dtype + try: + out0_state.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) in_state0.allowed_formats = 1 << int(trt_memory_format) @@ -566,21 +585,23 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0_state.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in_state0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - out0_state.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in_state0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(out0_state, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() diff --git a/qa/common/gen_qa_dyna_sequence_models.py b/qa/common/gen_qa_dyna_sequence_models.py index 1a26890f32..c4cb5bf26a 100755 --- a/qa/common/gen_qa_dyna_sequence_models.py +++ b/qa/common/gen_qa_dyna_sequence_models.py @@ -36,6 +36,7 @@ np_to_torch_dtype, np_to_trt_dtype, openvino_save_model, + trt_set_dynamic_range, ) FLAGS = None @@ -101,15 +102,26 @@ def create_plan_shape_tensor_modelfile( resized_out0 = resize_layer.get_output(0) shape_out0.get_output(0).name = "SHAPE_OUTPUT" - shape_out0.get_output(0).dtype = trt.int64 + # ITensor.dtype setter removed in TRT 11; shape/elementwise/resize + # outputs already have the correct dtype. + try: + shape_out0.get_output(0).dtype = trt.int64 + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output_for_shapes(shape_out0.get_output(0)) out0.name = "OUTPUT" - out0.dtype = trt.int32 + try: + out0.dtype = trt.int32 + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(out0) resized_out0.name = "RESIZED_OUTPUT" - resized_out0.dtype = trt_dtype + try: + resized_out0.dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(resized_out0) shape_in0.allowed_formats = 1 << int(trt_memory_format) @@ -121,20 +133,22 @@ def create_plan_shape_tensor_modelfile( resized_out0.allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - dummy_in0.dynamic_range = (-128.0, 127.0) - resized_out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - end0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(dummy_in0, -128.0, 127.0) + trt_set_dynamic_range(resized_out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(end0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_prefix = [] @@ -353,7 +367,12 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).name = "OUTPUT" network.mark_output(out0.get_output(0)) - out0.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output already has + # trt_dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format) @@ -361,21 +380,23 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - end0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - corrid0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(end0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) + trt_set_dynamic_range(corrid0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_shape = [] diff --git a/qa/common/gen_qa_identity_models.py b/qa/common/gen_qa_identity_models.py index 426d939d9e..bc87333205 100755 --- a/qa/common/gen_qa_identity_models.py +++ b/qa/common/gen_qa_identity_models.py @@ -37,6 +37,7 @@ np_to_onnx_dtype, np_to_trt_dtype, openvino_save_model, + trt_set_dynamic_range, ) FLAGS = None @@ -552,14 +553,18 @@ def create_plan_dynamic_rf_modelfile( out_node = network.add_identity(in_node) out_node.get_output(0).name = "OUTPUT{}".format(io_num) - out_node.get_output(0).dtype = trt_dtype + # Identity preserves input dtype; the ITensor.dtype setter was + # removed in TensorRT 11. Older TRT versions still accept it. + try: + out_node.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(out_node.get_output(0)) out_node.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in_node.dynamic_range = (-128.0, 127.0) - out_node.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in_node, -128.0, 127.0) + trt_set_dynamic_range(out_node.get_output(0), -128.0, 127.0) min_shape = [] opt_shape = [] max_shape = [] @@ -583,14 +588,17 @@ def create_plan_dynamic_rf_modelfile( profile.set_shape("INPUT{}".format(io_num), min_shape, opt_shape, max_shape) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() config.flags = flags @@ -668,18 +676,26 @@ def create_plan_shape_tensor_modelfile( dummy_out_node.name = "DUMMY_OUTPUT{}".format(io_num) - dummy_out_node.dtype = trt_dtype + # The ITensor.dtype setter was removed in TensorRT 11; resize and + # shape layers already produce the correct dtype, so suppress the + # AttributeError instead of changing the older-TRT behavior. + try: + dummy_out_node.dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(dummy_out_node) dummy_out_node.allowed_formats = 1 << int(trt_memory_format) - out_node.get_output(0).dtype = trt.int64 + try: + out_node.get_output(0).dtype = trt.int64 + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output_for_shapes(out_node.get_output(0)) out_node.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in_node.dynamic_range = (-128.0, 127.0) - out_node.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in_node, -128.0, 127.0) + trt_set_dynamic_range(out_node.get_output(0), -128.0, 127.0) config = builder.create_builder_config() min_prefix = [] @@ -707,14 +723,17 @@ def create_plan_shape_tensor_modelfile( config.add_optimization_profile(profile) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config.flags = flags diff --git a/qa/common/gen_qa_implicit_models.py b/qa/common/gen_qa_implicit_models.py index c0800098ec..4b91ca24ea 100755 --- a/qa/common/gen_qa_implicit_models.py +++ b/qa/common/gen_qa_implicit_models.py @@ -37,6 +37,7 @@ np_to_onnx_dtype, np_to_torch_dtype, np_to_trt_dtype, + trt_set_dynamic_range, ) FLAGS = None @@ -923,7 +924,11 @@ def create_plan_modelfile(models_dir, model_version, max_batch, dtype, shape): not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype (both inputs do), so this + # call was a no-op on modern TRT. Guard for older versions. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) internal_state = network.add_elementwise( in_state0, not_start.get_output(0), trt.ElementWiseOperation.PROD ) @@ -1033,7 +1038,11 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype (both inputs do), so this + # call was a no-op on modern TRT. Guard for older versions. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) internal_state = network.add_elementwise( in_state0, not_start.get_output(0), trt.ElementWiseOperation.PROD ) @@ -1049,8 +1058,16 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0_state.get_output(0).name = "OUTPUT_STATE" network.mark_output(out0_state.get_output(0)) - out0.get_output(0).dtype = trt_dtype - out0_state.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output dtype + # already matches trt_dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ + try: + out0_state.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format) @@ -1058,20 +1075,22 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in_state0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in_state0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_shape = [] diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 1038897618..ca873fe8f1 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -626,7 +626,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: openvino.ubuntu.$CI_JOB_ID" enroot create --name openvino.ubuntu.$CI_JOB_ID /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: openvino.ubuntu.$CI_JOB_ID" - enroot start --root --rw -m /tmp:/tmp openvino.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT + enroot start --root --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR openvino.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${OPENVINOSCRIPT} failed" exit 1 @@ -636,7 +636,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: onnxruntime.ubuntu.$CI_JOB_ID" enroot create --name onnxruntime.ubuntu.$CI_JOB_ID /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: onnxruntime.ubuntu.$CI_JOB_ID" - enroot start --root --rw -m /tmp:/tmp onnxruntime.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT + enroot start --root --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR onnxruntime.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${ONNXSCRIPT} failed" exit 1 @@ -653,7 +653,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: pytorch.$CI_JOB_ID" enroot create --name pytorch.$CI_JOB_ID /tmp/pytorch.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: pytorch.$CI_JOB_ID" - enroot start --rw -m /tmp:/tmp pytorch.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT + enroot start --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR pytorch.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${TORCHSCRIPT} failed" exit 1 @@ -664,7 +664,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: tensorrt.$CI_JOB_ID" enroot create --name tensorrt.$CI_JOB_ID /tmp/tensorrt.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: tensorrt.$CI_JOB_ID" - enroot start --rw -m /tmp:/tmp tensorrt.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TRTSCRIPT + enroot start --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR tensorrt.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TRTSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${TRTSCRIPT} failed" exit 1 diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py index d305f07ce3..2008cd824a 100755 --- a/qa/common/gen_qa_models.py +++ b/qa/common/gen_qa_models.py @@ -40,6 +40,8 @@ np_to_torch_dtype, np_to_trt_dtype, openvino_save_model, + trt_cast_tensor, + trt_set_dynamic_range, ) FLAGS = None @@ -95,11 +97,11 @@ def create_plan_dynamic_rf_modelfile( # FIXME: Remove support check when jetson supports TRT 8.5 (DLIS-4256) if tu.support_trt_uint8(): if trt_input_dtype == trt.uint8: - in0_cast = network.add_identity(in0) - in0_cast.set_output_type(0, trt.float32) + # TensorRT 11 removed set_output_type on identity layers; the + # equivalent on TRT 8.5+ is add_cast. + in0_cast = trt_cast_tensor(network, in0, trt.float32) in0 = in0_cast.get_output(0) - in1_cast = network.add_identity(in1) - in1_cast.set_output_type(0, trt.float32) + in1_cast = trt_cast_tensor(network, in1, trt.float32) in1 = in1_cast.get_output(0) add = network.add_elementwise(in0, in1, trt.ElementWiseOperation.SUM) @@ -111,19 +113,25 @@ def create_plan_dynamic_rf_modelfile( # FIXME: Remove support check when jetson supports TRT 8.5 (DLIS-4256) if tu.support_trt_uint8(): if trt_output0_dtype == trt.uint8: - out0 = network.add_identity(out0.get_output(0)) - out0.set_output_type(0, trt.uint8) + out0 = trt_cast_tensor(network, out0.get_output(0), trt.uint8) if trt_output1_dtype == trt.uint8: - out1 = network.add_identity(out1.get_output(0)) - out1.set_output_type(0, trt.uint8) + out1 = trt_cast_tensor(network, out1.get_output(0), trt.uint8) out0.get_output(0).name = "OUTPUT0" out1.get_output(0).name = "OUTPUT1" network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - out0.get_output(0).dtype = trt_output0_dtype - out1.get_output(0).dtype = trt_output1_dtype + # ITensor.dtype setter removed in TRT 11; cast above already produced + # the desired output dtype on modern TRT. + try: + out0.get_output(0).dtype = trt_output0_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ + try: + out1.get_output(0).dtype = trt_output1_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) in1.allowed_formats = 1 << int(trt_memory_format) @@ -131,13 +139,12 @@ def create_plan_dynamic_rf_modelfile( out1.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_input_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in1.dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in1, -128.0, 127.0) if trt_output0_dtype == trt.int8: - out0.get_output(0).dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(out0.get_output(0), -128.0, 127.0) if trt_output1_dtype == trt.int8: - out1.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(out1.get_output(0), -128.0, 127.0) min_shape = [] opt_shape = [] max_shape = [] @@ -159,15 +166,19 @@ def create_plan_dynamic_rf_modelfile( profile.set_shape("INPUT0", min_shape, opt_shape, max_shape) profile.set_shape("INPUT1", min_shape, opt_shape, max_shape) - flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + flags = 0 + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() config.flags = flags @@ -416,8 +427,15 @@ def create_plan_fixed_rf_modelfile( network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - out0.get_output(0).dtype = trt_output0_dtype - out1.get_output(0).dtype = trt_output1_dtype + # ITensor.dtype setter removed in TRT 11; output dtype already matches. + try: + out0.get_output(0).dtype = trt_output0_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ + try: + out1.get_output(0).dtype = trt_output1_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) in1.allowed_formats = 1 << int(trt_memory_format) @@ -425,13 +443,12 @@ def create_plan_fixed_rf_modelfile( out1.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_input_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in1.dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in1, -128.0, 127.0) if trt_output0_dtype == trt.int8: - out0.get_output(0).dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(out0.get_output(0), -128.0, 127.0) if trt_output1_dtype == trt.int8: - out1.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(out1.get_output(0), -128.0, 127.0) config = builder.create_builder_config() min_shape = [] @@ -450,15 +467,19 @@ def create_plan_fixed_rf_modelfile( profile.set_shape("INPUT0", min_shape, opt_shape, max_shape) profile.set_shape("INPUT1", min_shape, opt_shape, max_shape) - flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + flags = 0 + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() diff --git a/qa/common/gen_qa_sequence_models.py b/qa/common/gen_qa_sequence_models.py index f8d89a5f9e..6de55c3ea5 100755 --- a/qa/common/gen_qa_sequence_models.py +++ b/qa/common/gen_qa_sequence_models.py @@ -37,6 +37,7 @@ np_to_torch_dtype, np_to_trt_dtype, openvino_save_model, + trt_set_dynamic_range, ) FLAGS = None @@ -90,15 +91,28 @@ def create_plan_shape_tensor_modelfile( shape_out0 = network.add_shape(resized_out0) shape_out0.get_output(0).name = "SHAPE_OUTPUT" - shape_out0.get_output(0).dtype = trt.int64 + # The ITensor.dtype setter was removed in TensorRT 11. The shape, resize + # and elementwise outputs already have the correct dtype, so on TRT 11+ + # this assignment is unnecessary; keep the explicit form for older TRT + # under a guard. + try: + shape_out0.get_output(0).dtype = trt.int64 + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output_for_shapes(shape_out0.get_output(0)) out0.name = "OUTPUT" - out0.dtype = trt_dtype + try: + out0.dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(out0) resized_out0.name = "RESIZED_OUTPUT" - resized_out0.dtype = trt_dtype + try: + resized_out0.dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(resized_out0) in0.allowed_formats = 1 << int(trt_memory_format) @@ -110,20 +124,22 @@ def create_plan_shape_tensor_modelfile( resized_out0.allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - resized_out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(resized_out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_prefix = [] @@ -306,7 +322,12 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).name = "OUTPUT" network.mark_output(out0.get_output(0)) - out0.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output already has + # the correct dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format) @@ -314,19 +335,21 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_shape = [] diff --git a/qa/common/gen_qa_trt_format_models.py b/qa/common/gen_qa_trt_format_models.py index 5f2cadd69e..fc59d7c9e1 100755 --- a/qa/common/gen_qa_trt_format_models.py +++ b/qa/common/gen_qa_trt_format_models.py @@ -32,7 +32,7 @@ import numpy as np import tensorrt as trt import test_util as tu -from gen_common import np_to_model_dtype, np_to_trt_dtype +from gen_common import np_to_model_dtype, np_to_trt_dtype, trt_set_dynamic_range np_dtype_string = np.dtype(object) @@ -98,30 +98,35 @@ def create_plan_modelfile( add = network.add_elementwise(in0, in1, trt.ElementWiseOperation.SUM) sub = network.add_elementwise(in0, in1, trt.ElementWiseOperation.SUB) - out0 = network.add_identity(add.get_output(0)) - out1 = network.add_identity(sub.get_output(0)) + # TRT 11 removed Layer.set_output_type; on modern TRT use add_cast to + # produce the desired output dtype. On older TRT, fall back to the + # original identity + set_output_type pattern. + if hasattr(network, "add_cast"): + out0 = network.add_cast(add.get_output(0), trt_output0_dtype) + out1 = network.add_cast(sub.get_output(0), trt_output1_dtype) + else: + out0 = network.add_identity(add.get_output(0)) + out1 = network.add_identity(sub.get_output(0)) + out0.set_output_type(0, trt_output0_dtype) + out1.set_output_type(0, trt_output1_dtype) out0.get_output(0).name = "OUTPUT0" out1.get_output(0).name = "OUTPUT1" network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - out0.set_output_type(0, trt_output0_dtype) - out1.set_output_type(0, trt_output1_dtype) - in0.allowed_formats = 1 << int(trt_input_memory_format) in1.allowed_formats = 1 << int(trt_input_memory_format) out0.get_output(0).allowed_formats = 1 << int(trt_output_memory_format) out1.get_output(0).allowed_formats = 1 << int(trt_output_memory_format) if trt_input_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in1.dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in1, -128.0, 127.0) if trt_output0_dtype == trt.int8: - out0.get_output(0).dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(out0.get_output(0), -128.0, 127.0) if trt_output1_dtype == trt.int8: - out1.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(out1.get_output(0), -128.0, 127.0) min_shape = [] opt_shape = [] max_shape = [] @@ -146,14 +151,18 @@ def create_plan_modelfile( # Commenting this because from I/O Formats from TensorRT Developer Guide: # The build will fail if TensorRT cannot build an engine without introducing such reformatting. The failure may happen only for some target platforms, because of what formats are supported by kernels for those platforms. # flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + flags = 0 + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() config.flags = flags diff --git a/qa/common/gen_qa_trt_plugin_models.py b/qa/common/gen_qa_trt_plugin_models.py index 9fd23d92a8..9c01de6d6f 100755 --- a/qa/common/gen_qa_trt_plugin_models.py +++ b/qa/common/gen_qa_trt_plugin_models.py @@ -40,6 +40,13 @@ trt.init_libnvinfer_plugins(TRT_LOGGER, "") +# TRT 11 removed the IPluginV2 registry surface (plugin_creator_list). +# Decide V2 vs V3 once at import time and use the same flag for both +# plugin creation and network.add_plugin_v* dispatch. add_plugin_v2 is +# still bound on INetworkDefinition in TRT 11, so hasattr() on the +# network is not a safe gate -- only this registry probe is. +TRT_USES_V3_PLUGINS = not hasattr(trt.get_plugin_registry(), "plugin_creator_list") + def get_trt_plugin(plugin_name): plugin = None @@ -48,8 +55,9 @@ def get_trt_plugin(plugin_name): # branches and V3 on rel-11.0 (and TRT 11 removed the V2 plugin # registry surface). Pick the matching API at runtime. registry = trt.get_plugin_registry() - use_v3 = not hasattr(registry, "plugin_creator_list") - plugin_creators = registry.all_creators if use_v3 else registry.plugin_creator_list + plugin_creators = ( + registry.all_creators if TRT_USES_V3_PLUGINS else registry.plugin_creator_list + ) for plugin_creator in plugin_creators: if (plugin_creator.name == "CustomHardmax") and ( plugin_name == "CustomHardmax" @@ -62,7 +70,7 @@ def get_trt_plugin(plugin_name): if field_collection is None: raise RuntimeError("Plugin not found: " + plugin_name) - if use_v3: + if TRT_USES_V3_PLUGINS: plugin = plugin_creator.create_plugin( name=plugin_name, field_collection=field_collection, @@ -116,16 +124,17 @@ def create_plan_modelfile( input_layer = network.add_input( name="INPUT0", dtype=trt_input_dtype, shape=input_with_batchsize ) - # add_plugin_v2 was removed in TRT 11; add_plugin_v3 has existed since - # TRT 10.0. Pick the API that exists on this TRT install; the plugin - # object returned by get_trt_plugin() is matched to the same version. + # add_plugin_v2 is still bound on INetworkDefinition in TRT 11 but + # rejects IPluginV3 objects; dispatch on the same TRT_USES_V3_PLUGINS + # flag that get_trt_plugin() used to pick the plugin object kind so + # both halves agree on V2 vs V3. plugin_obj = get_trt_plugin(plugin_name) - if hasattr(network, "add_plugin_v2"): - plugin_layer = network.add_plugin_v2(inputs=[input_layer], plugin=plugin_obj) - else: + if TRT_USES_V3_PLUGINS: plugin_layer = network.add_plugin_v3( inputs=[input_layer], shape_inputs=[], plugin=plugin_obj ) + else: + plugin_layer = network.add_plugin_v2(inputs=[input_layer], plugin=plugin_obj) plugin_layer.get_output(0).name = "OUTPUT0" network.mark_output(plugin_layer.get_output(0)) diff --git a/qa/common/test_util.py b/qa/common/test_util.py index 46a42668bf..6f918ded6c 100755 --- a/qa/common/test_util.py +++ b/qa/common/test_util.py @@ -137,6 +137,12 @@ def validate_for_trt_model( # FIXME: Remove this check when jetson supports TRT 8.5 (DLIS-4256) if not support_trt_uint8(): supported_datatypes.remove(np.uint8) + # TRT 11+ removed the implicit-precision INT8 path (BuilderFlag.INT8 + # + dynamic_range); strongly-typed networks require explicit QDQ which + # the QA generators don't emit. Exclude int8 plan models on TRT 11+. + if not support_trt_int8_implicit_precision(): + if np.int8 in supported_datatypes: + supported_datatypes.remove(np.int8) if not input_dtype in supported_datatypes: return False if not output0_dtype in supported_datatypes: @@ -355,6 +361,17 @@ def support_trt_uint8(): return hasattr(trt, "uint8") +def support_trt_int8_implicit_precision(): + """Return True if the installed TensorRT supports the implicit-precision + INT8 path (BuilderFlag.INT8 + per-tensor dynamic_range). Removed in + TensorRT 11+ where strongly-typed networks are mandatory.""" + try: + import tensorrt as trt + except ImportError: + return False + return hasattr(trt.BuilderFlag, "INT8") + + def check_gpus_compute_capability(min_capability): """ Check if all GPUs have a compute capability greater than or equal to the given value.