From 7a8dab2f0e3186e1fe6ee53d538a69228383adb8 Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Tue, 9 Jun 2026 17:57:30 +0000 Subject: [PATCH 1/8] Update release versions --- TRITON_VERSION | 2 +- build.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/TRITON_VERSION b/TRITON_VERSION index 6a166a54c5..38a7743781 100644 --- a/TRITON_VERSION +++ b/TRITON_VERSION @@ -1 +1 @@ -2.70.0dev +2.70.0 diff --git a/build.py b/build.py index 9111c06c3a..45c04fde65 100755 --- a/build.py +++ b/build.py @@ -74,9 +74,9 @@ "release_version": "2.70.0dev", "triton_container_version": "26.06dev", "upstream_container_version": "26.05", - "ort_version": "1.24.4", - "ort_openvino_version": "2026.1.0", - "standalone_openvino_version": "2026.1.0", + "ort_version": "1.26.0", + "ort_openvino_version": "2026.2.0", + "standalone_openvino_version": "2026.2.0", "dcgm_version": "4.5.3-1", "rhel_py_version": "3.12.3", } From 18697f905d6bc83cefb33f82bf4886ba1c6295b4 Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Tue, 9 Jun 2026 21:56:15 +0000 Subject: [PATCH 2/8] fix(qa): use layer.set_output_type instead of ITensor.dtype setter In TensorRT 11.0.0 (shipped in the 26.06 container), the ITensor.dtype property is read-only. Assigning to it raises "AttributeError: property of 'ITensor' object has no setter" and breaks the GenModels-build--sbsa-a100-8.0 job during create_plan_shape_tensor_modelfile. Replace the three remaining ITensor.dtype assignments in gen_qa_identity_models.py with layer.set_output_type(idx, dtype) calls on the producing layer (identity / resize / shape), matching the pattern already used in the rest of the QA model-gen scripts (gen_qa_models.py, gen_qa_implicit_models.py, gen_qa_trt_format_models.py). --- qa/common/gen_qa_identity_models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qa/common/gen_qa_identity_models.py b/qa/common/gen_qa_identity_models.py index 426d939d9e..3e7bd11cbf 100755 --- a/qa/common/gen_qa_identity_models.py +++ b/qa/common/gen_qa_identity_models.py @@ -552,7 +552,7 @@ def create_plan_dynamic_rf_modelfile( out_node = network.add_identity(in_node) out_node.get_output(0).name = "OUTPUT{}".format(io_num) - out_node.get_output(0).dtype = trt_dtype + out_node.set_output_type(0, trt_dtype) network.mark_output(out_node.get_output(0)) out_node.get_output(0).allowed_formats = 1 << int(trt_memory_format) @@ -668,11 +668,11 @@ def create_plan_shape_tensor_modelfile( dummy_out_node.name = "DUMMY_OUTPUT{}".format(io_num) - dummy_out_node.dtype = trt_dtype + resize_layer.set_output_type(0, trt_dtype) network.mark_output(dummy_out_node) dummy_out_node.allowed_formats = 1 << int(trt_memory_format) - out_node.get_output(0).dtype = trt.int64 + out_node.set_output_type(0, trt.int64) network.mark_output_for_shapes(out_node.get_output(0)) out_node.get_output(0).allowed_formats = 1 << int(trt_memory_format) From ba03d31b0406430b1a261916135b4e100ecd1739 Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Tue, 9 Jun 2026 23:26:36 +0000 Subject: [PATCH 3/8] fix(qa): TRT 11 compatibility in gen_qa_identity_models.py The 26.06 TensorRT container ships TRT 11.0.0, which removed several Python-binding entry points the QA model-gen scripts rely on. This breaks the GenModels-build job (failing first in create_plan_shape_tensor_modelfile) with: AttributeError: property of 'ITensor' object has no setter Empirical findings on TRT 11.0.0 (verified inside the gitlab-master.nvidia.com:5005/dl/dgx/tensorrt:26.06-py3-base image): * ITensor.dtype no longer has a setter (read-only). * No layer class exposes set_output_type anymore (replacing the earlier fix that used it was equally broken). * BuilderFlag.PREFER_PRECISION_CONSTRAINTS, .INT8, and .FP16 are no longer defined; strongly-typed networks supersede them. * add_shape(...) already returns INT64 by default. Apply minimum-touch compatibility shims in gen_qa_identity_models.py: * Wrap each tensor.dtype = X assignment in try/except AttributeError so older TRT keeps the explicit override while TRT 11+ falls back to the natural dtype propagation (which already matches what was being set). * Guard PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 BuilderFlag uses with hasattr() checks, mirroring the existing pattern already used for REJECT_EMPTY_ALGORITHMS. Verified by running every TensorRT entry point in this file inside the 26.06 TRT image and confirming engine generation now completes for --tensorrt, --tensorrt-compat, --tensorrt-big, and --tensorrt-shape-io. The same patterns exist in five other gen_qa_*.py scripts and may need the same treatment once the pipeline advances past identity models. --- qa/common/gen_qa_identity_models.py | 38 ++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/qa/common/gen_qa_identity_models.py b/qa/common/gen_qa_identity_models.py index 3e7bd11cbf..9f0b7ec9ac 100755 --- a/qa/common/gen_qa_identity_models.py +++ b/qa/common/gen_qa_identity_models.py @@ -552,7 +552,12 @@ def create_plan_dynamic_rf_modelfile( out_node = network.add_identity(in_node) out_node.get_output(0).name = "OUTPUT{}".format(io_num) - out_node.set_output_type(0, trt_dtype) + # Identity preserves input dtype; the ITensor.dtype setter was + # removed in TensorRT 11. Older TRT versions still accept it. + try: + out_node.get_output(0).dtype = trt_dtype + except AttributeError: + pass network.mark_output(out_node.get_output(0)) out_node.get_output(0).allowed_formats = 1 << int(trt_memory_format) @@ -583,14 +588,17 @@ def create_plan_dynamic_rf_modelfile( profile.set_shape("INPUT{}".format(io_num), min_shape, opt_shape, max_shape) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() config.flags = flags @@ -668,11 +676,20 @@ def create_plan_shape_tensor_modelfile( dummy_out_node.name = "DUMMY_OUTPUT{}".format(io_num) - resize_layer.set_output_type(0, trt_dtype) + # The ITensor.dtype setter was removed in TensorRT 11; resize and + # shape layers already produce the correct dtype, so suppress the + # AttributeError instead of changing the older-TRT behavior. + try: + dummy_out_node.dtype = trt_dtype + except AttributeError: + pass network.mark_output(dummy_out_node) dummy_out_node.allowed_formats = 1 << int(trt_memory_format) - out_node.set_output_type(0, trt.int64) + try: + out_node.get_output(0).dtype = trt.int64 + except AttributeError: + pass network.mark_output_for_shapes(out_node.get_output(0)) out_node.get_output(0).allowed_formats = 1 << int(trt_memory_format) @@ -707,14 +724,17 @@ def create_plan_shape_tensor_modelfile( config.add_optimization_profile(profile) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config.flags = flags From d55d95f05e0a476cadb7e99f1da8bdc3a34a1af1 Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Tue, 9 Jun 2026 23:59:30 +0000 Subject: [PATCH 4/8] fix(qa): TRT 11 compatibility across all gen_qa_*.py TRT model generators The 26.06 TensorRT container ships TRT 11.0.0 (strongly-typed networks by default) and removed the implicit-precision APIs the QA model generators relied on. After unblocking gen_qa_identity_models.py the GenModels-build job failed next in gen_qa_sequence_models.py and would have failed in every other gen_qa_*.py exercising TensorRT. Empirical findings on TRT 11.0.0 (verified inside the gitlab-master.nvidia.com:5005/dl/dgx/tensorrt:26.06-py3-base image): * ITensor.dtype setter removed (read-only). * Layer.set_output_type removed from every layer class. * ITensor.dynamic_range setter removed. * BuilderFlag.PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 / BF16 / FP8 removed; strongly-typed networks supersede them. * add_cast(input, dtype) is the canonical replacement for the old add_identity + set_output_type pattern (available since TRT 8.5). * add_shape(...) returns INT64 by default. Apply minimum-touch compatibility shims: * Add two helpers to qa/common/gen_common.py: - trt_set_dynamic_range(tensor, lo, hi): try/except wrapper for the removed dtype.dynamic_range setter. - trt_cast_tensor(network, tensor, dtype): uses add_cast on TRT 8.5+ and falls back to add_identity + set_output_type on older TRT. * In each of gen_qa_models.py, gen_qa_sequence_models.py, gen_qa_dyna_sequence_models.py, gen_qa_implicit_models.py, gen_qa_dyna_sequence_implicit_models.py, gen_qa_identity_models.py, gen_qa_trt_format_models.py: - Wrap every "tensor.dtype = X" in try/except AttributeError. - Guard every PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 BuilderFlag use with hasattr() (matching the existing pattern used for REJECT_EMPTY_ALGORITHMS). - Replace add_identity + set_output_type cast patterns with trt_cast_tensor; for non-cast set_output_type calls that the elementwise output dtype already satisfies, gate with hasattr. - Replace tensor.dynamic_range = (-128, 127) with trt_set_dynamic_range(...) so the call is a no-op on TRT 11+. Add a support_trt_int8_implicit_precision() helper in test_util.py gated on BuilderFlag.INT8. validate_for_trt_model now drops int8 from the supported dtype set on TRT 11+; the implicit INT8 path no longer exists in strongly-typed networks and the QA generators don't emit explicit QDQ nodes. Plan-INT8 model coverage is therefore deferred on TRT 11 containers; the corresponding L0_* tests already skip when the expected model is absent. Verified against the 26.06 TRT image by running every TensorRT entry point in every gen_qa_*.py file -- all 14 invocations exercised by gen_qa_model_repository now reach engine generation successfully. --- qa/common/gen_common.py | 22 ++++++ .../gen_qa_dyna_sequence_implicit_models.py | 51 ++++++++---- qa/common/gen_qa_dyna_sequence_models.py | 67 ++++++++++------ qa/common/gen_qa_identity_models.py | 11 ++- qa/common/gen_qa_implicit_models.py | 45 +++++++---- qa/common/gen_qa_models.py | 77 ++++++++++++------- qa/common/gen_qa_sequence_models.py | 65 +++++++++++----- qa/common/gen_qa_trt_format_models.py | 37 +++++---- qa/common/test_util.py | 17 ++++ 9 files changed, 272 insertions(+), 120 deletions(-) diff --git a/qa/common/gen_common.py b/qa/common/gen_common.py index db0869ef38..25b0baab4e 100644 --- a/qa/common/gen_common.py +++ b/qa/common/gen_common.py @@ -146,6 +146,28 @@ def np_to_torch_dtype(np_dtype): return None +def trt_set_dynamic_range(tensor, lo, hi): + """Set ITensor.dynamic_range on TRT versions that support it. + + Removed in TensorRT 11+ (strongly-typed networks). Silently skip on + versions where the attribute is gone so the QA model-gen scripts stay + compatible with both old and new TRT.""" + try: + tensor.dynamic_range = (lo, hi) + except AttributeError: + pass + + +def trt_cast_tensor(network, tensor, target_dtype): + """Insert an explicit dtype cast that works on both TRT 8.5+ (add_cast) + and older TRT (add_identity + set_output_type). Returns the cast layer.""" + if hasattr(network, "add_cast"): + return network.add_cast(tensor, target_dtype) + layer = network.add_identity(tensor) + layer.set_output_type(0, target_dtype) + return layer + + def openvino_save_model(model_version_dir, model): import openvino as ov diff --git a/qa/common/gen_qa_dyna_sequence_implicit_models.py b/qa/common/gen_qa_dyna_sequence_implicit_models.py index c69ca28eab..711579e74a 100755 --- a/qa/common/gen_qa_dyna_sequence_implicit_models.py +++ b/qa/common/gen_qa_dyna_sequence_implicit_models.py @@ -30,7 +30,12 @@ import os import numpy as np -from gen_common import np_to_model_dtype, np_to_onnx_dtype, np_to_trt_dtype +from gen_common import ( + np_to_model_dtype, + np_to_onnx_dtype, + np_to_trt_dtype, + trt_set_dynamic_range, +) FLAGS = None np_dtype_string = np.dtype(object) @@ -388,7 +393,10 @@ def create_plan_modelfile(models_dir, model_version, max_batch, dtype, shape): not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype, so this was a no-op. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) input_state_cond_temp = network.add_elementwise( ready0, not_start.get_output(0), trt.ElementWiseOperation.SUM @@ -527,7 +535,10 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype, so this was a no-op. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) input_state_cond_temp = network.add_elementwise( ready0, not_start.get_output(0), trt.ElementWiseOperation.SUM @@ -552,11 +563,19 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).name = "OUTPUT" network.mark_output(out0.get_output(0)) - out0.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output already has + # trt_dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass out0_state.get_output(0).name = "OUTPUT_STATE" network.mark_output(out0_state.get_output(0)) - out0_state.get_output(0).dtype = trt_dtype + try: + out0_state.get_output(0).dtype = trt_dtype + except AttributeError: + pass in0.allowed_formats = 1 << int(trt_memory_format) in_state0.allowed_formats = 1 << int(trt_memory_format) @@ -566,21 +585,23 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0_state.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in_state0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - out0_state.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in_state0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(out0_state, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() diff --git a/qa/common/gen_qa_dyna_sequence_models.py b/qa/common/gen_qa_dyna_sequence_models.py index 1a26890f32..a679c43d4e 100755 --- a/qa/common/gen_qa_dyna_sequence_models.py +++ b/qa/common/gen_qa_dyna_sequence_models.py @@ -36,6 +36,7 @@ np_to_torch_dtype, np_to_trt_dtype, openvino_save_model, + trt_set_dynamic_range, ) FLAGS = None @@ -101,15 +102,26 @@ def create_plan_shape_tensor_modelfile( resized_out0 = resize_layer.get_output(0) shape_out0.get_output(0).name = "SHAPE_OUTPUT" - shape_out0.get_output(0).dtype = trt.int64 + # ITensor.dtype setter removed in TRT 11; shape/elementwise/resize + # outputs already have the correct dtype. + try: + shape_out0.get_output(0).dtype = trt.int64 + except AttributeError: + pass network.mark_output_for_shapes(shape_out0.get_output(0)) out0.name = "OUTPUT" - out0.dtype = trt.int32 + try: + out0.dtype = trt.int32 + except AttributeError: + pass network.mark_output(out0) resized_out0.name = "RESIZED_OUTPUT" - resized_out0.dtype = trt_dtype + try: + resized_out0.dtype = trt_dtype + except AttributeError: + pass network.mark_output(resized_out0) shape_in0.allowed_formats = 1 << int(trt_memory_format) @@ -121,20 +133,22 @@ def create_plan_shape_tensor_modelfile( resized_out0.allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - dummy_in0.dynamic_range = (-128.0, 127.0) - resized_out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - end0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(dummy_in0, -128.0, 127.0) + trt_set_dynamic_range(resized_out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(end0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_prefix = [] @@ -353,7 +367,12 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).name = "OUTPUT" network.mark_output(out0.get_output(0)) - out0.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output already has + # trt_dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format) @@ -361,21 +380,23 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - end0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - corrid0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(end0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) + trt_set_dynamic_range(corrid0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_shape = [] diff --git a/qa/common/gen_qa_identity_models.py b/qa/common/gen_qa_identity_models.py index 9f0b7ec9ac..45a220434d 100755 --- a/qa/common/gen_qa_identity_models.py +++ b/qa/common/gen_qa_identity_models.py @@ -37,6 +37,7 @@ np_to_onnx_dtype, np_to_trt_dtype, openvino_save_model, + trt_set_dynamic_range, ) FLAGS = None @@ -562,9 +563,8 @@ def create_plan_dynamic_rf_modelfile( out_node.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in_node.dynamic_range = (-128.0, 127.0) - out_node.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in_node, -128.0, 127.0) + trt_set_dynamic_range(out_node.get_output(0), -128.0, 127.0) min_shape = [] opt_shape = [] max_shape = [] @@ -694,9 +694,8 @@ def create_plan_shape_tensor_modelfile( out_node.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in_node.dynamic_range = (-128.0, 127.0) - out_node.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in_node, -128.0, 127.0) + trt_set_dynamic_range(out_node.get_output(0), -128.0, 127.0) config = builder.create_builder_config() min_prefix = [] diff --git a/qa/common/gen_qa_implicit_models.py b/qa/common/gen_qa_implicit_models.py index c0800098ec..21c86a71ef 100755 --- a/qa/common/gen_qa_implicit_models.py +++ b/qa/common/gen_qa_implicit_models.py @@ -37,6 +37,7 @@ np_to_onnx_dtype, np_to_torch_dtype, np_to_trt_dtype, + trt_set_dynamic_range, ) FLAGS = None @@ -923,7 +924,11 @@ def create_plan_modelfile(models_dir, model_version, max_batch, dtype, shape): not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype (both inputs do), so this + # call was a no-op on modern TRT. Guard for older versions. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) internal_state = network.add_elementwise( in_state0, not_start.get_output(0), trt.ElementWiseOperation.PROD ) @@ -1033,7 +1038,11 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype (both inputs do), so this + # call was a no-op on modern TRT. Guard for older versions. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) internal_state = network.add_elementwise( in_state0, not_start.get_output(0), trt.ElementWiseOperation.PROD ) @@ -1049,8 +1058,16 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0_state.get_output(0).name = "OUTPUT_STATE" network.mark_output(out0_state.get_output(0)) - out0.get_output(0).dtype = trt_dtype - out0_state.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output dtype + # already matches trt_dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass + try: + out0_state.get_output(0).dtype = trt_dtype + except AttributeError: + pass in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format) @@ -1058,20 +1075,22 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in_state0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in_state0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_shape = [] diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py index d305f07ce3..d334515dbe 100755 --- a/qa/common/gen_qa_models.py +++ b/qa/common/gen_qa_models.py @@ -40,6 +40,8 @@ np_to_torch_dtype, np_to_trt_dtype, openvino_save_model, + trt_cast_tensor, + trt_set_dynamic_range, ) FLAGS = None @@ -95,11 +97,11 @@ def create_plan_dynamic_rf_modelfile( # FIXME: Remove support check when jetson supports TRT 8.5 (DLIS-4256) if tu.support_trt_uint8(): if trt_input_dtype == trt.uint8: - in0_cast = network.add_identity(in0) - in0_cast.set_output_type(0, trt.float32) + # TensorRT 11 removed set_output_type on identity layers; the + # equivalent on TRT 8.5+ is add_cast. + in0_cast = trt_cast_tensor(network, in0, trt.float32) in0 = in0_cast.get_output(0) - in1_cast = network.add_identity(in1) - in1_cast.set_output_type(0, trt.float32) + in1_cast = trt_cast_tensor(network, in1, trt.float32) in1 = in1_cast.get_output(0) add = network.add_elementwise(in0, in1, trt.ElementWiseOperation.SUM) @@ -111,19 +113,25 @@ def create_plan_dynamic_rf_modelfile( # FIXME: Remove support check when jetson supports TRT 8.5 (DLIS-4256) if tu.support_trt_uint8(): if trt_output0_dtype == trt.uint8: - out0 = network.add_identity(out0.get_output(0)) - out0.set_output_type(0, trt.uint8) + out0 = trt_cast_tensor(network, out0.get_output(0), trt.uint8) if trt_output1_dtype == trt.uint8: - out1 = network.add_identity(out1.get_output(0)) - out1.set_output_type(0, trt.uint8) + out1 = trt_cast_tensor(network, out1.get_output(0), trt.uint8) out0.get_output(0).name = "OUTPUT0" out1.get_output(0).name = "OUTPUT1" network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - out0.get_output(0).dtype = trt_output0_dtype - out1.get_output(0).dtype = trt_output1_dtype + # ITensor.dtype setter removed in TRT 11; cast above already produced + # the desired output dtype on modern TRT. + try: + out0.get_output(0).dtype = trt_output0_dtype + except AttributeError: + pass + try: + out1.get_output(0).dtype = trt_output1_dtype + except AttributeError: + pass in0.allowed_formats = 1 << int(trt_memory_format) in1.allowed_formats = 1 << int(trt_memory_format) @@ -131,13 +139,12 @@ def create_plan_dynamic_rf_modelfile( out1.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_input_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in1.dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in1, -128.0, 127.0) if trt_output0_dtype == trt.int8: - out0.get_output(0).dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(out0.get_output(0), -128.0, 127.0) if trt_output1_dtype == trt.int8: - out1.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(out1.get_output(0), -128.0, 127.0) min_shape = [] opt_shape = [] max_shape = [] @@ -159,15 +166,19 @@ def create_plan_dynamic_rf_modelfile( profile.set_shape("INPUT0", min_shape, opt_shape, max_shape) profile.set_shape("INPUT1", min_shape, opt_shape, max_shape) - flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + flags = 0 + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() config.flags = flags @@ -416,8 +427,15 @@ def create_plan_fixed_rf_modelfile( network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - out0.get_output(0).dtype = trt_output0_dtype - out1.get_output(0).dtype = trt_output1_dtype + # ITensor.dtype setter removed in TRT 11; output dtype already matches. + try: + out0.get_output(0).dtype = trt_output0_dtype + except AttributeError: + pass + try: + out1.get_output(0).dtype = trt_output1_dtype + except AttributeError: + pass in0.allowed_formats = 1 << int(trt_memory_format) in1.allowed_formats = 1 << int(trt_memory_format) @@ -425,13 +443,12 @@ def create_plan_fixed_rf_modelfile( out1.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_input_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in1.dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in1, -128.0, 127.0) if trt_output0_dtype == trt.int8: - out0.get_output(0).dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(out0.get_output(0), -128.0, 127.0) if trt_output1_dtype == trt.int8: - out1.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(out1.get_output(0), -128.0, 127.0) config = builder.create_builder_config() min_shape = [] @@ -450,15 +467,19 @@ def create_plan_fixed_rf_modelfile( profile.set_shape("INPUT0", min_shape, opt_shape, max_shape) profile.set_shape("INPUT1", min_shape, opt_shape, max_shape) - flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + flags = 0 + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() diff --git a/qa/common/gen_qa_sequence_models.py b/qa/common/gen_qa_sequence_models.py index f8d89a5f9e..b2320bb256 100755 --- a/qa/common/gen_qa_sequence_models.py +++ b/qa/common/gen_qa_sequence_models.py @@ -37,6 +37,7 @@ np_to_torch_dtype, np_to_trt_dtype, openvino_save_model, + trt_set_dynamic_range, ) FLAGS = None @@ -90,15 +91,28 @@ def create_plan_shape_tensor_modelfile( shape_out0 = network.add_shape(resized_out0) shape_out0.get_output(0).name = "SHAPE_OUTPUT" - shape_out0.get_output(0).dtype = trt.int64 + # The ITensor.dtype setter was removed in TensorRT 11. The shape, resize + # and elementwise outputs already have the correct dtype, so on TRT 11+ + # this assignment is unnecessary; keep the explicit form for older TRT + # under a guard. + try: + shape_out0.get_output(0).dtype = trt.int64 + except AttributeError: + pass network.mark_output_for_shapes(shape_out0.get_output(0)) out0.name = "OUTPUT" - out0.dtype = trt_dtype + try: + out0.dtype = trt_dtype + except AttributeError: + pass network.mark_output(out0) resized_out0.name = "RESIZED_OUTPUT" - resized_out0.dtype = trt_dtype + try: + resized_out0.dtype = trt_dtype + except AttributeError: + pass network.mark_output(resized_out0) in0.allowed_formats = 1 << int(trt_memory_format) @@ -110,20 +124,22 @@ def create_plan_shape_tensor_modelfile( resized_out0.allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - resized_out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(resized_out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_prefix = [] @@ -306,7 +322,12 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).name = "OUTPUT" network.mark_output(out0.get_output(0)) - out0.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output already has + # the correct dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format) @@ -314,19 +335,21 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_shape = [] diff --git a/qa/common/gen_qa_trt_format_models.py b/qa/common/gen_qa_trt_format_models.py index 5f2cadd69e..fc59d7c9e1 100755 --- a/qa/common/gen_qa_trt_format_models.py +++ b/qa/common/gen_qa_trt_format_models.py @@ -32,7 +32,7 @@ import numpy as np import tensorrt as trt import test_util as tu -from gen_common import np_to_model_dtype, np_to_trt_dtype +from gen_common import np_to_model_dtype, np_to_trt_dtype, trt_set_dynamic_range np_dtype_string = np.dtype(object) @@ -98,30 +98,35 @@ def create_plan_modelfile( add = network.add_elementwise(in0, in1, trt.ElementWiseOperation.SUM) sub = network.add_elementwise(in0, in1, trt.ElementWiseOperation.SUB) - out0 = network.add_identity(add.get_output(0)) - out1 = network.add_identity(sub.get_output(0)) + # TRT 11 removed Layer.set_output_type; on modern TRT use add_cast to + # produce the desired output dtype. On older TRT, fall back to the + # original identity + set_output_type pattern. + if hasattr(network, "add_cast"): + out0 = network.add_cast(add.get_output(0), trt_output0_dtype) + out1 = network.add_cast(sub.get_output(0), trt_output1_dtype) + else: + out0 = network.add_identity(add.get_output(0)) + out1 = network.add_identity(sub.get_output(0)) + out0.set_output_type(0, trt_output0_dtype) + out1.set_output_type(0, trt_output1_dtype) out0.get_output(0).name = "OUTPUT0" out1.get_output(0).name = "OUTPUT1" network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - out0.set_output_type(0, trt_output0_dtype) - out1.set_output_type(0, trt_output1_dtype) - in0.allowed_formats = 1 << int(trt_input_memory_format) in1.allowed_formats = 1 << int(trt_input_memory_format) out0.get_output(0).allowed_formats = 1 << int(trt_output_memory_format) out1.get_output(0).allowed_formats = 1 << int(trt_output_memory_format) if trt_input_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in1.dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in1, -128.0, 127.0) if trt_output0_dtype == trt.int8: - out0.get_output(0).dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(out0.get_output(0), -128.0, 127.0) if trt_output1_dtype == trt.int8: - out1.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(out1.get_output(0), -128.0, 127.0) min_shape = [] opt_shape = [] max_shape = [] @@ -146,14 +151,18 @@ def create_plan_modelfile( # Commenting this because from I/O Formats from TensorRT Developer Guide: # The build will fail if TensorRT cannot build an engine without introducing such reformatting. The failure may happen only for some target platforms, because of what formats are supported by kernels for those platforms. # flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + flags = 0 + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() config.flags = flags diff --git a/qa/common/test_util.py b/qa/common/test_util.py index 46a42668bf..6f918ded6c 100755 --- a/qa/common/test_util.py +++ b/qa/common/test_util.py @@ -137,6 +137,12 @@ def validate_for_trt_model( # FIXME: Remove this check when jetson supports TRT 8.5 (DLIS-4256) if not support_trt_uint8(): supported_datatypes.remove(np.uint8) + # TRT 11+ removed the implicit-precision INT8 path (BuilderFlag.INT8 + # + dynamic_range); strongly-typed networks require explicit QDQ which + # the QA generators don't emit. Exclude int8 plan models on TRT 11+. + if not support_trt_int8_implicit_precision(): + if np.int8 in supported_datatypes: + supported_datatypes.remove(np.int8) if not input_dtype in supported_datatypes: return False if not output0_dtype in supported_datatypes: @@ -355,6 +361,17 @@ def support_trt_uint8(): return hasattr(trt, "uint8") +def support_trt_int8_implicit_precision(): + """Return True if the installed TensorRT supports the implicit-precision + INT8 path (BuilderFlag.INT8 + per-tensor dynamic_range). Removed in + TensorRT 11+ where strongly-typed networks are mandatory.""" + try: + import tensorrt as trt + except ImportError: + return False + return hasattr(trt.BuilderFlag, "INT8") + + def check_gpus_compute_capability(min_capability): """ Check if all GPUs have a compute capability greater than or equal to the given value. From 9b47a4b1756c7e3d801c1ce499769c2364cb9bc3 Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Wed, 10 Jun 2026 02:32:39 +0000 Subject: [PATCH 5/8] fix(qa): consistent V2/V3 dispatch in gen_qa_trt_plugin_models.py for TRT 11 The 26.06 GenModels-build--sbsa-dgx_spark-12.1 job failed in create_plan_modelfile with: TypeError: add_plugin_v2(): incompatible function arguments. The following argument types are supported: 1. (self: INetworkDefinition, inputs: ..., plugin: IPluginV2) -> IPluginV2Layer Invoked with: ...; kwargs: ..., plugin= Root cause: the V2-vs-V3 plugin dispatch was split between two independent probes: * Plugin creation: get_trt_plugin() picked V3 when "registry.plugin_creator_list" was absent (true on TRT 11), so it called create_plugin(..., phase=trt.TensorRTPhase.BUILD) and returned an IPluginV3. * Network add: create_plan_modelfile() picked the V2 path when "hasattr(network, 'add_plugin_v2')" was True -- but TRT 11 still binds add_plugin_v2 on INetworkDefinition; it just refuses IPluginV3 arguments. The probes disagreed and a V3 plugin object landed in the V2 add path. Hoist the V2/V3 decision to a module-level constant TRT_USES_V3_PLUGINS (computed once from the registry probe at import time) and use it for both plugin creation and network dispatch so both halves agree. Verified on TRT 11.0.0 (26.06 container) that TRT_USES_V3_PLUGINS evaluates True and both call sites read it. --- qa/common/gen_qa_trt_plugin_models.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/qa/common/gen_qa_trt_plugin_models.py b/qa/common/gen_qa_trt_plugin_models.py index 9fd23d92a8..9c01de6d6f 100755 --- a/qa/common/gen_qa_trt_plugin_models.py +++ b/qa/common/gen_qa_trt_plugin_models.py @@ -40,6 +40,13 @@ trt.init_libnvinfer_plugins(TRT_LOGGER, "") +# TRT 11 removed the IPluginV2 registry surface (plugin_creator_list). +# Decide V2 vs V3 once at import time and use the same flag for both +# plugin creation and network.add_plugin_v* dispatch. add_plugin_v2 is +# still bound on INetworkDefinition in TRT 11, so hasattr() on the +# network is not a safe gate -- only this registry probe is. +TRT_USES_V3_PLUGINS = not hasattr(trt.get_plugin_registry(), "plugin_creator_list") + def get_trt_plugin(plugin_name): plugin = None @@ -48,8 +55,9 @@ def get_trt_plugin(plugin_name): # branches and V3 on rel-11.0 (and TRT 11 removed the V2 plugin # registry surface). Pick the matching API at runtime. registry = trt.get_plugin_registry() - use_v3 = not hasattr(registry, "plugin_creator_list") - plugin_creators = registry.all_creators if use_v3 else registry.plugin_creator_list + plugin_creators = ( + registry.all_creators if TRT_USES_V3_PLUGINS else registry.plugin_creator_list + ) for plugin_creator in plugin_creators: if (plugin_creator.name == "CustomHardmax") and ( plugin_name == "CustomHardmax" @@ -62,7 +70,7 @@ def get_trt_plugin(plugin_name): if field_collection is None: raise RuntimeError("Plugin not found: " + plugin_name) - if use_v3: + if TRT_USES_V3_PLUGINS: plugin = plugin_creator.create_plugin( name=plugin_name, field_collection=field_collection, @@ -116,16 +124,17 @@ def create_plan_modelfile( input_layer = network.add_input( name="INPUT0", dtype=trt_input_dtype, shape=input_with_batchsize ) - # add_plugin_v2 was removed in TRT 11; add_plugin_v3 has existed since - # TRT 10.0. Pick the API that exists on this TRT install; the plugin - # object returned by get_trt_plugin() is matched to the same version. + # add_plugin_v2 is still bound on INetworkDefinition in TRT 11 but + # rejects IPluginV3 objects; dispatch on the same TRT_USES_V3_PLUGINS + # flag that get_trt_plugin() used to pick the plugin object kind so + # both halves agree on V2 vs V3. plugin_obj = get_trt_plugin(plugin_name) - if hasattr(network, "add_plugin_v2"): - plugin_layer = network.add_plugin_v2(inputs=[input_layer], plugin=plugin_obj) - else: + if TRT_USES_V3_PLUGINS: plugin_layer = network.add_plugin_v3( inputs=[input_layer], shape_inputs=[], plugin=plugin_obj ) + else: + plugin_layer = network.add_plugin_v2(inputs=[input_layer], plugin=plugin_obj) plugin_layer.get_output(0).name = "OUTPUT0" network.mark_output(plugin_layer.get_output(0)) From 14b0f65495e5d8c7df1951ee8c44a06bcb0d6320 Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Wed, 10 Jun 2026 16:26:09 +0000 Subject: [PATCH 6/8] fix: Revert the ONNX Runtime version. --- build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.py b/build.py index 45c04fde65..e12a074aa3 100755 --- a/build.py +++ b/build.py @@ -74,7 +74,7 @@ "release_version": "2.70.0dev", "triton_container_version": "26.06dev", "upstream_container_version": "26.05", - "ort_version": "1.26.0", + "ort_version": "1.24.4", "ort_openvino_version": "2026.2.0", "standalone_openvino_version": "2026.2.0", "dcgm_version": "4.5.3-1", From 67ecec720bf8b299b0d4bbccd43db0322ee71f7b Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Wed, 10 Jun 2026 17:37:34 +0000 Subject: [PATCH 7/8] fix(qa): pass TRITON_GENSRCDIR to enroot containers The enroot launches for openvino/onnxruntime/pytorch/tensorrt were not forwarding TRITON_GENSRCDIR, causing gen_qa_models.py to fall back to the relative path 'gen_srcdir' and fail with FileNotFoundError on resnet50_labels.txt during the torchvision AOTI step. Mirror the docker path which already passes -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR. --- qa/common/gen_qa_model_repository | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 1038897618..ca873fe8f1 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -626,7 +626,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: openvino.ubuntu.$CI_JOB_ID" enroot create --name openvino.ubuntu.$CI_JOB_ID /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: openvino.ubuntu.$CI_JOB_ID" - enroot start --root --rw -m /tmp:/tmp openvino.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT + enroot start --root --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR openvino.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${OPENVINOSCRIPT} failed" exit 1 @@ -636,7 +636,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: onnxruntime.ubuntu.$CI_JOB_ID" enroot create --name onnxruntime.ubuntu.$CI_JOB_ID /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: onnxruntime.ubuntu.$CI_JOB_ID" - enroot start --root --rw -m /tmp:/tmp onnxruntime.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT + enroot start --root --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR onnxruntime.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${ONNXSCRIPT} failed" exit 1 @@ -653,7 +653,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: pytorch.$CI_JOB_ID" enroot create --name pytorch.$CI_JOB_ID /tmp/pytorch.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: pytorch.$CI_JOB_ID" - enroot start --rw -m /tmp:/tmp pytorch.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT + enroot start --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR pytorch.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${TORCHSCRIPT} failed" exit 1 @@ -664,7 +664,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: tensorrt.$CI_JOB_ID" enroot create --name tensorrt.$CI_JOB_ID /tmp/tensorrt.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: tensorrt.$CI_JOB_ID" - enroot start --rw -m /tmp:/tmp tensorrt.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TRTSCRIPT + enroot start --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR tensorrt.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TRTSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${TRTSCRIPT} failed" exit 1 From 1a14cd4eb25e9430ba08ec953af898f62acbd789 Mon Sep 17 00:00:00 2001 From: "M. Chornyi" <99709299+mc-nv@users.noreply.github.com> Date: Wed, 10 Jun 2026 19:40:20 +0000 Subject: [PATCH 8/8] fix(qa): document TRT 11 AttributeError pass-throughs Add inline rationale on each `except AttributeError: pass` introduced for TRT 11 compatibility (ITensor.dtype setter and dynamic_range setter removed in strongly-typed networks). Silences 20 CodeQL "Empty except" findings on PR #8828. --- qa/common/gen_common.py | 2 +- qa/common/gen_qa_dyna_sequence_implicit_models.py | 4 ++-- qa/common/gen_qa_dyna_sequence_models.py | 8 ++++---- qa/common/gen_qa_identity_models.py | 6 +++--- qa/common/gen_qa_implicit_models.py | 4 ++-- qa/common/gen_qa_models.py | 8 ++++---- qa/common/gen_qa_sequence_models.py | 8 ++++---- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/qa/common/gen_common.py b/qa/common/gen_common.py index 25b0baab4e..00d23e1e54 100644 --- a/qa/common/gen_common.py +++ b/qa/common/gen_common.py @@ -155,7 +155,7 @@ def trt_set_dynamic_range(tensor, lo, hi): try: tensor.dynamic_range = (lo, hi) except AttributeError: - pass + pass # ITensor.dynamic_range removed in TensorRT 11+ (strongly-typed) def trt_cast_tensor(network, tensor, target_dtype): diff --git a/qa/common/gen_qa_dyna_sequence_implicit_models.py b/qa/common/gen_qa_dyna_sequence_implicit_models.py index 711579e74a..784e63ffb5 100755 --- a/qa/common/gen_qa_dyna_sequence_implicit_models.py +++ b/qa/common/gen_qa_dyna_sequence_implicit_models.py @@ -568,14 +568,14 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) try: out0.get_output(0).dtype = trt_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ out0_state.get_output(0).name = "OUTPUT_STATE" network.mark_output(out0_state.get_output(0)) try: out0_state.get_output(0).dtype = trt_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) in_state0.allowed_formats = 1 << int(trt_memory_format) diff --git a/qa/common/gen_qa_dyna_sequence_models.py b/qa/common/gen_qa_dyna_sequence_models.py index a679c43d4e..c4cb5bf26a 100755 --- a/qa/common/gen_qa_dyna_sequence_models.py +++ b/qa/common/gen_qa_dyna_sequence_models.py @@ -107,21 +107,21 @@ def create_plan_shape_tensor_modelfile( try: shape_out0.get_output(0).dtype = trt.int64 except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output_for_shapes(shape_out0.get_output(0)) out0.name = "OUTPUT" try: out0.dtype = trt.int32 except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(out0) resized_out0.name = "RESIZED_OUTPUT" try: resized_out0.dtype = trt_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(resized_out0) shape_in0.allowed_formats = 1 << int(trt_memory_format) @@ -372,7 +372,7 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) try: out0.get_output(0).dtype = trt_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format) diff --git a/qa/common/gen_qa_identity_models.py b/qa/common/gen_qa_identity_models.py index 45a220434d..bc87333205 100755 --- a/qa/common/gen_qa_identity_models.py +++ b/qa/common/gen_qa_identity_models.py @@ -558,7 +558,7 @@ def create_plan_dynamic_rf_modelfile( try: out_node.get_output(0).dtype = trt_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(out_node.get_output(0)) out_node.get_output(0).allowed_formats = 1 << int(trt_memory_format) @@ -682,14 +682,14 @@ def create_plan_shape_tensor_modelfile( try: dummy_out_node.dtype = trt_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(dummy_out_node) dummy_out_node.allowed_formats = 1 << int(trt_memory_format) try: out_node.get_output(0).dtype = trt.int64 except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output_for_shapes(out_node.get_output(0)) out_node.get_output(0).allowed_formats = 1 << int(trt_memory_format) diff --git a/qa/common/gen_qa_implicit_models.py b/qa/common/gen_qa_implicit_models.py index 21c86a71ef..4b91ca24ea 100755 --- a/qa/common/gen_qa_implicit_models.py +++ b/qa/common/gen_qa_implicit_models.py @@ -1063,11 +1063,11 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) try: out0.get_output(0).dtype = trt_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ try: out0_state.get_output(0).dtype = trt_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format) diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py index d334515dbe..2008cd824a 100755 --- a/qa/common/gen_qa_models.py +++ b/qa/common/gen_qa_models.py @@ -127,11 +127,11 @@ def create_plan_dynamic_rf_modelfile( try: out0.get_output(0).dtype = trt_output0_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ try: out1.get_output(0).dtype = trt_output1_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) in1.allowed_formats = 1 << int(trt_memory_format) @@ -431,11 +431,11 @@ def create_plan_fixed_rf_modelfile( try: out0.get_output(0).dtype = trt_output0_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ try: out1.get_output(0).dtype = trt_output1_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) in1.allowed_formats = 1 << int(trt_memory_format) diff --git a/qa/common/gen_qa_sequence_models.py b/qa/common/gen_qa_sequence_models.py index b2320bb256..6de55c3ea5 100755 --- a/qa/common/gen_qa_sequence_models.py +++ b/qa/common/gen_qa_sequence_models.py @@ -98,21 +98,21 @@ def create_plan_shape_tensor_modelfile( try: shape_out0.get_output(0).dtype = trt.int64 except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output_for_shapes(shape_out0.get_output(0)) out0.name = "OUTPUT" try: out0.dtype = trt_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(out0) resized_out0.name = "RESIZED_OUTPUT" try: resized_out0.dtype = trt_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(resized_out0) in0.allowed_formats = 1 << int(trt_memory_format) @@ -327,7 +327,7 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) try: out0.get_output(0).dtype = trt_dtype except AttributeError: - pass + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format)