From 1e0a4c96f6da6bcbc63482482fb896f4beb26a50 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Mon, 13 Oct 2025 14:00:27 -0700 Subject: [PATCH 01/10] update: Versions (#8459) --- Dockerfile.sdk | 2 +- README.md | 10 +++++----- build.py | 2 +- deploy/aws/values.yaml | 2 +- deploy/fleetcommand/Chart.yaml | 2 +- deploy/fleetcommand/values.yaml | 6 +++--- deploy/gcp/values.yaml | 2 +- .../benchmark/perf-analyzer-script/triton_client.yaml | 2 +- .../server-deployer/build_and_push.sh | 6 +++--- .../server-deployer/chart/triton/Chart.yaml | 2 +- .../server-deployer/chart/triton/values.yaml | 6 +++--- .../server-deployer/data-test/schema.yaml | 2 +- deploy/gke-marketplace-app/server-deployer/schema.yaml | 4 ++-- deploy/gke-marketplace-app/trt-engine/README.md | 6 +++--- deploy/k8s-onprem/values.yaml | 2 +- deploy/oci/values.yaml | 2 +- docs/introduction/release_notes.md | 4 ++-- python/openai/README.md | 2 +- qa/common/gen_jetson_trt_models | 2 +- qa/common/gen_qa_custom_ops | 2 +- qa/common/gen_qa_model_repository | 2 +- 21 files changed, 35 insertions(+), 35 deletions(-) diff --git a/Dockerfile.sdk b/Dockerfile.sdk index e919b3f50d..1804041101 100644 --- a/Dockerfile.sdk +++ b/Dockerfile.sdk @@ -29,7 +29,7 @@ # # Base image on the minimum Triton container -ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.09-py3-min +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.10-py3-min ARG TRITON_CLIENT_REPO_SUBDIR=clientrepo ARG TRITON_REPO_ORGANIZATION=http://github.com/triton-inference-server diff --git a/README.md b/README.md index 7be70a9ecc..cdd14ee53e 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ >[!WARNING] >You are currently on the `main` branch which tracks under-development progress ->towards the next release. The current release is version [2.61.0](https://github.com/triton-inference-server/server/releases/latest) ->and corresponds to the 25.09 container release on NVIDIA GPU Cloud (NGC). +>towards the next release. The current release is version [2.62.0](https://github.com/triton-inference-server/server/releases/latest) +>and corresponds to the 25.10 container release on NVIDIA GPU Cloud (NGC). # Triton Inference Server @@ -90,16 +90,16 @@ Inference Server with the ```bash # Step 1: Create the example model repository -git clone -b r25.09 https://github.com/triton-inference-server/server.git +git clone -b r25.10 https://github.com/triton-inference-server/server.git cd server/docs/examples ./fetch_models.sh # Step 2: Launch triton from the NGC Triton container -docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:25.09-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx +docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:25.10-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx # Step 3: Sending an Inference Request # In a separate console, launch the image_client example from the NGC Triton SDK container -docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:25.09-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg +docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:25.10-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg # Inference should return the following Image '/workspace/images/mug.jpg': diff --git a/build.py b/build.py index 03c84a7876..2a0fbb9ed2 100755 --- a/build.py +++ b/build.py @@ -73,7 +73,7 @@ DEFAULT_TRITON_VERSION_MAP = { "release_version": "2.63.0dev", "triton_container_version": "25.11dev", - "upstream_container_version": "25.09", + "upstream_container_version": "25.10", "ort_version": "1.23.1", "ort_openvino_version": "2025.3.0", "standalone_openvino_version": "2025.3.0", diff --git a/deploy/aws/values.yaml b/deploy/aws/values.yaml index 27bec3dd10..7c782b1ed9 100644 --- a/deploy/aws/values.yaml +++ b/deploy/aws/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.09-py3 + imageName: nvcr.io/nvidia/tritonserver:25.10-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/fleetcommand/Chart.yaml b/deploy/fleetcommand/Chart.yaml index 8b0eed9bae..2cbc1cbb34 100644 --- a/deploy/fleetcommand/Chart.yaml +++ b/deploy/fleetcommand/Chart.yaml @@ -26,7 +26,7 @@ apiVersion: v1 # appVersion is the Triton version; update when changing release -appVersion: 2.61.0" +appVersion: 2.62.0" description: Triton Inference Server (Fleet Command) name: triton-inference-server # version is the Chart version; update when changing anything in the chart diff --git a/deploy/fleetcommand/values.yaml b/deploy/fleetcommand/values.yaml index aa2e45e192..830407c9fe 100644 --- a/deploy/fleetcommand/values.yaml +++ b/deploy/fleetcommand/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.09-py3 + imageName: nvcr.io/nvidia/tritonserver:25.10-py3 pullPolicy: IfNotPresent numGpus: 1 serverCommand: tritonserver @@ -47,13 +47,13 @@ image: # # To set model control mode, uncomment and configure below # TODO: Fix the following url, it is invalid - # See https://github.com/triton-inference-server/server/blob/r25.09/docs/user_guide/model_management.md + # See https://github.com/triton-inference-server/server/blob/r25.10/docs/user_guide/model_management.md # for more details #- --model-control-mode=explicit|poll|none # # Additional server args # - # see https://github.com/triton-inference-server/server/blob/r25.09/README.md + # see https://github.com/triton-inference-server/server/blob/r25.10/README.md # for more details service: diff --git a/deploy/gcp/values.yaml b/deploy/gcp/values.yaml index c4c2ff9951..b66268bce2 100644 --- a/deploy/gcp/values.yaml +++ b/deploy/gcp/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.09-py3 + imageName: nvcr.io/nvidia/tritonserver:25.10-py3 pullPolicy: IfNotPresent modelRepositoryPath: gs://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml index 4b21697c29..d65a50862f 100644 --- a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml +++ b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml @@ -33,7 +33,7 @@ metadata: namespace: default spec: containers: - - image: nvcr.io/nvidia/tritonserver:25.09-py3-sdk + - image: nvcr.io/nvidia/tritonserver:25.10-py3-sdk imagePullPolicy: Always name: nv-triton-client securityContext: diff --git a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh index b960b8ff94..9e2c400146 100755 --- a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh +++ b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh @@ -27,9 +27,9 @@ export REGISTRY=gcr.io/$(gcloud config get-value project | tr ':' '/') export APP_NAME=tritonserver -export MAJOR_VERSION=2.61 -export MINOR_VERSION=2.61.0 -export NGC_VERSION=25.09-py3 +export MAJOR_VERSION=2.62 +export MINOR_VERSION=2.62.0 +export NGC_VERSION=25.10-py3 docker pull nvcr.io/nvidia/$APP_NAME:$NGC_VERSION diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml index e86c2b6a21..87822f7953 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml @@ -28,4 +28,4 @@ apiVersion: v1 appVersion: "2.60" description: Triton Inference Server name: triton-inference-server -version: 2.61.0 +version: 2.62.0 diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml index 9f87933228..3d8a3d14a2 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml @@ -31,14 +31,14 @@ maxReplicaCount: 3 tritonProtocol: HTTP # HPA GPU utilization autoscaling target HPATargetAverageValue: 85 -modelRepositoryPath: gs://triton_sample_models/25.09 -publishedVersion: '2.61.0' +modelRepositoryPath: gs://triton_sample_models/25.10 +publishedVersion: '2.62.0' gcpMarketplace: true image: registry: gcr.io repository: nvidia-ngc-public/tritonserver - tag: 25.09-py3 + tag: 25.10-py3 pullPolicy: IfNotPresent # modify the model repository here to match your GCP storage bucket numGpus: 1 diff --git a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml index 107c3ce161..75af2cf0b7 100644 --- a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.61.0' + publishedVersion: '2.62.0' publishedVersionMetadata: releaseNote: >- Initial release. diff --git a/deploy/gke-marketplace-app/server-deployer/schema.yaml b/deploy/gke-marketplace-app/server-deployer/schema.yaml index 4bc5fd29e6..1e61d21839 100644 --- a/deploy/gke-marketplace-app/server-deployer/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/schema.yaml @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.61.0' + publishedVersion: '2.62.0' publishedVersionMetadata: releaseNote: >- Initial release. @@ -89,7 +89,7 @@ properties: modelRepositoryPath: type: string title: Bucket where models are stored. Please make sure the user/service account to create the GKE app has permission to this GCS bucket. Read Triton documentation on configs and formatting details, supporting TensorRT, TensorFlow, Pytorch, Onnx ... etc. - default: gs://triton_sample_models/25.09 + default: gs://triton_sample_models/25.10 image.ldPreloadPath: type: string title: Leave this empty by default. Triton allows users to create custom layers for backend such as TensorRT plugin, the compiled shared library must be provided via LD_PRELOAD environment variable. diff --git a/deploy/gke-marketplace-app/trt-engine/README.md b/deploy/gke-marketplace-app/trt-engine/README.md index f2118a1bc3..eb84e23fda 100644 --- a/deploy/gke-marketplace-app/trt-engine/README.md +++ b/deploy/gke-marketplace-app/trt-engine/README.md @@ -33,7 +33,7 @@ ``` docker run --gpus all -it --network host \ --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \ - -v ~:/scripts nvcr.io/nvidia/tensorrt:25.09-py3 + -v ~:/scripts nvcr.io/nvidia/tensorrt:25.10-py3 pip install onnx six torch tf2onnx tensorflow @@ -57,7 +57,7 @@ mkdir -p engines python3 builder.py -m models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/model.ckpt -o engines/bert_large_int8_bs1_s128.engine -b 1 -s 128 -c models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/ -v models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/vocab.txt --int8 --fp16 --strict --calib-num 1 -iln -imh -gsutil cp bert_large_int8_bs1_s128.engine gs://triton_sample_models/25.09/bert/1/model.plan +gsutil cp bert_large_int8_bs1_s128.engine gs://triton_sample_models/25.10/bert/1/model.plan ``` -For each Triton upgrade, container version used to generate the model, and the model path in GCS `gs://triton_sample_models/25.09/` should be updated accordingly with the correct version. +For each Triton upgrade, container version used to generate the model, and the model path in GCS `gs://triton_sample_models/25.10/` should be updated accordingly with the correct version. diff --git a/deploy/k8s-onprem/values.yaml b/deploy/k8s-onprem/values.yaml index 66f17e7c03..322ee46de2 100644 --- a/deploy/k8s-onprem/values.yaml +++ b/deploy/k8s-onprem/values.yaml @@ -30,7 +30,7 @@ tags: openshift: false image: - imageName: nvcr.io/nvidia/tritonserver:25.09-py3 + imageName: nvcr.io/nvidia/tritonserver:25.10-py3 pullPolicy: IfNotPresent modelRepositoryServer: < Replace with the IP Address of your file server > modelRepositoryPath: /srv/models diff --git a/deploy/oci/values.yaml b/deploy/oci/values.yaml index be4929a0f1..ba42839e92 100644 --- a/deploy/oci/values.yaml +++ b/deploy/oci/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.09-py3 + imageName: nvcr.io/nvidia/tritonserver:25.10-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://https://.compat.objectstorage..oraclecloud.com:443/triton-inference-server-repository numGpus: 1 diff --git a/docs/introduction/release_notes.md b/docs/introduction/release_notes.md index 31fa2784c9..156ac4a7af 100644 --- a/docs/introduction/release_notes.md +++ b/docs/introduction/release_notes.md @@ -25,9 +25,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> -# [Triton Inference Server Release 25.09](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-25-09.html#rel-25-09) +# [Triton Inference Server Release 25.10](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-25-10.html#rel-25-10) -The Triton Inference Server container image, release 25.09, is available +The Triton Inference Server container image, release 25.10, is available on [NGC](https://ngc.nvidia.com/catalog/containers/nvidia:tritonserver) and is open source on [GitHub](https://github.com/triton-inference-server/server). Release notes can diff --git a/python/openai/README.md b/python/openai/README.md index 5dda869bab..572373390f 100644 --- a/python/openai/README.md +++ b/python/openai/README.md @@ -51,7 +51,7 @@ docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ - nvcr.io/nvidia/tritonserver:25.09-vllm-python-py3 + nvcr.io/nvidia/tritonserver:25.10-vllm-python-py3 ``` 2. Launch the OpenAI-compatible Triton Inference Server: diff --git a/qa/common/gen_jetson_trt_models b/qa/common/gen_jetson_trt_models index 9b652a6977..693d233f91 100755 --- a/qa/common/gen_jetson_trt_models +++ b/qa/common/gen_jetson_trt_models @@ -34,7 +34,7 @@ # Make all generated files accessible outside of container umask 0000 # Set the version of the models -TRITON_VERSION=${TRITON_VERSION:=25.09} +TRITON_VERSION=${TRITON_VERSION:=25.10} # Set the CUDA device to use CUDA_DEVICE=${RUNNER_ID:=0} # Set TensorRT image diff --git a/qa/common/gen_qa_custom_ops b/qa/common/gen_qa_custom_ops index 1d7f6db05a..a196fb3f8e 100755 --- a/qa/common/gen_qa_custom_ops +++ b/qa/common/gen_qa_custom_ops @@ -37,7 +37,7 @@ ## ############################################################################ -TRITON_VERSION=${TRITON_VERSION:=25.09} +TRITON_VERSION=${TRITON_VERSION:=25.10} NVIDIA_UPSTREAM_VERSION=${NVIDIA_UPSTREAM_VERSION:=$TRITON_VERSION} PYTORCH_IMAGE=${PYTORCH_IMAGE:=nvcr.io/nvidia/pytorch:$NVIDIA_UPSTREAM_VERSION-py3} UBUNTU_IMAGE=${UBUNTU_IMAGE:=ubuntu:24.04} diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 9ac88218e5..753f6a672d 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -48,7 +48,7 @@ ## ############################################################################ -TRITON_VERSION=${TRITON_VERSION:=25.09} +TRITON_VERSION=${TRITON_VERSION:=25.10} # ONNX. Use ONNX_OPSET 0 to use the default for ONNX version ONNX_VERSION=1.16.1 From 36f845ed5669b8064b01a60329d10bb62552f4fa Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Thu, 16 Oct 2025 08:50:10 -0700 Subject: [PATCH 02/10] Add libnvshmem installation (#8460) --- build.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 2a0fbb9ed2..19db09ae15 100755 --- a/build.py +++ b/build.py @@ -78,7 +78,7 @@ "ort_openvino_version": "2025.3.0", "standalone_openvino_version": "2025.3.0", "dcgm_version": "4.4.0-1", - "vllm_version": "0.10.1.1", + "vllm_version": "0.10.2", "rhel_py_version": "3.12.3", } @@ -660,6 +660,10 @@ def pytorch_cmake_args(images): cargs.append( cmake_backend_enable("pytorch", "TRITON_ENABLE_NVTX", FLAGS.enable_nvtx) ) + if target_platform() == "igpu": + cargs.append( + cmake_backend_enable("pytorch", "TRITON_PYTORCH_NVSHMEM", False) + ) return cargs @@ -1514,6 +1518,22 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach ENV PYTHONPATH=/opt/tritonserver/backends/dali/wheel/dali:$PYTHONPATH """ + if target_platform() not in ["igpu", "windows", "rhel"]: + repo_arch = "sbsa" if target_machine == "aarch64" else "x86_64" + df += f""" +RUN curl -o /tmp/cuda-keyring.deb \\ + https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/{repo_arch}/cuda-keyring_1.1-1_all.deb \\ + && apt install /tmp/cuda-keyring.deb \\ + && rm /tmp/cuda-keyring.deb \\ + && apt update -qq \\ + && apt install --yes --no-install-recommends libnvshmem3-cuda-13 \\ + && rm -rf /var/lib/apt/lists/* \\ + && dpkg -L libnvshmem3-cuda-13 | grep libnvshmem_host.so | sed -e 's/libnvshmem_host.*//g' | sort -u > /etc/ld.so.conf.d/libnvshmem3-cuda-13.conf \\ + && ldconfig +""".format( + repo_arch=repo_arch + ) + df += """ WORKDIR /opt/tritonserver RUN rm -fr /opt/tritonserver/* From 3fb0c14efa1b3729c7b0072ba647a996fb9be0d7 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Mon, 20 Oct 2025 11:30:29 -0700 Subject: [PATCH 03/10] fix: Add environment variable defaults to BACKENDS (#8465) Co-authored-by: Dmitry Tokarev --- qa/L0_shared_memory/shared_memory_test.py | 8 +++++++- qa/L0_shared_memory/test.sh | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/qa/L0_shared_memory/shared_memory_test.py b/qa/L0_shared_memory/shared_memory_test.py index 6f8ba94337..5cb23d9869 100755 --- a/qa/L0_shared_memory/shared_memory_test.py +++ b/qa/L0_shared_memory/shared_memory_test.py @@ -301,7 +301,13 @@ def test_too_big_shm(self): def test_large_shm_register_offset(self): # Test for out of bounds read vulnerability when registering system shared memory with large offset - for platform in ["python", "onnx", "libtorch", "plan", "openvino"]: + + platforms = ( + ["python", "onnx", "libtorch", "plan", "openvino"] + if os.environ.get("BACKENDS") is None + else os.environ.get("BACKENDS").split() + ) + for platform in platforms: model_name = f"{platform}_int32_int32_int32" # Test for large offset diff --git a/qa/L0_shared_memory/test.sh b/qa/L0_shared_memory/test.sh index 3f5758bf36..e33c9e3756 100755 --- a/qa/L0_shared_memory/test.sh +++ b/qa/L0_shared_memory/test.sh @@ -160,7 +160,7 @@ done # Test large system shared memory offset rm -rf models/* # prepare add_sub model of various backends -BACKENDS="python onnx libtorch plan openvino" +BACKENDS=${BACKENDS:-"python onnx libtorch plan openvino"} for backend in ${BACKENDS} ; do model="${backend}_int32_int32_int32" model_dir="models/${model}" From 31c44cf9a7618e2445b7b791f8f79894691f0d6f Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Mon, 27 Oct 2025 10:23:25 -0700 Subject: [PATCH 04/10] Removing unused values for TensorRT-LLM container build (#8472) --- build.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/build.py b/build.py index 19db09ae15..a901fdad09 100755 --- a/build.py +++ b/build.py @@ -1283,10 +1283,9 @@ def create_dockerfile_linux( pip3 uninstall -y setuptools ENV LD_LIBRARY_PATH=/usr/local/tensorrt/lib/:/opt/tritonserver/backends/tensorrtllm:$LD_LIBRARY_PATH -# There are some ucc issues when spawning mpi processes with ompi v4.1.7a1. -# Downgrade to ompi v4.1.5rc2 to avoid the issue. -RUN rm -fr /opt/hpcx/ompi -COPY --from=nvcr.io/nvidia/tritonserver:24.02-py3-min /opt/hpcx/ompi /opt/hpcx/ompi +RUN apt-get update && apt-get install -y --no-install-recommends \ + libprotobuf-dev \ + && rm -rf /var/lib/apt/lists/* """ with open(os.path.join(ddir, dockerfile_name), "w") as dfile: dfile.write(df) @@ -1465,14 +1464,6 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach virtualenv \\ && rm -rf /var/lib/apt/lists/* """ - if "tensorrtllm" in backends: - df += """ -# Updating the openssh-client to fix for the CVE-2024-6387. This can be removed when trtllm uses a later CUDA container(12.5 or later) -RUN apt-get update \\ - && apt-get install -y --no-install-recommends \\ - openssh-client \\ - && rm -rf /var/lib/apt/lists/* - """ if "vllm" in backends: df += f""" From d6c1c2b60adb389514e13aae72c77b26666253d9 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:23:28 -0700 Subject: [PATCH 05/10] rollback: Reverting TensorRT-LLM Backend changes. (#8478) --- build.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build.py b/build.py index a901fdad09..9f3a7e7eda 100755 --- a/build.py +++ b/build.py @@ -1282,10 +1282,6 @@ def create_dockerfile_linux( pip3 install --no-cache-dir grpcio-tools==1.64.0 && \\ pip3 uninstall -y setuptools ENV LD_LIBRARY_PATH=/usr/local/tensorrt/lib/:/opt/tritonserver/backends/tensorrtllm:$LD_LIBRARY_PATH - -RUN apt-get update && apt-get install -y --no-install-recommends \ - libprotobuf-dev \ - && rm -rf /var/lib/apt/lists/* """ with open(os.path.join(ddir, dockerfile_name), "w") as dfile: dfile.write(df) From 9f54813a6c253413c19abb792a1f356da3e6ecf6 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Wed, 29 Oct 2025 22:43:14 +0530 Subject: [PATCH 06/10] ci: Fix TensorRT engine build error for vision models (#8479) --- qa/common/gen_qa_image_models.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/qa/common/gen_qa_image_models.py b/qa/common/gen_qa_image_models.py index a32ba92f1b..970d08662c 100755 --- a/qa/common/gen_qa_image_models.py +++ b/qa/common/gen_qa_image_models.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -99,7 +99,8 @@ def export_vgg19(models_dir, model_name="model.onnx"): model.eval() dummy_input = torch.randn(1, 3, 224, 224) # (batch, channels, height, width) - # Export the model to ONNX format + # Use legacy TorchScript-based ONNX export + # TODO: Update to use new torch.export-based ONNX exporter (default dynamo=True) torch.onnx.export( model, dummy_input, @@ -107,6 +108,7 @@ def export_vgg19(models_dir, model_name="model.onnx"): input_names=["input"], output_names=["output"], dynamic_axes={"input": {0: "batch_size"}, "output": {0: "batch_size"}}, + dynamo=False, ) print(f"VGG19 model exported to: {model_path}") @@ -129,7 +131,8 @@ def export_resnet152(models_dir, model_name="model.onnx"): model.eval() dummy_input = torch.randn(1, 3, 224, 224) # (batch, channels, height, width) - # Export the model to ONNX format + # Use legacy TorchScript-based ONNX export + # TODO: Update to use new torch.export-based ONNX exporter (default dynamo=True) torch.onnx.export( model, dummy_input, @@ -137,6 +140,7 @@ def export_resnet152(models_dir, model_name="model.onnx"): input_names=["input"], output_names=["output"], dynamic_axes={"input": {0: "batch_size"}, "output": {0: "batch_size"}}, + dynamo=False, ) print(f"ResNet-152 model exported to: {model_path}") @@ -159,7 +163,8 @@ def export_resnet50(models_dir, model_name="model.onnx"): model.eval() dummy_input = torch.randn(1, 3, 224, 224) # (batch, channels, height, width) - # Export the model to ONNX format + # Use legacy TorchScript-based ONNX export + # TODO: Update to use new torch.export-based ONNX exporter (default dynamo=True) torch.onnx.export( model, dummy_input, @@ -167,6 +172,7 @@ def export_resnet50(models_dir, model_name="model.onnx"): input_names=["input"], output_names=["output"], dynamic_axes={"input": {0: "batch_size"}, "output": {0: "batch_size"}}, + dynamo=False, ) print(f"ResNet-50 model exported to: {model_path}") From 37459d65c1da80896056a0b0a125bde3a4caa2f5 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Wed, 29 Oct 2025 10:45:15 -0700 Subject: [PATCH 07/10] Remove additional installation (#8480) --- build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index 9f3a7e7eda..0f42ab0589 100755 --- a/build.py +++ b/build.py @@ -1505,7 +1505,10 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach ENV PYTHONPATH=/opt/tritonserver/backends/dali/wheel/dali:$PYTHONPATH """ - if target_platform() not in ["igpu", "windows", "rhel"]: + if ( + target_platform() not in ["igpu", "windows", "rhel"] + and "tensorrtllm" not in backends + ): repo_arch = "sbsa" if target_machine == "aarch64" else "x86_64" df += f""" RUN curl -o /tmp/cuda-keyring.deb \\ From 053faea896d46ca60a53993f2cef510a68ab8295 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Wed, 29 Oct 2025 15:52:06 -0700 Subject: [PATCH 08/10] docs: Update release related versions and values (#8481) --- docs/introduction/compatibility.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/introduction/compatibility.md b/docs/introduction/compatibility.md index 1e1dbb5e3c..c90899361b 100644 --- a/docs/introduction/compatibility.md +++ b/docs/introduction/compatibility.md @@ -38,6 +38,7 @@ | Triton release version | NGC Tag | Python version | Torch version | TensorRT version | TensorRT-LLM version | CUDA version | CUDA Driver version | Size | | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 25.10 | nvcr.io/nvidia/tritonserver:25.10-trtllm-python-py3 | Python 3.12.3 | 2.8.0a0+5228986c39.nv25.6 | 10.11.0.33 | 1.0.0 | 12.9.1.010 | 575.57.08 | 16.21 GB | | 25.09 | nvcr.io/nvidia/tritonserver:25.09-trtllm-python-py3 | Python 3.12.3 | 2.8.0a0+5228986c39.nv25.6 | 10.11.0.33 | 1.0.0 | 12.9.1.010 | 575.57.08 | 16.25 GB | | 25.08 | nvcr.io/nvidia/tritonserver:25.08-trtllm-python-py3 | Python 3.12.3 | 2.8.0a0+5228986c39.nv25.5 | 10.11.0.33 | 0.21.0 | 12.9.0.043 | 575.51.03 | 20.49 GB | | 25.07 | nvcr.io/nvidia/tritonserver:25.07-trtllm-python-py3 | Python 3.12.3 | 2.7.0a0+79aa17489c.nv25.4 | 10.10.0.31 | 0.20.0 | 12.9.0.036 | 575.51.03 | 18.3G | @@ -61,6 +62,7 @@ | Triton release version | NGC Tag | Python version | vLLM version | CUDA version | CUDA Driver version | Size | | --- | --- | --- | --- | --- | --- | --- | +| 25.10 | nvcr.io/nvidia/tritonserver:25.10-vllm-python-py3 | Python 3.12.3 | 0.10.2+9dd9ca32.nv25.10.cu130 | 13.0.2.006 | 580.95.05 | 8.34G | | 25.09 | nvcr.io/nvidia/tritonserver:25.09-vllm-python-py3 | Python 3.12.3 | 0.10.1.1+381074ae.nv25.9.cu130 | 13.0.1.012 | 580.82.07 | 7.78G | | 25.08 | nvcr.io/nvidia/tritonserver:25.08-vllm-python-py3 | Python 3.12.3 | 0.9.2+4ef1e343.nv25.8.post1.cu130 | 13.0.1.012 | 580.82.07 | 8.1G | | 25.07 | nvcr.io/nvidia/tritonserver:25.07-vllm-python-py3 | Python 3.12.3 | 0.9.0rc1+1958ee56.nv25.6.cu129 | 12.9.0.043 | 575.51.03 | 10G | @@ -84,6 +86,7 @@ | Triton release version | ONNX Runtime | | --- | --- | +| 25.10 | 1.23.1 | | 25.09 | 1.23.0 | | 25.08 | 1.23.0+1d1712fdaf | | 25.07 | 1.22.0 | From 81540e4a7bd790135e413b041db183f75eac79b7 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Wed, 29 Oct 2025 21:53:00 -0700 Subject: [PATCH 09/10] security: Update package version due to CVE-2025-62727 (#8482) --- python/openai/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/openai/requirements.txt b/python/openai/requirements.txt index 608f3b5ac3..e523449265 100644 --- a/python/openai/requirements.txt +++ b/python/openai/requirements.txt @@ -33,4 +33,4 @@ openai==1.107.3 partial-json-parser # used for parsing partial JSON outputs # Minimum starlette version needed to address CVE: # https://github.com/advisories/GHSA-f96h-pmfr-66vw -starlette>=0.40.0 +starlette>=0.49.1 From 00fbe53bb3c658ab7c23a8a132859604a65ec31a Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Thu, 30 Oct 2025 09:24:29 -0700 Subject: [PATCH 10/10] Revert "security: Update package version due to CVE-2025-62727" (#8484) --- python/openai/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/openai/requirements.txt b/python/openai/requirements.txt index e523449265..608f3b5ac3 100644 --- a/python/openai/requirements.txt +++ b/python/openai/requirements.txt @@ -33,4 +33,4 @@ openai==1.107.3 partial-json-parser # used for parsing partial JSON outputs # Minimum starlette version needed to address CVE: # https://github.com/advisories/GHSA-f96h-pmfr-66vw -starlette>=0.49.1 +starlette>=0.40.0