diff --git a/Dockerfile.sdk b/Dockerfile.sdk index 169bd73a24..a1e75e96d2 100644 --- a/Dockerfile.sdk +++ b/Dockerfile.sdk @@ -29,7 +29,7 @@ # # Base image on the minimum Triton container -ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.10-py3-min +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.11-py3-min ARG TRITON_CLIENT_REPO_SUBDIR=clientrepo ARG TRITON_REPO_ORGANIZATION=http://github.com/triton-inference-server diff --git a/README.md b/README.md index cdd14ee53e..9035fa2bc4 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ >[!WARNING] >You are currently on the `main` branch which tracks under-development progress ->towards the next release. The current release is version [2.62.0](https://github.com/triton-inference-server/server/releases/latest) ->and corresponds to the 25.10 container release on NVIDIA GPU Cloud (NGC). +>towards the next release. The current release is version [2.63.0](https://github.com/triton-inference-server/server/releases/latest) +>and corresponds to the 25.11 container release on NVIDIA GPU Cloud (NGC). # Triton Inference Server @@ -90,16 +90,16 @@ Inference Server with the ```bash # Step 1: Create the example model repository -git clone -b r25.10 https://github.com/triton-inference-server/server.git +git clone -b r25.11 https://github.com/triton-inference-server/server.git cd server/docs/examples ./fetch_models.sh # Step 2: Launch triton from the NGC Triton container -docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:25.10-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx +docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:25.11-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx # Step 3: Sending an Inference Request # In a separate console, launch the image_client example from the NGC Triton SDK container -docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:25.10-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg +docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:25.11-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg # Inference should return the following Image '/workspace/images/mug.jpg': diff --git a/TRITON_VERSION b/TRITON_VERSION index 867f66f09b..952e951595 100644 --- a/TRITON_VERSION +++ b/TRITON_VERSION @@ -1 +1 @@ -2.63.0dev +2.64.0dev diff --git a/build.py b/build.py index 950bf48968..7ecf9b9632 100755 --- a/build.py +++ b/build.py @@ -71,14 +71,14 @@ # DEFAULT_TRITON_VERSION_MAP = { - "release_version": "2.63.0dev", + "release_version": "2.64.0dev", "triton_container_version": "25.11dev", - "upstream_container_version": "25.10", + "upstream_container_version": "25.11", "ort_version": "1.23.2", "ort_openvino_version": "2025.3.0", "standalone_openvino_version": "2025.3.0", "dcgm_version": "4.4.0-1", - "vllm_version": "0.10.2", + "vllm_version": "0.11.0", "rhel_py_version": "3.12.3", } @@ -1500,6 +1500,15 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach ARG PYVER=3.12 ENV LD_LIBRARY_PATH /usr/local/lib:/usr/local/lib/python${{PYVER}}/dist-packages/torch/lib:${{LD_LIBRARY_PATH}} +""" + if "tensorrtllm" in backends or "vllm" in backends: + df += """ +ENV TRITON_CUDACRT_PATH=/usr/local/cuda/include \\ + TRITON_CUDART_PATH=/usr/local/cuda/include \\ + TRITON_CUOBJDUMP_PATH=/usr/local/cuda/bin/cuobjdump \\ + TRITON_CUPTI_PATH=/usr/local/cuda/include \\ + TRITON_NVDISASM_PATH=/usr/local/cuda/bin/nvdisasm \\ + TRITON_PTXAS_PATH=/usr/local/cuda/bin/ptxas """ if "dali" in backends: diff --git a/deploy/aws/values.yaml b/deploy/aws/values.yaml index 7c782b1ed9..cc8748f0d1 100644 --- a/deploy/aws/values.yaml +++ b/deploy/aws/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.10-py3 + imageName: nvcr.io/nvidia/tritonserver:25.11-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/fleetcommand/Chart.yaml b/deploy/fleetcommand/Chart.yaml index 2cbc1cbb34..b0e42586ca 100644 --- a/deploy/fleetcommand/Chart.yaml +++ b/deploy/fleetcommand/Chart.yaml @@ -26,7 +26,7 @@ apiVersion: v1 # appVersion is the Triton version; update when changing release -appVersion: 2.62.0" +appVersion: 2.63.0" description: Triton Inference Server (Fleet Command) name: triton-inference-server # version is the Chart version; update when changing anything in the chart diff --git a/deploy/fleetcommand/values.yaml b/deploy/fleetcommand/values.yaml index 830407c9fe..d36b0f5d13 100644 --- a/deploy/fleetcommand/values.yaml +++ b/deploy/fleetcommand/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.10-py3 + imageName: nvcr.io/nvidia/tritonserver:25.11-py3 pullPolicy: IfNotPresent numGpus: 1 serverCommand: tritonserver @@ -47,13 +47,13 @@ image: # # To set model control mode, uncomment and configure below # TODO: Fix the following url, it is invalid - # See https://github.com/triton-inference-server/server/blob/r25.10/docs/user_guide/model_management.md + # See https://github.com/triton-inference-server/server/blob/r25.11/docs/user_guide/model_management.md # for more details #- --model-control-mode=explicit|poll|none # # Additional server args # - # see https://github.com/triton-inference-server/server/blob/r25.10/README.md + # see https://github.com/triton-inference-server/server/blob/r25.11/README.md # for more details service: diff --git a/deploy/gcp/values.yaml b/deploy/gcp/values.yaml index b66268bce2..53d960feb8 100644 --- a/deploy/gcp/values.yaml +++ b/deploy/gcp/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.10-py3 + imageName: nvcr.io/nvidia/tritonserver:25.11-py3 pullPolicy: IfNotPresent modelRepositoryPath: gs://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml index d65a50862f..47fe79cec1 100644 --- a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml +++ b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml @@ -33,7 +33,7 @@ metadata: namespace: default spec: containers: - - image: nvcr.io/nvidia/tritonserver:25.10-py3-sdk + - image: nvcr.io/nvidia/tritonserver:25.11-py3-sdk imagePullPolicy: Always name: nv-triton-client securityContext: diff --git a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh index 9e2c400146..79e72774f4 100755 --- a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh +++ b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh @@ -27,9 +27,9 @@ export REGISTRY=gcr.io/$(gcloud config get-value project | tr ':' '/') export APP_NAME=tritonserver -export MAJOR_VERSION=2.62 -export MINOR_VERSION=2.62.0 -export NGC_VERSION=25.10-py3 +export MAJOR_VERSION=2.63 +export MINOR_VERSION=2.63.0 +export NGC_VERSION=25.11-py3 docker pull nvcr.io/nvidia/$APP_NAME:$NGC_VERSION diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml index 87822f7953..3063bd0270 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. apiVersion: v1 -appVersion: "2.60" +appVersion: "2.63" description: Triton Inference Server name: triton-inference-server -version: 2.62.0 +version: 2.63.0 diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml index 3d8a3d14a2..2aa7fc63d1 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml @@ -31,14 +31,14 @@ maxReplicaCount: 3 tritonProtocol: HTTP # HPA GPU utilization autoscaling target HPATargetAverageValue: 85 -modelRepositoryPath: gs://triton_sample_models/25.10 -publishedVersion: '2.62.0' +modelRepositoryPath: gs://triton_sample_models/25.11 +publishedVersion: '2.63.0' gcpMarketplace: true image: registry: gcr.io repository: nvidia-ngc-public/tritonserver - tag: 25.10-py3 + tag: 25.11-py3 pullPolicy: IfNotPresent # modify the model repository here to match your GCP storage bucket numGpus: 1 diff --git a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml index 75af2cf0b7..1f19f2e3b0 100644 --- a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.62.0' + publishedVersion: '2.63.0' publishedVersionMetadata: releaseNote: >- Initial release. diff --git a/deploy/gke-marketplace-app/server-deployer/schema.yaml b/deploy/gke-marketplace-app/server-deployer/schema.yaml index 1e61d21839..18bfeb9005 100644 --- a/deploy/gke-marketplace-app/server-deployer/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/schema.yaml @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.62.0' + publishedVersion: '2.63.0' publishedVersionMetadata: releaseNote: >- Initial release. @@ -89,7 +89,7 @@ properties: modelRepositoryPath: type: string title: Bucket where models are stored. Please make sure the user/service account to create the GKE app has permission to this GCS bucket. Read Triton documentation on configs and formatting details, supporting TensorRT, TensorFlow, Pytorch, Onnx ... etc. - default: gs://triton_sample_models/25.10 + default: gs://triton_sample_models/25.11 image.ldPreloadPath: type: string title: Leave this empty by default. Triton allows users to create custom layers for backend such as TensorRT plugin, the compiled shared library must be provided via LD_PRELOAD environment variable. diff --git a/deploy/gke-marketplace-app/trt-engine/README.md b/deploy/gke-marketplace-app/trt-engine/README.md index eb84e23fda..e7a3af632b 100644 --- a/deploy/gke-marketplace-app/trt-engine/README.md +++ b/deploy/gke-marketplace-app/trt-engine/README.md @@ -33,7 +33,7 @@ ``` docker run --gpus all -it --network host \ --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \ - -v ~:/scripts nvcr.io/nvidia/tensorrt:25.10-py3 + -v ~:/scripts nvcr.io/nvidia/tensorrt:25.11-py3 pip install onnx six torch tf2onnx tensorflow @@ -57,7 +57,7 @@ mkdir -p engines python3 builder.py -m models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/model.ckpt -o engines/bert_large_int8_bs1_s128.engine -b 1 -s 128 -c models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/ -v models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/vocab.txt --int8 --fp16 --strict --calib-num 1 -iln -imh -gsutil cp bert_large_int8_bs1_s128.engine gs://triton_sample_models/25.10/bert/1/model.plan +gsutil cp bert_large_int8_bs1_s128.engine gs://triton_sample_models/25.11/bert/1/model.plan ``` -For each Triton upgrade, container version used to generate the model, and the model path in GCS `gs://triton_sample_models/25.10/` should be updated accordingly with the correct version. +For each Triton upgrade, container version used to generate the model, and the model path in GCS `gs://triton_sample_models/25.11/` should be updated accordingly with the correct version. diff --git a/deploy/k8s-onprem/values.yaml b/deploy/k8s-onprem/values.yaml index 322ee46de2..96073f0014 100644 --- a/deploy/k8s-onprem/values.yaml +++ b/deploy/k8s-onprem/values.yaml @@ -30,7 +30,7 @@ tags: openshift: false image: - imageName: nvcr.io/nvidia/tritonserver:25.10-py3 + imageName: nvcr.io/nvidia/tritonserver:25.11-py3 pullPolicy: IfNotPresent modelRepositoryServer: < Replace with the IP Address of your file server > modelRepositoryPath: /srv/models diff --git a/deploy/oci/values.yaml b/deploy/oci/values.yaml index ba42839e92..a0910fc1e2 100644 --- a/deploy/oci/values.yaml +++ b/deploy/oci/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.10-py3 + imageName: nvcr.io/nvidia/tritonserver:25.11-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://https://.compat.objectstorage..oraclecloud.com:443/triton-inference-server-repository numGpus: 1 diff --git a/docs/Dockerfile.docs b/docs/Dockerfile.docs index cd9e712dcd..b3f078baaa 100644 --- a/docs/Dockerfile.docs +++ b/docs/Dockerfile.docs @@ -24,7 +24,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM ubuntu:22.04 +FROM ubuntu:24.04 # various documentation dependencies RUN apt-get update -q=2 \ @@ -53,6 +53,7 @@ RUN wget https://github.com/pseudomuto/protoc-gen-doc/releases/download/v1.3.2/p && rm /tmp/protoc-gen-doc.tar.gz # install sphinx et al +ENV PIP_BREAK_SYSTEM_PACKAGES=1 RUN pip3 install \ ablog \ attrs \ @@ -75,8 +76,13 @@ RUN pip3 install \ sphinxcontrib-bibtex RUN pip3 install \ - --index-url https://urm.nvidia.com/artifactory/api/pypi/ct-omniverse-pypi/simple/ \ - nvidia-sphinx-theme + --extra-index-url https://pypi.nvidia.com \ + nvidia-sphinx-theme \ + sphinx==7.4.7 + +RUN curl -fL https://install-cli.jfrog.io | sh + +RUN git config --global --add safe.directory "*" # Set visitor script to be included on every HTML page ENV VISITS_COUNTING_SCRIPT="//assets.adobedtm.com/b92787824f2e0e9b68dc2e993f9bd995339fe417/satelliteLib-7ba51e58dc61bcb0e9311aadd02a0108ab24cc6c.js" diff --git a/docs/conf.py b/docs/conf.py index 0b44f7c8b2..387e0f3c37 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -35,9 +35,12 @@ # -- Path setup -------------------------------------------------------------- import json +import logging import os import re +import subprocess from datetime import date +from logging.handlers import RotatingFileHandler # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -61,6 +64,45 @@ # at the end of the file. # current_dir = os.getcwd() # os.chdir("docs") +# -- Setup logger ------------------------------------------------------------ + + +def setup_logger(name, log_file, level=logging.INFO, max_bytes=1048576, backup_count=5): + logger = logging.getLogger(name) + logger.setLevel(level) + + # Prevent adding multiple handlers if the function is called multiple times + if not logger.handlers: + # Create handlers + file_handler = RotatingFileHandler( + log_file, maxBytes=max_bytes, backupCount=backup_count + ) + console_handler = logging.StreamHandler() + + # Set the logging level for handlers + file_handler.setLevel(level) + console_handler.setLevel(level) + + # Create a logging format + BLUE = "\033[94m" + RESET = "\033[0m" + formatter = logging.Formatter( + f"{BLUE}%(asctime)s - %(name)s - %(levelname)s - {RESET}%(message)s" + ) + file_handler.setFormatter(formatter) + console_handler.setFormatter(formatter) + + # Add handlers to the logger + logger.addHandler(file_handler) + logger.addHandler(console_handler) + return logger + + +logger = setup_logger( + os.path.basename(__file__), + os.environ.get("TRITON_SERVER_DOCS_LOG_FILE", "/tmp/docs.log"), +) +logger.info(f"Defined logger for {os.path.basename(__file__)}") # -- Project information ----------------------------------------------------- @@ -70,14 +112,19 @@ # Get the version of Triton this is building. version_long = "0.0.0" +logger.info(f"Getting version from ../TRITON_VERSION") with open("../TRITON_VERSION") as f: version_long = f.readline() version_long = version_long.strip() + logger.info(f"Version: {version_long}") + version_short = re.match(r"^[\d]+\.[\d]+\.[\d]+", version_long).group(0) +logger.info(f"Version short: {version_short}") version_short_split = version_short.split(".") +logger.info(f"Version short split: {version_short_split}") one_before = f"{version_short_split[0]}.{int(version_short_split[1]) - 1}.{version_short_split[2]}" - +logger.info(f"One before: {one_before}") # maintain left-side bar toctrees in `contents` file # so it doesn't show up needlessly in the index page @@ -180,7 +227,7 @@ "json_url": "https://docs.nvidia.com/deeplearning/triton-inference-server/user-guide/docs/_static/switcher.json", "version_match": one_before if "dev" in version_long else version_short, }, - "navbar_start": ["navbar-logo", "version-switcher"], + # "navbar_start": ["navbar-logo", "version-switcher"], "primary_sidebar_end": [], } @@ -194,6 +241,8 @@ } ) +logger.info(f"html_theme_options: {html_theme_options}") + deploy_ngc_org = "nvidia" deploy_ngc_team = "triton" myst_substitutions = { @@ -203,6 +252,8 @@ else deploy_ngc_org, } +logger.info(f"myst_substitutions: {myst_substitutions}") + def ultimateReplace(app, docname, source): result = source[0] @@ -219,6 +270,7 @@ def ultimateReplace(app, docname, source): if deploy_ngc_team else deploy_ngc_org, } +logger.info(f"ultimate_replacements: {ultimate_replacements}") # bibtex_bibfiles = ["references.bib"] # To test that style looks good with common bibtex config @@ -233,25 +285,24 @@ def ultimateReplace(app, docname, source): # SETUP SWITCHER ############################### switcher_path = os.path.join(html_static_path[0], "switcher.json") +logger.info(f"switcher_path: {switcher_path}") versions = [] -# Triton 2 releases -correction = -1 if "dev" in version_long else 0 -upper_bound = version_short.split(".")[1] -for i in range(2, int(version_short.split(".")[1]) + correction): - versions.append((f"2.{i}.0", f"triton-inference-server-2{i}0")) -# Triton 1 releases -for i in range(0, 15): - versions.append((f"1.{i}.0", f"tensorrt_inference_server_1{i}0")) - -# Triton Beta Releases -for i in range(1, 11): - versions.append((f"0.{i}.0_beta", f"inference_server_0{i}0_beta")) +# Obtain Triton Server Release Tags. +tags = subprocess.run(["git", "tag", "--list", "v*"], capture_output=True, text=True) +tags_list = sorted(tags.stdout.strip().splitlines(), key=Version, reverse=True) +logger.info(f"Found source tags: {tags_list}") + +for v in tags_list: + versions.append( + ( + v.replace("v", ""), + f"triton-inference-server-{v.replace('v', '').replace('.', '')}", + ) + ) -# Patch releases -# Add here. +logger.info(f"Defined dictionary of versions: {versions}") -versions = sorted(versions, key=lambda v: Version(v[0]), reverse=True) # Build switcher data json_data = [] @@ -263,6 +314,7 @@ def ultimateReplace(app, docname, source): "url": f"https://docs.nvidia.com/deeplearning/triton-inference-server/archives/{v[1]}/user-guide/docs", } ) + if "dev" in version_long: json_data.insert( 0, @@ -284,6 +336,7 @@ def ultimateReplace(app, docname, source): # Trim to last N releases. json_data = json_data[0:12] +logger.info(f"Trimmed to last 12 release...") json_data.append( { @@ -293,19 +346,22 @@ def ultimateReplace(app, docname, source): } ) -# validate the links for i, d in enumerate(json_data): + logger.info(f"Validating link: {d['url']}") h = httplib2.Http() resp = h.request(d["url"], "HEAD") if int(resp[0]["status"]) >= 400: print(d["url"], "NOK", resp[0]["status"]) - exit(1) + # exit(1) -# Write switcher data to file +logger.info(f"Writing switcher data to file: {switcher_path}") with open(switcher_path, "w") as f: json.dump(json_data, f, ensure_ascii=False, indent=4) +logger.info("Configuration completed...") + + def setup(app): app.add_config_value("ultimate_replacements", {}, True) app.connect("source-read", ultimateReplace) diff --git a/docs/generate_docs.py b/docs/generate_docs.py index b7abe51ed0..a38b06166b 100755 --- a/docs/generate_docs.py +++ b/docs/generate_docs.py @@ -31,8 +31,8 @@ import os import re import subprocess -from collections import defaultdict from functools import partial +from logging.handlers import RotatingFileHandler # Global constants server_abspath = os.environ.get("SERVER_ABSPATH", os.getcwd()) @@ -59,208 +59,200 @@ triton_github_url_reg = re.compile( rf"{triton_repo_patn}/([^/#]+)(?:{tag_patn})?/*([^#]*)\s*(?=#|$)" ) -# relpath_patn = r"]\s*\(\s*([^)]+)\)" # Hyperlink in a .md file, excluding embedded images. hyperlink_reg = re.compile(r"((?". + Replace Triton Inference Server GitHub URLs with relative paths for: + 1. URL is a doc file (e.g., ".md" file). + 2. URL is a directory with README.md and ends with "#
". Examples: https://github.com/triton-inference-server/server/blob/main/docs/protocol#restricted-protocols https://github.com/triton-inference-server/server/blob/main/docs/protocol/extension_shared_memory.md https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#dynamic-batcher - - Keep URL in the following cases: - https://github.com/triton-inference-server/server/tree/r24.02 - https://github.com/triton-inference-server/server/blob/main/build.py - https://github.com/triton-inference-server/server/blob/main/qa - https://github.com/triton-inference-server/server/blob/main/CONTRIBUTING.md """ m = triton_github_url_reg.match(url) - # Do not replace URL if it is not a Triton GitHub file. if not m: return url target_repo_name = m.group(1) + logger.info(f"Found target repository: {target_repo_name}") target_relpath_from_target_repo = os.path.normpath(m.groups("")[1]) + logger.info( + f"Found target relative path from target repository: {target_relpath_from_target_repo}" + ) section = url[len(m.group(0)) :] + logger.info(f"Found section: {section}") valid_hashtag = section not in ["", "#"] and section.startswith("#") - if target_repo_name == "server": - target_path = os.path.join(server_abspath, target_relpath_from_target_repo) - else: - target_path = os.path.join( + target_path = ( + os.path.join(server_abspath, target_relpath_from_target_repo) + if target_repo_name == "server" + else os.path.join( server_docs_abspath, target_repo_name, target_relpath_from_target_repo ) - - # Return URL if it points to a path outside server/docs. + ) + logger.info(f"Found target path: {target_path}") + # Return URL if it points to a path outside server/docs if os.path.commonpath([server_docs_abspath, target_path]) != server_docs_abspath: return url - - if ( + logger.info( + f"Target path is under server/docs directory: {os.path.commonpath([server_docs_abspath, target_path]) == server_docs_abspath}" + ) + # Check if target is valid for conversion + is_md_file = ( os.path.isfile(target_path) and os.path.splitext(target_path)[1] == ".md" and not is_excluded(target_path) - ): - pass - elif ( + ) + logger.info(f"Target path is a valid .md file: {is_md_file}") + is_dir_with_readme = ( os.path.isdir(target_path) and os.path.isfile(os.path.join(target_path, "README.md")) and valid_hashtag and not is_excluded(os.path.join(target_path, "README.md")) - ): + ) + logger.info(f"Target path is a directory with README.md: {is_dir_with_readme}") + if is_md_file: + pass + elif is_dir_with_readme: target_path = os.path.join(target_path, "README.md") else: return url + logger.info( + f"Target path is a valid .md file or a directory with README.md: {is_md_file or is_dir_with_readme}" + ) - # The "target_path" must be a file at this line. relpath = os.path.relpath(target_path, start=os.path.dirname(src_doc_path)) + logger.info(f"Found relative path: {relpath}") return re.sub(triton_github_url_reg, relpath, url, 1) @@ -274,32 +266,26 @@ def replace_relpath_with_url(relpath, src_doc_path): Examples: ../examples/model_repository ../examples/model_repository/inception_graphdef/config.pbtxt - - Keep relpath in the following cases: - build.md - build.md#building-with-docker - #building-with-docker - ../getting_started/quickstart.md - ../protocol#restricted-protocols """ - target_path = relpath.rsplit("#")[0] + target_path = relpath.rsplit("#", 1)[0] section = relpath[len(target_path) :] valid_hashtag = section not in ["", "#"] + if relpath.startswith("#"): target_path = os.path.basename(src_doc_path) - target_path = os.path.join(os.path.dirname(src_doc_path), target_path) - target_path = os.path.normpath(target_path) + + target_path = os.path.normpath( + os.path.join(os.path.dirname(src_doc_path), target_path) + ) src_git_repo_name = get_git_repo_name(src_doc_path) - url = f"https://github.com/triton-inference-server/{src_git_repo_name}/blob/main/" - if src_git_repo_name == "server": - src_repo_abspath = server_abspath - # TODO: Assert the relative path not pointing to cloned repo, e.g. client. - # This requires more information which may be stored in a global variable. - else: - src_repo_abspath = os.path.join(server_docs_abspath, src_git_repo_name) + src_repo_abspath = ( + server_abspath + if src_git_repo_name == "server" + else os.path.join(server_docs_abspath, src_git_repo_name) + ) - # Assert target path is under the current repo directory. + # Assert target path is under the current repo directory assert os.path.commonpath([src_repo_abspath, target_path]) == src_repo_abspath target_path_from_src_repo = os.path.relpath(target_path, start=src_repo_abspath) @@ -310,9 +296,10 @@ def replace_relpath_with_url(relpath, src_doc_path): and valid_hashtag and os.path.isfile(os.path.join(target_path, "README.md")) ): - relpath = os.path.join(relpath.rsplit("#")[0], "README.md") + section + relpath = os.path.join(relpath.rsplit("#", 1)[0], "README.md") + section target_path = os.path.join(target_path, "README.md") + # Keep relpath if it's a valid .md file in docs if ( os.path.isfile(target_path) and os.path.splitext(target_path)[1] == ".md" @@ -321,53 +308,37 @@ def replace_relpath_with_url(relpath, src_doc_path): and not is_excluded(target_path) ): return relpath - else: - return url + target_path_from_src_repo + section + + return f"https://github.com/triton-inference-server/{src_git_repo_name}/blob/main/{target_path_from_src_repo}{section}" def replace_hyperlink(m, src_doc_path): """ - TODO: Support of HTML tags for future docs. - Markdown allows , e.g. ]+>. Whether we want to - find and replace the link depends on if they link to internal .md files - or allows relative paths. I haven't seen one such case in our doc so - should be safe for now. + Replace hyperlinks in markdown files. + TODO: Support HTML tags for future docs (e.g., ). """ - hyperlink_str = m.group(2) - match = http_reg.match(hyperlink_str) - - if match: - # Hyperlink is a URL. - res = replace_url_with_relpath(hyperlink_str, src_doc_path) - else: - # Hyperlink is a relative path. - res = replace_relpath_with_url(hyperlink_str, src_doc_path) - + res = ( + replace_url_with_relpath(hyperlink_str, src_doc_path) + if http_reg.match(hyperlink_str) + else replace_relpath_with_url(hyperlink_str, src_doc_path) + ) return m.group(1) + res + m.group(3) -def preprocess_docs(exclude_paths=[]): - # Find all ".md" files inside the current repo. - if exclude_paths: - cmd = ( - ["find", server_docs_abspath, "-type", "d", "\\("] - + " -o ".join([f"-path './{dir}'" for dir in exclude_paths]).split(" ") - + ["\\)", "-prune", "-o", "-type", "f", "-name", "'*.md'", "-print"] - ) - else: - cmd = ["find", server_docs_abspath, "-name", "'*.md'"] - cmd = " ".join(cmd) +def preprocess_docs(exclude_paths=None): + """Find all .md files and preprocess their hyperlinks.""" + # Find all ".md" files + cmd = f"find {server_docs_abspath} -name '*.md'" result = subprocess.run(cmd, check=True, capture_output=True, text=True, shell=True) - docs_list = list(filter(None, result.stdout.split("\n"))) + docs_list = [path for path in result.stdout.split("\n") if path] - # Read, preprocess and write back to each document file. + # Read, preprocess and write back to each document file for doc_abspath in docs_list: if is_excluded(doc_abspath): continue - content = None - with open(doc_abspath, "r") as f: + with open(doc_abspath) as f: content = f.read() content = hyperlink_reg.sub( @@ -380,35 +351,21 @@ def preprocess_docs(exclude_paths=[]): def main(): - args = parser.parse_args() - repo_tag = args.repo_tag - repository_filename = args.repo_file - github_org = args.github_organization - - # Change working directory to server/docs. + """Main function to clone repositories, preprocess docs, and build HTML.""" + logger.info("Starting setup Triton Server documentation for Sphinx build...") + logger.info(f"Collecting repositories from {args.repo_file}...") os.chdir(server_docs_abspath) - run_command("make clean") - repositories = "" - with open(repository_filename, "r") as f: - repositories = f.read() - f.close() + with open(args.repo_file) as f: + repository_list = f.read().strip().split("\n") - repository_list = repositories.strip().split("\n") + # Clone repositories for repository in repository_list: run_command(f"rm -rf {repository}") - clone_from_github(repository, repo_tag, github_org) + clone_from_github(repository, args.repo_tag, args.github_organization) - # Preprocess documents in server_docs_abspath after all repos are cloned. + # Preprocess documents after all repos are cloned preprocess_docs() - run_command("make html") - - # Clean up working directory. - for repository in repository_list: - run_command(f"rm -rf {repository}") - - # Return to previous working directory server/. - os.chdir(server_abspath) if __name__ == "__main__": diff --git a/docs/introduction/compatibility.md b/docs/introduction/compatibility.md index c90899361b..f4d254058c 100644 --- a/docs/introduction/compatibility.md +++ b/docs/introduction/compatibility.md @@ -38,6 +38,7 @@ | Triton release version | NGC Tag | Python version | Torch version | TensorRT version | TensorRT-LLM version | CUDA version | CUDA Driver version | Size | | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 25.11 | nvcr.io/nvidia/tritonserver:25.11-trtllm-python-py3 | Python 3.12.3 | 2.9.0a0+145a3a7bda.nv25.10 | 10.13.3.9 | 1.0.3.2510 | 13.0.2.006 | 580.95.05 | 12.25 GB | | 25.10 | nvcr.io/nvidia/tritonserver:25.10-trtllm-python-py3 | Python 3.12.3 | 2.8.0a0+5228986c39.nv25.6 | 10.11.0.33 | 1.0.0 | 12.9.1.010 | 575.57.08 | 16.21 GB | | 25.09 | nvcr.io/nvidia/tritonserver:25.09-trtllm-python-py3 | Python 3.12.3 | 2.8.0a0+5228986c39.nv25.6 | 10.11.0.33 | 1.0.0 | 12.9.1.010 | 575.57.08 | 16.25 GB | | 25.08 | nvcr.io/nvidia/tritonserver:25.08-trtllm-python-py3 | Python 3.12.3 | 2.8.0a0+5228986c39.nv25.5 | 10.11.0.33 | 0.21.0 | 12.9.0.043 | 575.51.03 | 20.49 GB | @@ -62,6 +63,7 @@ | Triton release version | NGC Tag | Python version | vLLM version | CUDA version | CUDA Driver version | Size | | --- | --- | --- | --- | --- | --- | --- | +| 25.11 | nvcr.io/nvidia/tritonserver:25.11-vllm-python-py3 | Python 3.12.3 | 0.11.0+582e4e37.nv25.11.cu130 | 13.0.2.006 | 580.95.05 | 8.72G | | 25.10 | nvcr.io/nvidia/tritonserver:25.10-vllm-python-py3 | Python 3.12.3 | 0.10.2+9dd9ca32.nv25.10.cu130 | 13.0.2.006 | 580.95.05 | 8.34G | | 25.09 | nvcr.io/nvidia/tritonserver:25.09-vllm-python-py3 | Python 3.12.3 | 0.10.1.1+381074ae.nv25.9.cu130 | 13.0.1.012 | 580.82.07 | 7.78G | | 25.08 | nvcr.io/nvidia/tritonserver:25.08-vllm-python-py3 | Python 3.12.3 | 0.9.2+4ef1e343.nv25.8.post1.cu130 | 13.0.1.012 | 580.82.07 | 8.1G | @@ -86,6 +88,7 @@ | Triton release version | ONNX Runtime | | --- | --- | +| 25.11 | 1.23.2 | | 25.10 | 1.23.1 | | 25.09 | 1.23.0 | | 25.08 | 1.23.0+1d1712fdaf | diff --git a/docs/introduction/release_notes.md b/docs/introduction/release_notes.md index 156ac4a7af..d73bfd9df0 100644 --- a/docs/introduction/release_notes.md +++ b/docs/introduction/release_notes.md @@ -25,9 +25,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> -# [Triton Inference Server Release 25.10](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-25-10.html#rel-25-10) +# [Triton Inference Server Release 25.11](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-25-10.html#rel-25-10) -The Triton Inference Server container image, release 25.10, is available +The Triton Inference Server container image, release 25.11, is available on [NGC](https://ngc.nvidia.com/catalog/containers/nvidia:tritonserver) and is open source on [GitHub](https://github.com/triton-inference-server/server). Release notes can diff --git a/python/openai/README.md b/python/openai/README.md index a609946493..71244b373e 100644 --- a/python/openai/README.md +++ b/python/openai/README.md @@ -51,7 +51,7 @@ docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ - nvcr.io/nvidia/tritonserver:25.10-vllm-python-py3 + nvcr.io/nvidia/tritonserver:25.11-vllm-python-py3 ``` 2. Launch the OpenAI-compatible Triton Inference Server: @@ -313,7 +313,7 @@ Currently, OpenAI-Compatible Frontend supports loading embedding models and embe docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ - nvcr.io/nvidia/tritonserver:25.10-vllm-python-py3 + nvcr.io/nvidia/tritonserver:25.11-vllm-python-py3 ``` 2. Launch the OpenAI-compatible Triton Inference Server: @@ -610,10 +610,9 @@ tool calling result: The weather in Dallas, Texas is 85 degrees fahrenheit. It i #### Named Tool Calling -The OpenAI frontend supports named function calling, utilizing guided decoding in the vLLM and TensorRT-LLM backends. Users can specify one of the tools in `tool_choice` to force the model to select a specific tool for function calling. +The OpenAI frontend supports named function calling, utilizing structured outputs in the vLLM backend and guided decoding in TensorRT-LLM backend. Users can specify one of the tools in `tool_choice` to force the model to select a specific tool for function calling. > [!NOTE] -> The latest release of TensorRT-LLM (v0.18.0) does not yet support guided decoding. To enable this feature, use a build from the main branch of TensorRT-LLM. > For instructions on enabling guided decoding in the TensorRT-LLM backend, please refer to [this guide](https://github.com/triton-inference-server/tensorrtllm_backend/blob/main/docs/guided_decoding.md) Example for making a named tool calling request: diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index 7310a674cc..268a90c4b9 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -696,7 +696,8 @@ def _get_streaming_response_delta( # check to make sure we haven't "forgotten" to stream # any tokens that were generated but previously # matched by partial json parsing, such as '}'. - # only happens if we are NOT using guided decoding + # only happens if we are NOT using structured outputs + # or guided decoding if ( self._should_check_for_unstreamed_tool_arg_tokens( response_delta=response_delta, diff --git a/python/openai/openai_frontend/engine/utils/triton.py b/python/openai/openai_frontend/engine/utils/triton.py index 00cc0c160e..545d9a72db 100644 --- a/python/openai/openai_frontend/engine/utils/triton.py +++ b/python/openai/openai_frontend/engine/utils/triton.py @@ -110,17 +110,15 @@ def _create_vllm_generate_request( if lora_name is not None: sampling_parameters["lora_name"] = lora_name - sampling_parameters = json.dumps(sampling_parameters) guided_json = _get_guided_json_from_tool(request) if guided_json is not None: - from vllm.sampling_params import GuidedDecodingParams + from vllm.sampling_params import StructuredOutputsParams - sampling_parameters_json = json.loads(sampling_parameters) - sampling_parameters_json["guided_decoding"] = json.dumps( - asdict(GuidedDecodingParams.from_optional(json=guided_json)) + sampling_parameters["structured_outputs"] = json.dumps( + asdict(StructuredOutputsParams(json=guided_json)) ) - sampling_parameters = json.dumps(sampling_parameters_json) + sampling_parameters = json.dumps(sampling_parameters) exclude_input_in_output = True echo = getattr(request, "echo", None) @@ -382,13 +380,12 @@ def _validate_triton_responses_non_streaming( responses: List[tritonserver._api._response.InferenceResponse], ): num_responses = len(responses) - if num_responses == 1 and responses[0].final != True: - raise ServerError("Unexpected internal error with incorrect response flags") - if num_responses == 2 and responses[-1].final != True: - raise ServerError("Unexpected internal error with incorrect response flags") - if num_responses > 2: + if 1 <= num_responses <= 2: + if responses[-1].final != True: + raise ServerError("Unexpected internal error with incorrect response flags") + else: raise ServerError( - f"Unexpected number of responses: {num_responses}, expected 1." + f"Unexpected number of responses: {num_responses}, expected 1 or 2." ) diff --git a/python/openai/tests/conftest.py b/python/openai/tests/conftest.py index 5c269a78c1..cff670f3ec 100644 --- a/python/openai/tests/conftest.py +++ b/python/openai/tests/conftest.py @@ -32,6 +32,13 @@ from tests.utils import OpenAIServer, setup_fastapi_app, setup_server +def pytest_configure(config): + """Register custom markers.""" + config.addinivalue_line( + "markers", "openai: mark test to run with OpenAI server (subprocess)" + ) + + ### TEST ENVIRONMENT SETUP ### def infer_test_environment(tool_call_parser): # Infer the test environment for simplicity in local dev/testing. diff --git a/python/openai/tests/test_chat_completions.py b/python/openai/tests/test_chat_completions.py index 2b410c3cc2..4034def2cd 100644 --- a/python/openai/tests/test_chat_completions.py +++ b/python/openai/tests/test_chat_completions.py @@ -590,7 +590,7 @@ def test_chat_completions_custom_tokenizer( # Tokenizers can be provided by a local file path to a directory containing # the relevant files such as tokenizer.json and tokenizer_config.json. custom_tokenizer_path = str(Path(__file__).parent / "custom_tokenizer") - download_cmd = f"huggingface-cli download --local-dir {custom_tokenizer_path} {tokenizer_model} --include *.json" + download_cmd = f"hf download --local-dir {custom_tokenizer_path} {tokenizer_model} --include *.json" print(f"Running download command: {download_cmd}") subprocess.run(download_cmd.split(), check=True) diff --git a/python/openai/tests/test_tool_calling.py b/python/openai/tests/test_tool_calling.py index c738b96303..8f794cc62c 100644 --- a/python/openai/tests/test_tool_calling.py +++ b/python/openai/tests/test_tool_calling.py @@ -341,10 +341,6 @@ async def test_tool_call_with_reply_response( # validate if steaming and non-streaming generates the same content assert "".join(chunks) == choice.message.content - @pytest.mark.skipif( - os.environ.get("IMAGE_KIND") == "TRTLLM", - reason="latest release version of Tensorrt LLM 0.18 doesn't support guided decoding", - ) @pytest.mark.asyncio async def test_tool_call_with_named_tool_choice( self, client: openai.AsyncOpenAI, model: str @@ -448,10 +444,6 @@ async def test_tool_call_with_named_tool_choice( assert choice.message.role == role_name assert choice.message.tool_calls[0].function.name == function_name - @pytest.mark.skipif( - os.environ.get("IMAGE_KIND") == "TRTLLM", - reason="latest release version of Tensorrt LLM 0.18 doesn't support guided decoding", - ) @pytest.mark.asyncio async def test_tool_call_with_required_tool_choice( self, client: openai.AsyncOpenAI, model: str diff --git a/qa/L0_openai/test.sh b/qa/L0_openai/test.sh index 8083c50444..a1db293436 100755 --- a/qa/L0_openai/test.sh +++ b/qa/L0_openai/test.sh @@ -74,6 +74,9 @@ function prepare_tensorrtllm() { MODEL_REPO="$2" ENGINE_PATH="$3" TENSORRTLLM_DIR="$4" + TRITON_BACKEND=tensorrtllm + XGRAMMAR_TOKENIZER_INFO_PATH=tokenizer_info/${MODEL}/xgrammar_tokenizer_info.json + GUIDED_DECODING_BACKEND=xgrammar mkdir -p ${MODEL_REPO} cp ${TENSORRTLLM_DIR}/triton_backend/all_models/inflight_batcher_llm/* "${MODEL_REPO}" -r @@ -83,12 +86,17 @@ function prepare_tensorrtllm() { # 1. Generate the model's trt engines python3 ../generate_engine.py --model "${MODEL}" --engine_path "${ENGINE_PATH}" - # 2. Prepare model repository + # 2. Generate the model's xgrammar tokenizer info. In order to run on C++ backend, we need an extra step to extract tokenizer’s information into json format. + XGRAMMAR_TOKENIZER_INFO_DIR=tokenizer_info/${MODEL} + rm -rf ${XGRAMMAR_TOKENIZER_INFO_DIR} + python3 /app/examples/generate_xgrammar_tokenizer_info.py --model_dir ${MODEL} --output_dir ${XGRAMMAR_TOKENIZER_INFO_DIR} + + # 3. Prepare model repository FILL_TEMPLATE="/app/tools/fill_template.py" python3 ${FILL_TEMPLATE} -i ${MODEL_REPO}/preprocessing/config.pbtxt tokenizer_dir:${ENGINE_PATH},triton_max_batch_size:64,preprocessing_instance_count:1,max_queue_size:0 python3 ${FILL_TEMPLATE} -i ${MODEL_REPO}/postprocessing/config.pbtxt tokenizer_dir:${ENGINE_PATH},triton_max_batch_size:64,postprocessing_instance_count:1 python3 ${FILL_TEMPLATE} -i ${MODEL_REPO}/tensorrt_llm_bls/config.pbtxt triton_max_batch_size:64,decoupled_mode:True,bls_instance_count:1,accumulate_tokens:False,logits_datatype:TYPE_FP32,prompt_embedding_table_data_type:TYPE_FP16 - python3 ${FILL_TEMPLATE} -i ${MODEL_REPO}/tensorrt_llm/config.pbtxt triton_backend:tensorrtllm,triton_max_batch_size:64,decoupled_mode:True,max_beam_width:1,engine_dir:${ENGINE_PATH},batching_strategy:inflight_fused_batching,max_queue_size:0,max_queue_delay_microseconds:1000,encoder_input_features_data_type:TYPE_FP16,logits_datatype:TYPE_FP32,exclude_input_in_output:True,prompt_embedding_table_data_type:TYPE_FP16 + python3 ${FILL_TEMPLATE} -i ${MODEL_REPO}/tensorrt_llm/config.pbtxt triton_backend:${TRITON_BACKEND},triton_max_batch_size:64,decoupled_mode:True,max_beam_width:1,engine_dir:${ENGINE_PATH},batching_strategy:inflight_fused_batching,max_queue_size:0,max_queue_delay_microseconds:1000,encoder_input_features_data_type:TYPE_FP16,logits_datatype:TYPE_FP32,exclude_input_in_output:True,prompt_embedding_table_data_type:TYPE_FP16,guided_decoding_backend:${GUIDED_DECODING_BACKEND},xgrammar_tokenizer_info_path:${XGRAMMAR_TOKENIZER_INFO_PATH} } function pre_test() { diff --git a/qa/common/gen_jetson_trt_models b/qa/common/gen_jetson_trt_models index 693d233f91..b763d2c490 100755 --- a/qa/common/gen_jetson_trt_models +++ b/qa/common/gen_jetson_trt_models @@ -34,7 +34,7 @@ # Make all generated files accessible outside of container umask 0000 # Set the version of the models -TRITON_VERSION=${TRITON_VERSION:=25.10} +TRITON_VERSION=${TRITON_VERSION:=25.11} # Set the CUDA device to use CUDA_DEVICE=${RUNNER_ID:=0} # Set TensorRT image diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index ce321ee11b..5b59beb715 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -60,7 +60,7 @@ log_message.status "Changing working directory to the script directory to: " "${ cd ${TRITON_MDLS_BASE_SCRIPT_DIR} log_message.status "define: default values" -TRITON_VERSION=${TRITON_VERSION:=25.10} +TRITON_VERSION=${TRITON_VERSION:=25.11} ONNX_VERSION=1.16.1 ONNX_OPSET=0 OPENVINO_VERSION=2024.5.0 @@ -325,10 +325,10 @@ chmod -R 777 $TRITON_MDLS_QA_TRT_FORMAT_MODEL nvidia-smi --query-gpu=compute_cap | grep -qz 11.0 && echo -e '\033[33m[WARNING]\033[0m Skipping model generation for data dependent shape' || python3 $TRITON_MDLS_SRC_DIR/gen_qa_trt_data_dependent_shape.py --models_dir=$TRITON_MDLS_QA_TRT_DATA_DEPENDENT_MODEL chmod -R 777 $TRITON_MDLS_QA_TRT_DATA_DEPENDENT_MODEL # Make shared library for custom Hardmax plugin. -if [ -d "/usr/src/tensorrt" ]; then +if [ -d "/usr/src/tensorrt/samples/python/onnx_custom_plugin" ]; then cd /usr/src/tensorrt/samples/python/onnx_custom_plugin else - git clone -b release/\${TRT_VERSION%.*.*} --depth 1 https://github.com/NVIDIA/TensorRT.git + git clone -b release/\${TRT_VERSION%.*.*} --depth 1 https://github.com/NVIDIA/TensorRT.git /workspace/TensorRT cd /workspace/TensorRT/samples/python/onnx_custom_plugin fi rm -rf build && mkdir build && \