diff --git a/Dockerfile.sdk b/Dockerfile.sdk index c517be2918..e919b3f50d 100644 --- a/Dockerfile.sdk +++ b/Dockerfile.sdk @@ -29,10 +29,9 @@ # # Base image on the minimum Triton container -ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.08-py3-min +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.09-py3-min ARG TRITON_CLIENT_REPO_SUBDIR=clientrepo -ARG TRITON_PA_REPO_SUBDIR=perfanalyzerrepo ARG TRITON_REPO_ORGANIZATION=http://github.com/triton-inference-server ARG TRITON_COMMON_REPO_TAG=main ARG TRITON_CORE_REPO_TAG=main @@ -41,7 +40,6 @@ ARG TRITON_THIRD_PARTY_REPO_TAG=main ARG TRITON_ENABLE_GPU=ON ARG JAVA_BINDINGS_MAVEN_VERSION=3.8.4 ARG JAVA_BINDINGS_JAVACPP_PRESETS_TAG=1.5.8 -ARG TRITON_PERF_ANALYZER_BUILD=1 # DCGM version to install for Model Analyzer ARG DCGM_VERSION=4.4.0-1 @@ -97,7 +95,6 @@ RUN rm -f /usr/bin/python && \ # Build the client library and examples ARG TRITON_REPO_ORGANIZATION ARG TRITON_CLIENT_REPO_SUBDIR -ARG TRITON_PA_REPO_SUBDIR ARG TRITON_COMMON_REPO_TAG ARG TRITON_CORE_REPO_TAG ARG TRITON_CLIENT_REPO_TAG @@ -106,14 +103,10 @@ ARG TRITON_ENABLE_GPU ARG JAVA_BINDINGS_MAVEN_VERSION ARG JAVA_BINDINGS_JAVACPP_PRESETS_TAG ARG TARGETPLATFORM -ARG TRITON_PERF_ANALYZER_BUILD - -ENV TRITON_PERF_ANALYZER_BUILD=${TRITON_PERF_ANALYZER_BUILD} WORKDIR /workspace COPY TRITON_VERSION . COPY ${TRITON_CLIENT_REPO_SUBDIR} client -COPY ${TRITON_PA_REPO_SUBDIR} perf_analyzer WORKDIR /workspace/client_build RUN cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ @@ -124,63 +117,11 @@ RUN cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ -DTRITON_THIRD_PARTY_REPO_TAG=${TRITON_THIRD_PARTY_REPO_TAG} \ -DTRITON_ENABLE_PERF_ANALYZER=OFF \ -DTRITON_ENABLE_CC_HTTP=ON -DTRITON_ENABLE_CC_GRPC=ON \ - -DTRITON_ENABLE_PYTHON_HTTP=OFF -DTRITON_ENABLE_PYTHON_GRPC=OFF \ + -DTRITON_ENABLE_PYTHON_HTTP=ON -DTRITON_ENABLE_PYTHON_GRPC=ON \ -DTRITON_ENABLE_JAVA_HTTP=ON \ -DTRITON_ENABLE_EXAMPLES=ON -DTRITON_ENABLE_TESTS=ON \ -DTRITON_ENABLE_GPU=${TRITON_ENABLE_GPU} /workspace/client -RUN cmake --build . -v --parallel --target cc-clients java-clients - -# TODO: PA will rebuild the CC clients since it depends on it. -# This should be optimized so that we do not have to build -# the CC clients twice. Similarly, because the SDK expectation is -# that PA is packaged with the python client, we hold off on building -# the python client until now. Post-migration we should focus -# effort on de-tangling these flows. -WORKDIR /workspace/pa_build -# NOTE: If TRITON_PERF_ANALYZER_BUILD=0, the Performance Analyzer (PA) binaries must already exist -# in the path specified by the ARG TRITON_PA_REPO_SUBDIR. -RUN if [ "$TRITON_PERF_ANALYZER_BUILD" = "1" ]; then \ - cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ - -DTRITON_VERSION=`cat /workspace/TRITON_VERSION` \ - -DTRITON_REPO_ORGANIZATION=${TRITON_REPO_ORGANIZATION} \ - -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} \ - -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} \ - -DTRITON_CLIENT_REPO_TAG=${TRITON_CLIENT_REPO_TAG} \ - -DTRITON_THIRD_PARTY_REPO_TAG=${TRITON_THIRD_PARTY_REPO_TAG} \ - -DTRITON_ENABLE_PERF_ANALYZER_C_API=ON \ - -DTRITON_ENABLE_PERF_ANALYZER_TFS=ON \ - -DTRITON_ENABLE_PERF_ANALYZER_TS=ON \ - -DTRITON_ENABLE_PERF_ANALYZER_OPENAI=ON \ - -DTRITON_ENABLE_CC_HTTP=ON \ - -DTRITON_ENABLE_CC_GRPC=ON \ - -DTRITON_ENABLE_PYTHON_HTTP=ON \ - -DTRITON_ENABLE_PYTHON_GRPC=ON \ - -DTRITON_PACKAGE_PERF_ANALYZER=ON \ - -DTRITON_ENABLE_GPU=${TRITON_ENABLE_GPU} \ - /workspace/perf_analyzer && \ - cmake --build . -v --parallel --target perf-analyzer python-clients && \ - pip3 install build && \ - cd /workspace/perf_analyzer/genai-perf && \ - python3 -m build --wheel --outdir /workspace/install/python; \ - else \ - ls /workspace/perf_analyzer/ && \ - tar -xzf /workspace/perf_analyzer/perf_analyzer*.tar.gz -C /workspace/install/bin && \ - echo "Perf Analyzer binaries was extracted and not build" && \ - cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ - -DTRITON_VERSION=`cat /workspace/TRITON_VERSION` \ - -DTRITON_REPO_ORGANIZATION=${TRITON_REPO_ORGANIZATION} \ - -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} \ - -DTRITON_CLIENT_REPO_TAG=${TRITON_CLIENT_REPO_TAG} \ - -DTRITON_THIRD_PARTY_REPO_TAG=${TRITON_THIRD_PARTY_REPO_TAG} \ - -DTRITON_ENABLE_PYTHON_HTTP=ON \ - -DTRITON_ENABLE_PYTHON_GRPC=ON \ - -DTRITON_PACKAGE_PERF_ANALYZER=ON \ - -DTRITON_ENABLE_GPU=${TRITON_ENABLE_GPU} \ - /workspace/perf_analyzer && \ - cmake --build . -v --parallel --target python-clients && \ - mkdir -p /workspace/install/python && \ - cp /workspace/perf_analyzer/genai_perf-*.whl /workspace/install/python/; \ - fi +RUN cmake --build . -v --parallel --target cc-clients java-clients python-clients # Install Java API Bindings RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \ @@ -226,7 +167,6 @@ RUN apt-get update && \ python3-pip \ python3-setuptools \ python3-wheel \ - software-properties-common \ vim \ wget && \ pip3 install "grpcio<1.68" "grpcio-tools<1.68" @@ -235,7 +175,6 @@ WORKDIR /workspace COPY TRITON_VERSION . COPY NVIDIA_Deep_Learning_Container_License.pdf . COPY --from=sdk_build /workspace/client/ client/ -COPY --from=sdk_build /workspace/perf_analyzer/ perf_analyzer/ COPY --from=sdk_build /workspace/install/ install/ RUN cd install && \ export VERSION=`cat /workspace/TRITON_VERSION` && \ @@ -253,8 +192,6 @@ COPY --from=sdk_build /workspace/client/src/python/library/tests/* qa/python_cli # Install an image needed by the quickstart and other documentation. COPY qa/images/mug.jpg images/mug.jpg -RUN pip3 install install/python/genai_perf-*.whl - # Install the dependencies needed to run the client examples. These # are not needed for building but including them allows this image to # be used to run the client examples. @@ -263,6 +200,9 @@ RUN pip3 install --upgrade "numpy<2" pillow attrdict && \ "tritonclient-*linux*.whl" | xargs printf -- '%s[all]' | \ xargs pip3 install --upgrade +# Install GenAI-Perf +RUN pip3 install genai-perf + # Install DCGM RUN if [ "$TRITON_ENABLE_GPU" = "ON" ]; then \ [ "$(uname -m)" != "x86_64" ] && arch="sbsa" || arch="x86_64" && \ diff --git a/README.md b/README.md index 1bf23ead01..a0cf5e1c61 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,6 @@ --> [![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) ->[!WARNING] ->You are currently on the `main` branch which tracks under-development progress ->towards the next release. The current release is version [2.60.0](https://github.com/triton-inference-server/server/releases/latest) ->and corresponds to the 25.08 container release on NVIDIA GPU Cloud (NGC). - # Triton Inference Server Triton Inference Server is an open source inference serving software that @@ -61,7 +56,7 @@ Major features include: - Provides [Backend API](https://github.com/triton-inference-server/backend) that allows adding custom backends and pre/post processing operations - Supports writing custom backends in python, a.k.a. - [Python-based backends.](https://github.com/triton-inference-server/backend/blob/main/docs/python_based_backends.md#python-based-backends) + [Python-based backends.](https://github.com/triton-inference-server/backend/blob/r25.09/docs/python_based_backends.md#python-based-backends) - Model pipelines using [Ensembling](docs/user_guide/architecture.md#ensemble-models) or [Business Logic Scripting @@ -90,16 +85,16 @@ Inference Server with the ```bash # Step 1: Create the example model repository -git clone -b r25.08 https://github.com/triton-inference-server/server.git +git clone -b r25.09 https://github.com/triton-inference-server/server.git cd server/docs/examples ./fetch_models.sh # Step 2: Launch triton from the NGC Triton container -docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:25.08-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx +docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:25.09-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx # Step 3: Sending an Inference Request # In a separate console, launch the image_client example from the NGC Triton SDK container -docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:25.08-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg +docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:25.09-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg # Inference should return the following Image '/workspace/images/mug.jpg': @@ -172,10 +167,10 @@ configuration](docs/user_guide/model_configuration.md) for the model. [Python](https://github.com/triton-inference-server/python_backend), and more - Not all the above backends are supported on every platform supported by Triton. Look at the - [Backend-Platform Support Matrix](https://github.com/triton-inference-server/backend/blob/main/docs/backend_platform_support_matrix.md) + [Backend-Platform Support Matrix](https://github.com/triton-inference-server/backend/blob/r25.09/docs/backend_platform_support_matrix.md) to learn which backends are supported on your target platform. - Learn how to [optimize performance](docs/user_guide/optimization.md) using the - [Performance Analyzer](https://github.com/triton-inference-server/perf_analyzer/blob/main/README.md) + [Performance Analyzer](https://github.com/triton-inference-server/perf_analyzer/blob/r25.09/README.md) and [Model Analyzer](https://github.com/triton-inference-server/model_analyzer) - Learn how to [manage loading and unloading models](docs/user_guide/model_management.md) in @@ -189,14 +184,14 @@ A Triton *client* application sends inference and other requests to Triton. The [Python and C++ client libraries](https://github.com/triton-inference-server/client) provide APIs to simplify this communication. -- Review client examples for [C++](https://github.com/triton-inference-server/client/blob/main/src/c%2B%2B/examples), - [Python](https://github.com/triton-inference-server/client/blob/main/src/python/examples), - and [Java](https://github.com/triton-inference-server/client/blob/main/src/java/src/main/java/triton/client/examples) +- Review client examples for [C++](https://github.com/triton-inference-server/client/blob/r25.09/src/c%2B%2B/examples), + [Python](https://github.com/triton-inference-server/client/blob/r25.09/src/python/examples), + and [Java](https://github.com/triton-inference-server/client/blob/r25.09/src/java/src/main/java/triton/client/examples) - Configure [HTTP](https://github.com/triton-inference-server/client#http-options) and [gRPC](https://github.com/triton-inference-server/client#grpc-options) client options - Send input data (e.g. a jpeg image) directly to Triton in the [body of an HTTP - request without any additional metadata](https://github.com/triton-inference-server/server/blob/main/docs/protocol/extension_binary_data.md#raw-binary-request) + request without any additional metadata](https://github.com/triton-inference-server/server/blob/r25.09/docs/protocol/extension_binary_data.md#raw-binary-request) ### Extend Triton @@ -205,7 +200,7 @@ designed for modularity and flexibility - [Customize Triton Inference Server container](docs/customization_guide/compose.md) for your use case - [Create custom backends](https://github.com/triton-inference-server/backend) - in either [C/C++](https://github.com/triton-inference-server/backend/blob/main/README.md#triton-backend-api) + in either [C/C++](https://github.com/triton-inference-server/backend/blob/r25.09/README.md#triton-backend-api) or [Python](https://github.com/triton-inference-server/python_backend) - Create [decoupled backends and models](docs/user_guide/decoupled_models.md) that can send multiple responses for a request or not send any responses for a request diff --git a/TRITON_VERSION b/TRITON_VERSION index c7cf1876bd..ac067ed8bf 100644 --- a/TRITON_VERSION +++ b/TRITON_VERSION @@ -1 +1 @@ -2.61.0dev +2.61.0 \ No newline at end of file diff --git a/build.py b/build.py index b9f621e9e8..68c48a2a58 100755 --- a/build.py +++ b/build.py @@ -71,14 +71,14 @@ # DEFAULT_TRITON_VERSION_MAP = { - "release_version": "2.61.0dev", - "triton_container_version": "25.09dev", - "upstream_container_version": "25.08", + "release_version": "2.61.0", + "triton_container_version": "25.09", + "upstream_container_version": "25.09", "ort_version": "1.23.0", "ort_openvino_version": "2025.3.0", "standalone_openvino_version": "2025.3.0", "dcgm_version": "4.4.0-1", - "vllm_version": "0.9.2", + "vllm_version": "0.10.1.1", "rhel_py_version": "3.12.3", } @@ -1384,7 +1384,6 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach libgoogle-perftools-dev \\ libjemalloc-dev \\ libnuma-dev \\ - software-properties-common \\ wget \\ {backend_dependencies} \\ python3-pip \\ diff --git a/deploy/aws/values.yaml b/deploy/aws/values.yaml index 6ec8264ae1..27bec3dd10 100644 --- a/deploy/aws/values.yaml +++ b/deploy/aws/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.08-py3 + imageName: nvcr.io/nvidia/tritonserver:25.09-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/fleetcommand/Chart.yaml b/deploy/fleetcommand/Chart.yaml index 5aebbe4957..8b0eed9bae 100644 --- a/deploy/fleetcommand/Chart.yaml +++ b/deploy/fleetcommand/Chart.yaml @@ -26,7 +26,7 @@ apiVersion: v1 # appVersion is the Triton version; update when changing release -appVersion: 2.60.0" +appVersion: 2.61.0" description: Triton Inference Server (Fleet Command) name: triton-inference-server # version is the Chart version; update when changing anything in the chart diff --git a/deploy/fleetcommand/values.yaml b/deploy/fleetcommand/values.yaml index 94aa860e77..aa2e45e192 100644 --- a/deploy/fleetcommand/values.yaml +++ b/deploy/fleetcommand/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.08-py3 + imageName: nvcr.io/nvidia/tritonserver:25.09-py3 pullPolicy: IfNotPresent numGpus: 1 serverCommand: tritonserver @@ -47,13 +47,13 @@ image: # # To set model control mode, uncomment and configure below # TODO: Fix the following url, it is invalid - # See https://github.com/triton-inference-server/server/blob/r25.08/docs/user_guide/model_management.md + # See https://github.com/triton-inference-server/server/blob/r25.09/docs/user_guide/model_management.md # for more details #- --model-control-mode=explicit|poll|none # # Additional server args # - # see https://github.com/triton-inference-server/server/blob/r25.08/README.md + # see https://github.com/triton-inference-server/server/blob/r25.09/README.md # for more details service: diff --git a/deploy/gcp/values.yaml b/deploy/gcp/values.yaml index 027cf85819..c4c2ff9951 100644 --- a/deploy/gcp/values.yaml +++ b/deploy/gcp/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.08-py3 + imageName: nvcr.io/nvidia/tritonserver:25.09-py3 pullPolicy: IfNotPresent modelRepositoryPath: gs://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml index 842362d6ce..4b21697c29 100644 --- a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml +++ b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml @@ -33,7 +33,7 @@ metadata: namespace: default spec: containers: - - image: nvcr.io/nvidia/tritonserver:25.08-py3-sdk + - image: nvcr.io/nvidia/tritonserver:25.09-py3-sdk imagePullPolicy: Always name: nv-triton-client securityContext: diff --git a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh index 8ff6eeb631..b960b8ff94 100755 --- a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh +++ b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh @@ -27,9 +27,9 @@ export REGISTRY=gcr.io/$(gcloud config get-value project | tr ':' '/') export APP_NAME=tritonserver -export MAJOR_VERSION=2.60 -export MINOR_VERSION=2.60.0 -export NGC_VERSION=25.08-py3 +export MAJOR_VERSION=2.61 +export MINOR_VERSION=2.61.0 +export NGC_VERSION=25.09-py3 docker pull nvcr.io/nvidia/$APP_NAME:$NGC_VERSION diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml index 04d80cbfd4..e86c2b6a21 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml @@ -28,4 +28,4 @@ apiVersion: v1 appVersion: "2.60" description: Triton Inference Server name: triton-inference-server -version: 2.60.0 +version: 2.61.0 diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml index 7c7862d3f7..9f87933228 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml @@ -31,14 +31,14 @@ maxReplicaCount: 3 tritonProtocol: HTTP # HPA GPU utilization autoscaling target HPATargetAverageValue: 85 -modelRepositoryPath: gs://triton_sample_models/25.08 -publishedVersion: '2.60.0' +modelRepositoryPath: gs://triton_sample_models/25.09 +publishedVersion: '2.61.0' gcpMarketplace: true image: registry: gcr.io repository: nvidia-ngc-public/tritonserver - tag: 25.08-py3 + tag: 25.09-py3 pullPolicy: IfNotPresent # modify the model repository here to match your GCP storage bucket numGpus: 1 diff --git a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml index 7a1774155f..107c3ce161 100644 --- a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.60.0' + publishedVersion: '2.61.0' publishedVersionMetadata: releaseNote: >- Initial release. diff --git a/deploy/gke-marketplace-app/server-deployer/schema.yaml b/deploy/gke-marketplace-app/server-deployer/schema.yaml index 607aafe4a5..4bc5fd29e6 100644 --- a/deploy/gke-marketplace-app/server-deployer/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/schema.yaml @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.60.0' + publishedVersion: '2.61.0' publishedVersionMetadata: releaseNote: >- Initial release. @@ -89,7 +89,7 @@ properties: modelRepositoryPath: type: string title: Bucket where models are stored. Please make sure the user/service account to create the GKE app has permission to this GCS bucket. Read Triton documentation on configs and formatting details, supporting TensorRT, TensorFlow, Pytorch, Onnx ... etc. - default: gs://triton_sample_models/25.08 + default: gs://triton_sample_models/25.09 image.ldPreloadPath: type: string title: Leave this empty by default. Triton allows users to create custom layers for backend such as TensorRT plugin, the compiled shared library must be provided via LD_PRELOAD environment variable. diff --git a/deploy/gke-marketplace-app/trt-engine/README.md b/deploy/gke-marketplace-app/trt-engine/README.md index 4651799daf..f2118a1bc3 100644 --- a/deploy/gke-marketplace-app/trt-engine/README.md +++ b/deploy/gke-marketplace-app/trt-engine/README.md @@ -33,7 +33,7 @@ ``` docker run --gpus all -it --network host \ --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \ - -v ~:/scripts nvcr.io/nvidia/tensorrt:25.08-py3 + -v ~:/scripts nvcr.io/nvidia/tensorrt:25.09-py3 pip install onnx six torch tf2onnx tensorflow @@ -57,7 +57,7 @@ mkdir -p engines python3 builder.py -m models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/model.ckpt -o engines/bert_large_int8_bs1_s128.engine -b 1 -s 128 -c models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/ -v models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/vocab.txt --int8 --fp16 --strict --calib-num 1 -iln -imh -gsutil cp bert_large_int8_bs1_s128.engine gs://triton_sample_models/25.08/bert/1/model.plan +gsutil cp bert_large_int8_bs1_s128.engine gs://triton_sample_models/25.09/bert/1/model.plan ``` -For each Triton upgrade, container version used to generate the model, and the model path in GCS `gs://triton_sample_models/25.08/` should be updated accordingly with the correct version. +For each Triton upgrade, container version used to generate the model, and the model path in GCS `gs://triton_sample_models/25.09/` should be updated accordingly with the correct version. diff --git a/deploy/k8s-onprem/values.yaml b/deploy/k8s-onprem/values.yaml index 402a52635f..66f17e7c03 100644 --- a/deploy/k8s-onprem/values.yaml +++ b/deploy/k8s-onprem/values.yaml @@ -30,7 +30,7 @@ tags: openshift: false image: - imageName: nvcr.io/nvidia/tritonserver:25.08-py3 + imageName: nvcr.io/nvidia/tritonserver:25.09-py3 pullPolicy: IfNotPresent modelRepositoryServer: < Replace with the IP Address of your file server > modelRepositoryPath: /srv/models diff --git a/deploy/oci/values.yaml b/deploy/oci/values.yaml index 7261b9c0a3..be4929a0f1 100644 --- a/deploy/oci/values.yaml +++ b/deploy/oci/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.08-py3 + imageName: nvcr.io/nvidia/tritonserver:25.09-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://https://.compat.objectstorage..oraclecloud.com:443/triton-inference-server-repository numGpus: 1 diff --git a/docs/examples/fetch_models.sh b/docs/examples/fetch_models.sh index 7af5a91c1f..ef76833c2b 100755 --- a/docs/examples/fetch_models.sh +++ b/docs/examples/fetch_models.sh @@ -33,9 +33,11 @@ mkdir -p model_repository/inception_onnx/1 wget -O /tmp/inception_v3_2016_08_28_frozen.pb.tar.gz \ https://storage.googleapis.com/download.tensorflow.org/models/inception_v3_2016_08_28_frozen.pb.tar.gz (cd /tmp && tar xzf inception_v3_2016_08_28_frozen.pb.tar.gz) +apt update -qq && apt install -y python3-venv +rm -rf tf2onnx python3 -m venv tf2onnx source ./tf2onnx/bin/activate -pip3 install "numpy<2" tensorflow tf2onnx +pip3 install "numpy<2" tensorflow==2.18.1 tf2onnx==1.16.1 onnx==1.16.1 python3 -m tf2onnx.convert --graphdef /tmp/inception_v3_2016_08_28_frozen.pb --output inception_v3_onnx.model.onnx --inputs input:0 --outputs InceptionV3/Predictions/Softmax:0 deactivate mv inception_v3_onnx.model.onnx model_repository/inception_onnx/1/model.onnx diff --git a/docs/introduction/compatibility.md b/docs/introduction/compatibility.md index 9ada3d122e..1e1dbb5e3c 100644 --- a/docs/introduction/compatibility.md +++ b/docs/introduction/compatibility.md @@ -38,6 +38,7 @@ | Triton release version | NGC Tag | Python version | Torch version | TensorRT version | TensorRT-LLM version | CUDA version | CUDA Driver version | Size | | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 25.09 | nvcr.io/nvidia/tritonserver:25.09-trtllm-python-py3 | Python 3.12.3 | 2.8.0a0+5228986c39.nv25.6 | 10.11.0.33 | 1.0.0 | 12.9.1.010 | 575.57.08 | 16.25 GB | | 25.08 | nvcr.io/nvidia/tritonserver:25.08-trtllm-python-py3 | Python 3.12.3 | 2.8.0a0+5228986c39.nv25.5 | 10.11.0.33 | 0.21.0 | 12.9.0.043 | 575.51.03 | 20.49 GB | | 25.07 | nvcr.io/nvidia/tritonserver:25.07-trtllm-python-py3 | Python 3.12.3 | 2.7.0a0+79aa17489c.nv25.4 | 10.10.0.31 | 0.20.0 | 12.9.0.036 | 575.51.03 | 18.3G | | 25.06 | nvcr.io/nvidia/tritonserver:25.06-trtllm-python-py3 | Python 3.12.3 | 2.7.0a0+79aa17489c.nv25.4 | 10.10.0.31 | 0.20.0 | 12.9.0.036 | 575.51.03 | 18.3G | @@ -60,7 +61,8 @@ | Triton release version | NGC Tag | Python version | vLLM version | CUDA version | CUDA Driver version | Size | | --- | --- | --- | --- | --- | --- | --- | -| 25.08 | nvcr.io/nvidia/tritonserver:25.08-vllm-python-py3 | Python 3.12.3 | 0.9.2+4ef1e343.nv25.8.post1.cu130 | 12.9.0.043 | 575.51.03 | 8.1G | +| 25.09 | nvcr.io/nvidia/tritonserver:25.09-vllm-python-py3 | Python 3.12.3 | 0.10.1.1+381074ae.nv25.9.cu130 | 13.0.1.012 | 580.82.07 | 7.78G | +| 25.08 | nvcr.io/nvidia/tritonserver:25.08-vllm-python-py3 | Python 3.12.3 | 0.9.2+4ef1e343.nv25.8.post1.cu130 | 13.0.1.012 | 580.82.07 | 8.1G | | 25.07 | nvcr.io/nvidia/tritonserver:25.07-vllm-python-py3 | Python 3.12.3 | 0.9.0rc1+1958ee56.nv25.6.cu129 | 12.9.0.043 | 575.51.03 | 10G | | 25.06 | nvcr.io/nvidia/tritonserver:25.06-vllm-python-py3 | Python 3.12.3 | 0.9.0rc1+1958ee56.nv25.6.cu129 | 12.9.0.043 | 575.51.03 | 10G | | 25.05 | nvcr.io/nvidia/tritonserver:25.05-vllm-python-py3 | Python 3.12.3 | 0.8.4+dc1a3e10.nv25.5.cu129 | 12.9.0.043 | 575.51.03 | 10G | @@ -82,6 +84,7 @@ | Triton release version | ONNX Runtime | | --- | --- | +| 25.09 | 1.23.0 | | 25.08 | 1.23.0+1d1712fdaf | | 25.07 | 1.22.0 | | 25.06 | 1.22.0 | diff --git a/docs/introduction/release_notes.md b/docs/introduction/release_notes.md index 7038abbb01..31fa2784c9 100644 --- a/docs/introduction/release_notes.md +++ b/docs/introduction/release_notes.md @@ -25,9 +25,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> -# [Triton Inference Server Release 25.06](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-25-06.html#rel-25-06) +# [Triton Inference Server Release 25.09](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-25-09.html#rel-25-09) -The Triton Inference Server container image, release 25.06, is available +The Triton Inference Server container image, release 25.09, is available on [NGC](https://ngc.nvidia.com/catalog/containers/nvidia:tritonserver) and is open source on [GitHub](https://github.com/triton-inference-server/server). Release notes can diff --git a/docs/user_guide/performance_tuning.md b/docs/user_guide/performance_tuning.md index 73cd6dfd9c..9264307617 100644 --- a/docs/user_guide/performance_tuning.md +++ b/docs/user_guide/performance_tuning.md @@ -187,8 +187,8 @@ other frameworks. mkdir -p ./models/densenet_onnx/1 # Download model and place it in model repository -wget -O models/densenet_onnx/1/model.onnx -https://contentmamluswest001.blob.core.windows.net/content/14b2744cf8d6418c87ffddc3f3127242/9502630827244d60a1214f250e3bbca7/08aed7327d694b8dbaee2c97b8d0fcba/densenet121-1.2.onnx +wget -O models/densenet_onnx/1/model.onnx \ + https://github.com/onnx/models/raw/main/validated/vision/classification/densenet-121/model/densenet-7.onnx ``` 2. Create a minimal [Model Configuration](model_configuration.md) for the diff --git a/python/openai/README.md b/python/openai/README.md index c9c14b636e..5dda869bab 100644 --- a/python/openai/README.md +++ b/python/openai/README.md @@ -51,7 +51,7 @@ docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ - nvcr.io/nvidia/tritonserver:25.08-vllm-python-py3 + nvcr.io/nvidia/tritonserver:25.09-vllm-python-py3 ``` 2. Launch the OpenAI-compatible Triton Inference Server: @@ -637,4 +637,55 @@ Example output: function name: get_n_day_weather_forecast function arguments: {"city": "Dallas", "state": "TX", "unit": "fahrenheit", num_days: 1} tool calling result: The weather in Dallas, Texas is 85 degrees fahrenheit in next 1 days. -``` \ No newline at end of file +``` + +## Limit Endpoint Access + +The OpenAI-compatible server supports restricting access to specific API endpoints through authentication headers. This feature allows you to protect sensitive endpoints while keeping others publicly accessible. + +### Configuration + +Use the `--openai-restricted-api` command-line argument to configure endpoint restrictions: + +``` +--openai-restricted-api ,,... +``` + +- **`API`**: A comma-separated list of APIs to be included in this group. Note that currently a given API is not allowed to be included in multiple groups. The following protocols / APIs are recognized: + - **inference**: Chat completions and text completions endpoints + - `POST /v1/chat/completions` + - `POST /v1/completions` + - **model-repository**: Model listing and information endpoints + - `GET /v1/models` + - `GET /v1/models/{model_name}` + - **metrics**: Server metrics endpoint + - `GET /metrics` + - **health**: Health check endpoint + - `GET /health/ready` + +- **`restricted-key`**: The HTTP request header to be checked when a request is received. +- **`restricted-value`**: The header value required to access the specified protocols. + +### Examples + +#### Restrict Inference API Endpoints Only +```bash +--openai-restricted-api "inference api-key my-secret-key" +``` + +Clients must include the header: +```bash +curl -H "api-key: my-secret-key" \ + -X POST http://localhost:9000/v1/chat/completions \ + -d '{"model": "my-model", "messages": [{"role": "user", "content": "Hello"}]}' +``` + +#### Restrict Multiple API Endpoints +```bash +# Different authentication for different APIs +--openai-restricted-api "inference user-key user-secret" \ +--openai-restricted-api "model-repository admin-key admin-secret" + +# Multiple APIs in single argument with shared authentication +--openai-restricted-api "inference,model-repository shared-key shared-secret" +``` diff --git a/python/openai/openai_frontend/engine/utils/triton.py b/python/openai/openai_frontend/engine/utils/triton.py index 3104c49911..636e58435d 100644 --- a/python/openai/openai_frontend/engine/utils/triton.py +++ b/python/openai/openai_frontend/engine/utils/triton.py @@ -28,6 +28,7 @@ import os import re from dataclasses import asdict, dataclass, field +from pathlib import Path from typing import Iterable, List, Optional, Union import numpy as np @@ -356,12 +357,28 @@ def _get_guided_json_from_tool( def _get_vllm_lora_names( model_repository: str | list[str], model_name: str, model_version: int ) -> None | List[str]: + if ( + len(model_name) == 0 + or model_name.isspace() + or "/" in model_name + or "\\" in model_name + ): + raise ValueError( + f"Invalid model name: '{model_name}'. Model names must be valid file-system-path segment names." + ) lora_names = [] repo_paths = model_repository if isinstance(repo_paths, str): repo_paths = [repo_paths] for repo_path in repo_paths: model_path = os.path.join(repo_path, model_name) + if (not Path(model_path).is_relative_to(repo_path)) or ( + os.path.normpath(model_path) != model_path + ): + raise ValueError( + f"Invalid model name: '{model_name}'. Model names must be valid file-system-path segment names." + ) + model_path = os.path.normpath(model_path) if not os.path.isdir(model_path): # Cloud path? return None diff --git a/python/openai/openai_frontend/frontend/fastapi/middleware/__init__.py b/python/openai/openai_frontend/frontend/fastapi/middleware/__init__.py new file mode 100644 index 0000000000..f3dec540e2 --- /dev/null +++ b/python/openai/openai_frontend/frontend/fastapi/middleware/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py b/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py new file mode 100644 index 0000000000..9b73169ade --- /dev/null +++ b/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py @@ -0,0 +1,221 @@ +# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from fastapi import Request +from fastapi.responses import JSONResponse +from starlette.middleware.base import BaseHTTPMiddleware + +# Mapping of API to their corresponding HTTP endpoints +ENDPOINT_MAPPING = { + "inference": ["POST /v1/chat/completions", "POST /v1/completions"], + "model-repository": ["GET /v1/models"], + "metrics": ["GET /metrics"], + "health": ["GET /health/ready"], +} + + +class RestrictedFeatures: + """ + Manages API endpoint restrictions and their authentication requirements. + + This class parses command-line arguments for restricted API configurations + and provides methods to check if specific APIs are restricted + and what authentication is required. + """ + + def __init__(self, args: list[str]): + """ + Initialize the RestrictedFeatures with command-line arguments. + + Args: + args: List of --openai-restricted-api argument strings + (e.g., [["inference", "infer-key", "infer-value"], + ["model-repository", "model-key", "model-value"]]) + """ + self._restrictions = {} + self.ParseRestrictedFeatureOption(args) + + def ParseRestrictedFeatureOption(self, args): + """ + Parse command-line arguments to extract API restrictions. + + Args: + args: List of restriction configuration strings + + Raises: + ValueError: If unknown API is specified or duplicate API configs are found + """ + for apis, key, value in args: + api_list = apis.split(",") + for api in api_list: + # Validate that the API is valid + if api not in ENDPOINT_MAPPING: + raise ValueError( + f"Unknown API '{api}'. Available APIs: {list(ENDPOINT_MAPPING.keys())}" + ) + + # Check for duplicate APIs across different arguments + if self.IsRestricted(api): + raise ValueError( + f"restricted api '{api}' can not be specified in multiple config groups" + ) + + self.Insert(api, (key, value)) + + def RestrictionDict(self) -> dict[str, tuple[str, str]]: + """ + Get a copy of the restrictions dictionary. + + Returns: + dict: Copy of the restrictions mapping API names to (header_key, header_value) tuples + """ + return self._restrictions.copy() + + def Insert(self, api: str, restriction: tuple[str, str]): + """ + Add a restriction for a specific API. + + Args: + api: The API name (e.g., "inference", "model-repository") + restriction: Tuple of (header_key, header_value) for authentication + """ + self._restrictions[api] = restriction + + def IsRestricted(self, api: str) -> bool: + """ + Check if a specific API is restricted. + + Args: + api: The API name to check + + Returns: + bool: True if the API is restricted, False otherwise + """ + return api in self._restrictions + + +class APIRestrictionMiddleware(BaseHTTPMiddleware): + """ + Middleware to restrict API endpoint access based on allowed APIs configuration. + + This middleware intercepts HTTP requests and checks if they match any restricted + API endpoints. If a request matches a restricted endpoint, it validates the + authentication headers before allowing the request to proceed. + + Similar to Triton Server's endpoint access control feature. + """ + + def __init__(self, app, restricted_apis: RestrictedFeatures): + """ + Initialize the API restriction middleware. + + Args: + app: The FastAPI application instance + restricted_apis: RestrictedFeatures instance containing the restriction configuration + """ + super().__init__(app) + self.restricted_apis = restricted_apis + + def _get_auth_header(self, request: Request) -> tuple[str, str] | None: + request_method = request.method + request_path = request.url.path + + # Check each restricted API to see if the request matches + for ( + restricted_api, + auth_spec, + ) in self.restricted_apis.RestrictionDict().items(): + # Check each endpoint in the API + for restricted_endpoint in ENDPOINT_MAPPING[restricted_api]: + restricted_method, restricted_path = restricted_endpoint.split(" ") + + # Match both HTTP method and path prefix + if request_method == restricted_method and request_path.startswith( + restricted_path + ): + return auth_spec + return None + + async def dispatch(self, request: Request, call_next): + """ + Main middleware dispatch method that processes each incoming request. + + Args: + request: The incoming HTTP request + call_next: The next middleware/handler in the chain + + Returns: + Response: Either the next handler's response or a 401 authentication error + """ + # Check if the request matches any restricted patterns + auth_header = self._get_auth_header(request) + + # If request not restricted, proceed with the request + if not auth_header: + return await call_next(request) + + # Check authentication for the matching restricted endpoint + auth_result = self._check_authentication(request, auth_header) + if auth_result["valid"]: + # Authentication passed, allow request to proceed + return await call_next(request) + else: + # Authentication failed, return 401 error + return JSONResponse( + status_code=401, + content={ + "error": { + "message": auth_result["message"], + "type": "authentication_error", + "code": "invalid_auth", + } + }, + ) + + def _check_authentication(self, request: Request, auth_header: tuple[str, str]): + """ + Check if the request contains valid authentication headers. + + Args: + request: The incoming HTTP request + auth_header: Tuple of (expected_header_key, expected_header_value) + + Returns: + dict: {"valid": bool, "message": str} - Authentication result and error message if invalid + """ + expected_key, expected_value = auth_header + + # Get the actual header value from the request + actual_value = request.headers.get(expected_key) + + # Validate the header value matches the expected value + if not actual_value or actual_value != expected_value: + return { + "valid": False, + "message": f"This API is restricted, expecting header '{expected_key}' with valid value", + } + + return {"valid": True} diff --git a/python/openai/openai_frontend/frontend/fastapi_frontend.py b/python/openai/openai_frontend/frontend/fastapi_frontend.py index adee4cbab3..2234d84208 100644 --- a/python/openai/openai_frontend/frontend/fastapi_frontend.py +++ b/python/openai/openai_frontend/frontend/fastapi_frontend.py @@ -1,4 +1,4 @@ -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -30,6 +30,10 @@ from engine.triton_engine import TritonLLMEngine from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from frontend.fastapi.middleware.api_restriction import ( + APIRestrictionMiddleware, + RestrictedFeatures, +) from frontend.fastapi.routers import chat, completions, models, observability from frontend.frontend import OpenAIFrontend @@ -41,10 +45,17 @@ def __init__( host: str = "localhost", port: int = 8000, log_level: str = "info", + restricted_apis: list = None, ): self.host: str = host self.port: int = port self.log_level: str = log_level + if restricted_apis: + self.restricted_apis: RestrictedFeatures = RestrictedFeatures( + restricted_apis + ) + else: + self.restricted_apis: RestrictedFeatures = None self.stopped: bool = False self.app = self._create_app() @@ -89,6 +100,8 @@ def _create_app(self): # NOTE: For debugging purposes, should generally be restricted or removed self._add_cors_middleware(app) + if self.restricted_apis != None: + self._add_api_restriction_middleware(app) return app @@ -107,3 +120,11 @@ def _add_cors_middleware(self, app: FastAPI): allow_methods=["*"], allow_headers=["*"], ) + + def _add_api_restriction_middleware(self, app: FastAPI): + app.add_middleware( + APIRestrictionMiddleware, restricted_apis=self.restricted_apis + ) + print( + f"[INFO] API restrictions enabled. Restricted API endpoints: {self.restricted_apis.RestrictionDict()}" + ) diff --git a/python/openai/openai_frontend/main.py b/python/openai/openai_frontend/main.py index 779a0d868a..88aed600d4 100755 --- a/python/openai/openai_frontend/main.py +++ b/python/openai/openai_frontend/main.py @@ -28,6 +28,7 @@ import argparse import signal +import sys from functools import partial import tritonserver @@ -162,6 +163,15 @@ def parse_args(): choices=["debug", "info", "warning", "error", "critical", "trace"], help="log level for uvicorn", ) + openai_group.add_argument( + "--openai-restricted-api", + type=str, + default=None, + nargs=3, + metavar=("APIs", "Restricted Key", "Restricted Value"), + action="append", + help="Restrict access to specific OpenAI API endpoints. Format: ',,... ' (e.g., 'inference,model-repository admin-key admin-value'). If not specified, all endpoints are allowed.", + ) # KServe Predict v2 Frontend kserve_group = parser.add_argument_group("Triton KServe Frontend") @@ -210,12 +220,20 @@ def main(): ) # Attach TritonLLMEngine as the backbone for inference and model management - openai_frontend: FastApiFrontend = FastApiFrontend( - engine=engine, - host=args.host, - port=args.openai_port, - log_level=args.uvicorn_log_level, - ) + try: + openai_frontend: FastApiFrontend = FastApiFrontend( + engine=engine, + host=args.host, + port=args.openai_port, + log_level=args.uvicorn_log_level, + restricted_apis=args.openai_restricted_api, + ) + except ValueError as e: + print( + f"[ERROR] Failed to initialize FastAPI frontend: {e}", + file=sys.stderr, + ) + sys.exit(1) # Optionally expose Triton KServe HTTP/GRPC Frontends kserve_http, kserve_grpc = None, None diff --git a/python/openai/requirements.txt b/python/openai/requirements.txt index 5536880c09..608f3b5ac3 100644 --- a/python/openai/requirements.txt +++ b/python/openai/requirements.txt @@ -29,7 +29,7 @@ fastapi==0.115.6 # Fix httpx version to avoid bug in openai library: # https://community.openai.com/t/error-with-openai-1-56-0-client-init-got-an-unexpected-keyword-argument-proxies/1040332/3 httpx==0.27.2 -openai==1.60.0 +openai==1.107.3 partial-json-parser # used for parsing partial JSON outputs # Minimum starlette version needed to address CVE: # https://github.com/advisories/GHSA-f96h-pmfr-66vw diff --git a/python/openai/tests/test_lora.py b/python/openai/tests/test_lora.py index 4a5ff10361..550273cf52 100644 --- a/python/openai/tests/test_lora.py +++ b/python/openai/tests/test_lora.py @@ -29,12 +29,50 @@ import shutil import unittest +import pytest from huggingface_hub import snapshot_download from openai import BadRequestError, NotFoundError +from openai_frontend.engine.utils.triton import ( + _get_vllm_lora_names as get_vllm_lora_names, +) from .utils import OpenAIServer +@pytest.mark.parametrize( + "model_repository,model_name,expect_error", + [ + ("openai_model_repository", "", True), # Empty string as model name. + ("openai_model_repository", " ", True), # Whitespace-only model name. + ("openai_model_repository", "invalid/path", True), + ("openai_model_repository", "invalid\\path", True), + ("openai_model_repository", "../outside/repo", True), + ("openai_model_repository", "../test_models/identity_py", True), + ("test_models", "../test_models/identity_py", True), + ("test_models", "identity_py", False), + ("test_models", "mock_llm", False), + ], +) +def test_get_vllm_lora_name(model_repository: str, model_name: str, expect_error: bool): + try: + get_vllm_lora_names(model_repository, model_name, 1) + except ValueError as e: + if expect_error: + assert ( + f"Invalid model name: '{model_name}'. Model names must be valid file-system-path segment names." + == str(e) + ) + else: + raise pytest.fail( + f"(model_repository='{model_repository}', model_name='{model_name}') raised ValueError unexpectedly: {e}" + ) + else: + if expect_error: + raise pytest.fail( + f"(model_repository='{model_repository}', model_name='{model_name}') did not raise ValueError as expected." + ) + + def is_vllm_installed(): try: import vllm as _ diff --git a/python/openai/tests/test_openai_restricted_apis.py b/python/openai/tests/test_openai_restricted_apis.py new file mode 100755 index 0000000000..8e26addb11 --- /dev/null +++ b/python/openai/tests/test_openai_restricted_apis.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 + +# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from pathlib import Path +from typing import Dict, List, Optional + +import pytest +import requests +from tests.utils import OpenAIServer + + +def assert_response_success( + response: requests.Response, expected_status: int = 200, description: str = "" +): + """Assert that a response was successful.""" + assert ( + response.status_code == expected_status + ), f"{description} should return {expected_status}, got {response.status_code}" + + +def assert_response_unauthorized( + response: requests.Response, expected_status: int = 401, description: str = "" +): + """Assert that a response was unauthorized.""" + assert ( + response.status_code == expected_status + ), f"{description} should be unauthorized with {expected_status}, got {response.status_code}" + + +def make_get_request( + base_url: str, + endpoint: str, + headers: Optional[Dict[str, str]] = None, + timeout: int = 10, +): + """Make a GET request to the specified endpoint.""" + url = f"{base_url}{endpoint}" + response = requests.get(url, headers=headers, timeout=timeout) + return response + + +def verify_inference_endpoints( + base_url, model, headers, expected_success, description_prefix +): + def make_chat_request( + base_url: str, + model: str, + messages: List[Dict[str, str]], + headers: Optional[Dict[str, str]] = None, + max_tokens: int = 10, + timeout: int = 10, + ): + """Make a POST request to the chat completions endpoint.""" + url = f"{base_url}/v1/chat/completions" + data = { + "model": model, + "messages": messages, + "max_tokens": max_tokens, + } + response = requests.post(url, json=data, headers=headers, timeout=timeout) + return response + + def make_completion_request( + base_url: str, + model: str, + prompt: str, + headers: Optional[Dict[str, str]] = None, + max_tokens: int = 10, + timeout: int = 10, + ): + """Make a POST request to the completions endpoint.""" + url = f"{base_url}/v1/completions" + data = { + "model": model, + "prompt": prompt, + "max_tokens": max_tokens, + } + response = requests.post(url, json=data, headers=headers, timeout=timeout) + return response + + messages = [{"role": "user", "content": "Hello"}] + response = make_chat_request(base_url, model, messages, headers=headers) + if expected_success: + assert_response_success( + response, description=f"{description_prefix} Chat completions endpoint" + ) + else: + assert_response_unauthorized( + response, description=f"{description_prefix} Chat completions endpoint" + ) + + prompt = "Hello" + response = make_completion_request(base_url, model, prompt, headers=headers) + if expected_success: + assert_response_success( + response, description=f"{description_prefix} Completions endpoint" + ) + else: + assert_response_unauthorized( + response, description=f"{description_prefix} Completions endpoint" + ) + + +def verify_model_repository_endpoints( + base_url, model, headers, expected_success, description_prefix +): + response = make_get_request(base_url, "/v1/models", headers=headers) + if expected_success: + assert_response_success( + response, description=f"{description_prefix} Models endpoint" + ) + else: + assert_response_unauthorized( + response, description=f"{description_prefix} Models endpoint" + ) + + response = make_get_request(base_url, f"/v1/models/{model}", headers=headers) + if expected_success: + assert_response_success( + response, description=f"{description_prefix} Specific model endpoint" + ) + else: + assert_response_unauthorized( + response, description=f"{description_prefix} Specific model endpoint" + ) + + +def verify_metrics_endpoint(base_url, headers, expected_success, description_prefix): + # Test metrics endpoint + response = make_get_request(base_url, "/metrics", headers=headers) + assert_response_success(response, description="Unrestricted Metrics endpoint") + + if expected_success: + assert_response_success( + response, description=f"{description_prefix} Metrics endpoint" + ) + else: + assert_response_unauthorized( + response, description=f"{description_prefix} Metrics endpoint" + ) + + +def verify_health_endpoint(base_url, headers, expected_success, description_prefix): + # Test health endpoint + response = make_get_request(base_url, "/health/ready", headers=headers) + if expected_success: + assert_response_success( + response, description=f"{description_prefix} Health endpoint" + ) + else: + assert_response_unauthorized( + response, description=f"{description_prefix} Health endpoint" + ) + + +@pytest.mark.openai +class TestRestrictedAPIInvalidArguments: + """Test cases for malformed --openai-restricted-api arguments.""" + + def _test_server_startup_failure( + self, + malformed_api_arg, + expected_error_pattern=None, + ): + """Helper method to test that server fails to start with malformed arguments.""" + args = [ + "--model-repository", + str( + Path(__file__).parent / f"test_models" + ), # Hardcode to simple models to speed up tests + ] + if type(malformed_api_arg[0]) == list: + for api_arg in malformed_api_arg: + args.append("--openai-restricted-api") + args.extend(api_arg) + else: + args.append("--openai-restricted-api") + args.extend(malformed_api_arg) + + # Server should fail to start with malformed arguments + with pytest.raises((ValueError, Exception)) as exc_info: + with OpenAIServer(args) as openai_server: + pass # Should not reach here + + if expected_error_pattern: + assert expected_error_pattern in str( + exc_info.value + ), f"Expected error pattern '{expected_error_pattern}' not found in: {exc_info.value}" + + @pytest.mark.parametrize( + "malformed_arg", + [ + ["unknown-endpoint", "auth-key", "auth-value"], + ["invalid,inference", "auth-key", "auth-value"], # Mix of invalid and valid + ["inference,unknown", "auth-key", "auth-value"], # Mix of valid and invalid + ], + ) + def test_unknown_endpoint_names(self, malformed_arg): + """Test that server handles unknown endpoint names gracefully.""" + self._test_server_startup_failure( + malformed_arg, + expected_error_pattern="Unknown API", + ) + + @pytest.mark.parametrize( + "malformed_arg", + [ + ["inference,inference", "auth-key", "auth-value"], + ], + ) + def test_duplicate_apis(self, malformed_arg): + """Test that server handles duplicate APIs gracefully.""" + self._test_server_startup_failure( + malformed_arg, + expected_error_pattern="restricted api 'inference' can not be specified in multiple config groups", + ) + + @pytest.mark.parametrize( + "malformed_arg", + [ + # API with different auth specs + [ + ["inference", "auth-key1", "value1"], + ["inference", "auth-key2", "value2"], + ], + # API with same auth specs + [["inference", "auth-key", "value"], ["inference", "auth-key", "value"]], + # Multiple APIs with one duplicate + [ + ["inference", "auth-key1", "value1"], + ["model-repository", "auth-key2", "value2"], + ["inference", "auth-key3", "value3"], + ], + # All APIs duplicated + [ + ["inference", "auth-key1", "value1"], + ["model-repository", "auth-key2", "value2"], + ["inference", "auth-key3", "value3"], + ["model-repository", "auth-key4", "value4"], + ], + ], + ) + def test_conflict_configs(self, malformed_arg): + """Test that server fails when duplicate APIs are specified in multiple arguments.""" + # Test cases where the same API name appears in multiple --openai-restricted-api arguments + self._test_server_startup_failure( + malformed_arg, + expected_error_pattern="restricted api 'inference' can not be specified in multiple config groups", + ) + + +@pytest.mark.openai +class TestOpenAIServerRestrictedAPIs: + """Test cases for OpenAI server with restricted APIs functionality.""" + + @pytest.fixture(scope="class") + def server_with_restrictions(self, model_repository, tokenizer_model, backend): + """Start server with restricted APIs enabled.""" + args = [ + "--model-repository", + model_repository, + "--tokenizer", + tokenizer_model, + "--backend", + backend, + "--openai-restricted-api", + "inference,model-repository", + "admin-key", + "admin-value", + ] + + with OpenAIServer(args) as openai_server: + yield openai_server + + @pytest.mark.parametrize( + "headers, expected_success, description", + [ + (None, False, "No auth"), + ({"admin-key": "admin-value"}, True, "Valid auth"), + ({"admin-key": "wrong-value"}, False, "Invalid auth value"), + ({"wrong-key": "admin-value"}, False, "Invalid auth key"), + ], + ) + def test_restricted_endpoints_with_auth( + self, server_with_restrictions, model, headers, expected_success, description + ): + """Test restricted endpoints with different authentication scenarios.""" + base_url = server_with_restrictions.url_root + + verify_model_repository_endpoints( + base_url, model, headers, expected_success, description + ) + verify_inference_endpoints( + base_url, model, headers, expected_success, description + ) + + def test_unrestricted_endpoints(self, server_with_restrictions): + """Test that unrestricted endpoints work without authentication.""" + base_url = server_with_restrictions.url_root + + verify_metrics_endpoint( + base_url, None, expected_success=True, description_prefix="Unrestricted" + ) + verify_health_endpoint( + base_url, None, expected_success=True, description_prefix="Unrestricted" + ) + + +@pytest.mark.openai +class TestOpenAIServerMultipleRestrictions: + """Test cases for OpenAI server with multiple restriction groups.""" + + @pytest.fixture(scope="class") + def server_multiple_restrictions(self, model_repository, tokenizer_model, backend): + """Start server with multiple restriction groups.""" + args = [ + "--model-repository", + model_repository, + "--tokenizer", + tokenizer_model, + "--backend", + backend, + "--openai-restricted-api", + "model-repository", + "model-key", + "model-value", + "--openai-restricted-api", + "inference", + "infer-key", + "infer-value", + ] + + with OpenAIServer(args) as openai_server: + yield openai_server + + def test_endpoint_groups_with_correct_auth( + self, server_multiple_restrictions, model + ): + """Test that endpoint groups work with their specific authentication keys.""" + base_url = server_multiple_restrictions.url_root + + # Test model repository endpoints with model key + model_headers = {"model-key": "model-value"} + verify_model_repository_endpoints( + base_url, + model, + model_headers, + expected_success=True, + description_prefix="Correct model key", + ) + + # Test inference endpoints with inference key + infer_headers = {"infer-key": "infer-value"} + verify_inference_endpoints( + base_url, + model, + infer_headers, + expected_success=True, + description_prefix="Correct inference key", + ) + + @pytest.mark.parametrize( + "model_headers, model_description, infer_headers, infer_description", + [ + (None, "No auth", None, "No auth"), + ( + {"infer-key": "infer-value"}, + "Model key for inference endpoints", + {"model-key": "model-value"}, + "Inference key for model endpoints", + ), + ( + {"wrong-key": "wrong-value"}, + "Completely wrong key", + {"wrong-key": "wrong-value"}, + "Completely wrong key", + ), + ], + ) + def test_endpoint_groups_with_wrong_auth( + self, + server_multiple_restrictions, + model, + model_headers, + model_description, + infer_headers, + infer_description, + ): + """Test that endpoint groups are blocked with wrong authentication keys.""" + base_url = server_multiple_restrictions.url_root + + # Test scenarios where wrong auth keys are used + verify_model_repository_endpoints( + base_url, + model, + model_headers, + expected_success=False, + description_prefix=model_description, + ) + verify_inference_endpoints( + base_url, + model, + infer_headers, + expected_success=False, + description_prefix=infer_description, + ) + + def test_unrestricted_endpoints(self, server_multiple_restrictions): + """Test that unrestricted endpoints work without authentication.""" + base_url = server_multiple_restrictions.url_root + + verify_metrics_endpoint( + base_url, None, expected_success=True, description_prefix="Unrestricted" + ) + verify_health_endpoint( + base_url, None, expected_success=True, description_prefix="Unrestricted" + ) diff --git a/python/openai/tests/utils.py b/python/openai/tests/utils.py index 4bacca6f99..1743725a4e 100644 --- a/python/openai/tests/utils.py +++ b/python/openai/tests/utils.py @@ -93,7 +93,7 @@ def __init__( ["python3", script_path] + cli_args, env=env, stdout=sys.stdout, - stderr=sys.stderr, + stderr=subprocess.PIPE, # Capture stderr ) # Wait until health endpoint is responsive self._wait_for_server( @@ -121,7 +121,19 @@ def _wait_for_server(self, *, url: str, timeout: float): except Exception as err: result = self.proc.poll() if result is not None and result != 0: - raise RuntimeError("Server exited unexpectedly.") from err + # Capture stderr output for better error messages + stderr_output = "" + try: + stderr_output = self.proc.stderr.read().decode("utf-8").strip() + except Exception: + pass # If we can't read stderr, just use the generic message + + if stderr_output: + raise RuntimeError( + f"Server exited unexpectedly: {stderr_output}" + ) from err + else: + raise RuntimeError("Server exited unexpectedly.") from err time.sleep(0.5) if time.time() - start > timeout: diff --git a/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json b/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json index 835e537349..b7ce0ee199 100644 --- a/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json +++ b/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json @@ -1 +1 @@ -{"model": "mistralai/Mistral-Nemo-Instruct-2407", "disable_log_requests": true, "gpu_memory_utilization": 0.9} \ No newline at end of file +{"model": "mistralai/Mistral-Nemo-Instruct-2407", "gpu_memory_utilization": 0.9} \ No newline at end of file diff --git a/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json b/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json index cb9b14c765..df85a05da0 100644 --- a/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json +++ b/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json @@ -1 +1 @@ -{"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "disable_log_requests": true, "gpu_memory_utilization": 0.9} +{"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "gpu_memory_utilization": 0.9} diff --git a/qa/L0_backend_python/setup_python_enviroment.sh b/qa/L0_backend_python/setup_python_enviroment.sh index cfa56cba51..97901e1bb6 100755 --- a/qa/L0_backend_python/setup_python_enviroment.sh +++ b/qa/L0_backend_python/setup_python_enviroment.sh @@ -99,6 +99,7 @@ echo "python environment 3.${PYTHON_ENV_VERSION}" # copy the stub out to /opt/tritonserver/backends/python/triton_python_backend_stub cp python_backend/builddir/triton_python_backend_stub /opt/tritonserver/backends/python/triton_python_backend_stub # Set up environment and stub for each test +apt-get update -qq && apt-get install -y software-properties-common add-apt-repository ppa:deadsnakes/ppa -y apt-get update && apt-get -y install \ "python3.${PYTHON_ENV_VERSION}-dev" \ diff --git a/qa/L0_client_build_variants/test.sh b/qa/L0_client_build_variants/test.sh index ffba174e8e..cafc55acc7 100755 --- a/qa/L0_client_build_variants/test.sh +++ b/qa/L0_client_build_variants/test.sh @@ -138,87 +138,6 @@ else exit 1 fi -# TODO: TPRD-342 These tests should be PA CI test -# cases not Triton test cases -rm -fr /workspace/build -mkdir -p /workspace/build -# -# Build without C API in Perf Analyzer -# -(cd /workspace/build && \ - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && \ - cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ - -DTRITON_ENABLE_CC_HTTP=ON \ - -DTRITON_ENABLE_CC_GRPC=ON \ - -DTRITON_ENABLE_PERF_ANALYZER_C_API=OFF \ - -DTRITON_ENABLE_PERF_ANALYZER_TFS=ON \ - -DTRITON_ENABLE_PERF_ANALYZER_TS=ON \ - -DTRITON_ENABLE_GPU=ON \ - -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ - -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} \ - -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} \ - -DTRITON_CLIENT_REPO_TAG=${TRITON_CLIENT_REPO_TAG} \ - /workspace/perf_analyzer && \ - make -j16 perf-analyzer) -if [ $? -eq 0 ]; then - echo -e "\n***\n*** No-CAPI Passed\n***" -else - echo -e "\n***\n*** No-CAPI FAILED\n***" - exit 1 -fi - -# -# Build without TensorFlow Serving in Perf Analyzer -# -(cd /workspace/build && \ - rm -fr cc_clients perf_analyzer && \ - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && \ - cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ - -DTRITON_ENABLE_CC_HTTP=ON \ - -DTRITON_ENABLE_CC_GRPC=ON \ - -DTRITON_ENABLE_PERF_ANALYZER_C_API=ON \ - -DTRITON_ENABLE_PERF_ANALYZER_TFS=OFF \ - -DTRITON_ENABLE_PERF_ANALYZER_TS=ON \ - -DTRITON_ENABLE_GPU=ON \ - -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ - -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} \ - -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} \ - -DTRITON_CLIENT_REPO_TAG=${TRITON_CLIENT_REPO_TAG} \ - /workspace/perf_analyzer && \ - make -j16 perf-analyzer) -if [ $? -eq 0 ]; then - echo -e "\n***\n*** No-TF-Serving Passed\n***" -else - echo -e "\n***\n*** No-TF-Serving FAILED\n***" - exit 1 -fi - -# -# Build without TorchServe in Perf Analyzer -# -(cd /workspace/build && \ - rm -fr cc_clients perf_analyzer && \ - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && \ - cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ - -DTRITON_ENABLE_CC_HTTP=ON \ - -DTRITON_ENABLE_CC_GRPC=ON \ - -DTRITON_ENABLE_PERF_ANALYZER_C_API=ON \ - -DTRITON_ENABLE_PERF_ANALYZER_TFS=ON \ - -DTRITON_ENABLE_PERF_ANALYZER_TS=OFF \ - -DTRITON_ENABLE_GPU=ON \ - -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ - -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} \ - -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} \ - -DTRITON_CLIENT_REPO_TAG=${TRITON_CLIENT_REPO_TAG} \ - /workspace/perf_analyzer && \ - make -j16 perf-analyzer) -if [ $? -eq 0 ]; then - echo -e "\n***\n*** No-TorchServe Passed\n***" -else - echo -e "\n***\n*** No-TorchServe FAILED\n***" - exit 1 -fi - set -e echo -e "\n***\n*** Test Passed\n***" diff --git a/qa/L0_infer/infer_test.py b/qa/L0_infer/infer_test.py index 8f16a7fe10..3d01160b2e 100755 --- a/qa/L0_infer/infer_test.py +++ b/qa/L0_infer/infer_test.py @@ -42,6 +42,7 @@ TEST_CUDA_SHARED_MEMORY = bool(int(os.environ.get("TEST_CUDA_SHARED_MEMORY", 0))) CPU_ONLY = os.environ.get("TRITON_SERVER_CPU_ONLY") is not None TEST_VALGRIND = bool(int(os.environ.get("TEST_VALGRIND", 0))) +VALGRIND_TESTS = bool(int(os.environ.get("VALGRIND_TESTS", 0))) USE_GRPC = os.environ.get("USE_GRPC", 1) != "0" USE_HTTP = os.environ.get("USE_HTTP", 1) != "0" @@ -69,7 +70,6 @@ def _full_exact( output0_raw, output1_raw, swap, - network_timeout=NETWORK_TIMEOUT, ): def _infer_exact_helper( tester, @@ -163,31 +163,6 @@ def _infer_exact_helper( ): ensemble_prefix.append(prefix) - if tu.validate_for_onnx_model( - input_dtype, - output0_dtype, - output1_dtype, - (input_size,), - (input_size,), - (input_size,), - ): - for prefix in ensemble_prefix: - for pf in ["onnx"]: - if pf in BACKENDS: - _infer_exact_helper( - self, - prefix + pf, - (input_size,), - 8, - input_dtype, - output0_dtype, - output1_dtype, - output0_raw=output0_raw, - output1_raw=output1_raw, - swap=swap, - network_timeout=network_timeout, - ) - if not CPU_ONLY and tu.validate_for_trt_model( input_dtype, output0_dtype, @@ -666,16 +641,59 @@ def test_mix_iff(self): swap=False, ) - def test_raw_version_latest_1(self): - input_size = 16 - tensor_shape = (1, input_size) + if not VALGRIND_TESTS: + + def test_raw_version_latest_1(self): + input_size = 16 + tensor_shape = (1, input_size) + + # There are 3 versions of onnx_int8_int8_int8 but + # only version 3 should be available + for platform in ["onnx"]: + if platform not in BACKENDS: + continue + try: + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.int8, + np.int8, + np.int8, + model_version=1, + swap=False, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + except InferenceServerException as ex: + self.assertTrue( + ex.message().startswith("Request for unknown model") + ) + + try: + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.int8, + np.int8, + np.int8, + model_version=2, + swap=True, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + except InferenceServerException as ex: + self.assertTrue( + ex.message().startswith("Request for unknown model") + ) - # There are 3 versions of onnx_int8_int8_int8 but - # only version 3 should be available - for platform in ["onnx"]: - if platform not in BACKENDS: - continue - try: iu.infer_exact( self, platform, @@ -684,25 +702,52 @@ def test_raw_version_latest_1(self): np.int8, np.int8, np.int8, - model_version=1, - swap=False, + model_version=3, + swap=True, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - except InferenceServerException as ex: - self.assertTrue(ex.message().startswith("Request for unknown model")) - try: + def test_raw_version_latest_2(self): + input_size = 16 + tensor_shape = (1, input_size) + + # There are 3 versions of onnx_int16_int16_int16 but only + # versions 2 and 3 should be available + for platform in ["onnx"]: + if platform not in BACKENDS: + continue + try: + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.int16, + np.int16, + np.int16, + model_version=1, + swap=False, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + except InferenceServerException as ex: + self.assertTrue( + ex.message().startswith("Request for unknown model") + ) + iu.infer_exact( self, platform, tensor_shape, 1, - np.int8, - np.int8, - np.int8, + np.int16, + np.int16, + np.int16, model_version=2, swap=True, use_http=USE_HTTP, @@ -710,35 +755,6 @@ def test_raw_version_latest_1(self): use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - except InferenceServerException as ex: - self.assertTrue(ex.message().startswith("Request for unknown model")) - - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.int8, - np.int8, - np.int8, - model_version=3, - swap=True, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - - def test_raw_version_latest_2(self): - input_size = 16 - tensor_shape = (1, input_size) - - # There are 3 versions of onnx_int16_int16_int16 but only - # versions 2 and 3 should be available - for platform in ["onnx"]: - if platform not in BACKENDS: - continue - try: iu.infer_exact( self, platform, @@ -747,136 +763,46 @@ def test_raw_version_latest_2(self): np.int16, np.int16, np.int16, - model_version=1, - swap=False, + model_version=3, + swap=True, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - except InferenceServerException as ex: - self.assertTrue(ex.message().startswith("Request for unknown model")) - - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.int16, - np.int16, - np.int16, - model_version=2, - swap=True, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.int16, - np.int16, - np.int16, - model_version=3, - swap=True, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - - def test_raw_version_all(self): - input_size = 16 - tensor_shape = (1, input_size) - # There are 3 versions of *_int32_int32_int32 and all should - # be available. - for platform in ["onnx"]: - if platform not in BACKENDS: - continue - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.int32, - np.int32, - np.int32, - model_version=1, - swap=False, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.int32, - np.int32, - np.int32, - model_version=2, - swap=True, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.int32, - np.int32, - np.int32, - model_version=3, - swap=True, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - - def test_raw_version_specific_1(self): - input_size = 16 - tensor_shape = (1, input_size) - - # There are 3 versions of *_float16_float16_float16 but only - # version 1 should be available. - for platform in ["onnx"]: - if platform not in BACKENDS: - continue - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.float16, - np.float16, - np.float16, - model_version=1, - swap=False, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) + def test_raw_version_all(self): + input_size = 16 + tensor_shape = (1, input_size) - try: + # There are 3 versions of onnx_int32_int32_int32 and all should + # be available. + for platform in ["onnx"]: + if platform not in BACKENDS: + continue iu.infer_exact( self, platform, tensor_shape, 1, - np.float16, - np.float16, - np.float16, + np.int32, + np.int32, + np.int32, + model_version=1, + swap=False, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.int32, + np.int32, + np.int32, model_version=2, swap=True, use_http=USE_HTTP, @@ -884,18 +810,14 @@ def test_raw_version_specific_1(self): use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - except InferenceServerException as ex: - self.assertTrue(ex.message().startswith("Request for unknown model")) - - try: iu.infer_exact( self, platform, tensor_shape, 1, - np.float16, - np.float16, - np.float16, + np.int32, + np.int32, + np.int32, model_version=3, swap=True, use_http=USE_HTTP, @@ -903,143 +825,149 @@ def test_raw_version_specific_1(self): use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - except InferenceServerException as ex: - self.assertTrue(ex.message().startswith("Request for unknown model")) - def test_raw_version_specific_1_3(self): - input_size = 16 - - # There are 3 versions of *_float32_float32_float32 but only - # versions 1 and 3 should be available. - for platform in ("onnx", "plan"): - if platform == "plan" and CPU_ONLY: - continue - if platform not in BACKENDS: - continue + def test_raw_version_specific_1(self): + input_size = 16 tensor_shape = (1, input_size) - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.float32, - np.float32, - np.float32, - model_version=1, - swap=False, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - try: + # There are 3 versions of onnx_float16_float16_float16 but only + # version 1 should be available. + for platform in ["onnx"]: + if platform not in BACKENDS: + continue iu.infer_exact( self, platform, tensor_shape, 1, - np.float32, - np.float32, - np.float32, - model_version=2, - swap=True, + np.float16, + np.float16, + np.float16, + model_version=1, + swap=False, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - except InferenceServerException as ex: - self.assertTrue(ex.message().startswith("Request for unknown model")) - - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.float32, - np.float32, - np.float32, - model_version=3, - swap=True, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - if ENSEMBLES: - if all(x in BACKENDS for x in ["onnx", "plan"]): - - def test_ensemble_mix_platform(self): - # Skip on CPU only machine as TensorRT model is used in this ensemble - if CPU_ONLY: - return - for bs in (1, 8): + try: iu.infer_exact( self, - "mix_platform", - (bs, 16), - bs, - np.float32, - np.float32, - np.float32, + platform, + tensor_shape, + 1, + np.float16, + np.float16, + np.float16, + model_version=2, + swap=True, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) + except InferenceServerException as ex: + self.assertTrue( + ex.message().startswith("Request for unknown model") + ) - if "onnx" in BACKENDS: - - def test_ensemble_mix_type(self): - for bs in (1, 8): + try: iu.infer_exact( self, - "mix_type", - (bs, 16), - bs, - np.int32, - np.float32, - np.float32, + platform, + tensor_shape, + 1, + np.float16, + np.float16, + np.float16, + model_version=3, + swap=True, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) + except InferenceServerException as ex: + self.assertTrue( + ex.message().startswith("Request for unknown model") + ) - if all(x in BACKENDS for x in ["onnx", "plan"]): + def test_raw_version_specific_1_3(self): + input_size = 16 + + # There are 3 versions of *_float32_float32_float32 but only + # versions 1 and 3 should be available. + for platform in ("onnx", "plan"): + if platform == "plan" and CPU_ONLY: + continue + if platform not in BACKENDS: + continue + tensor_shape = (1, input_size) + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.float32, + np.float32, + np.float32, + model_version=1, + swap=False, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) - def test_ensemble_mix_ensemble(self): - for bs in (1, 8): + try: iu.infer_exact( self, - "mix_ensemble", - (bs, 16), - bs, - np.int32, + platform, + tensor_shape, + 1, + np.float32, np.float32, np.float32, + model_version=2, + swap=True, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) + except InferenceServerException as ex: + self.assertTrue( + ex.message().startswith("Request for unknown model") + ) - if all( - x in BACKENDS - for x in [ - "onnx", - ] - ): + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.float32, + np.float32, + np.float32, + model_version=3, + swap=True, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + + if ENSEMBLES: + if all(x in BACKENDS for x in ["onnx", "libtorch"]): - def test_ensemble_mix_batch_nobatch(self): - base_names = ["batch_to_nobatch", "nobatch_to_batch"] - for name in base_names: + def test_ensemble_mix_platform(self): + # Skip on CPU only machine as TensorRT model is used in this ensemble + if CPU_ONLY: + return for bs in (1, 8): iu.infer_exact( self, - name, + "mix_platform", (bs, 16), bs, np.float32, @@ -1050,60 +978,28 @@ def test_ensemble_mix_batch_nobatch(self): use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - iu.infer_exact( - self, - name + "_nobatch", - (8, 16), - 1, - np.float32, - np.float32, - np.float32, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - - # batch -> nobatch -> batch - for bs in (1, 8): - iu.infer_exact( - self, - "mix_nobatch_batch", - (bs, 16), - bs, - np.float32, - np.float32, - np.float32, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - if not (TEST_SYSTEM_SHARED_MEMORY or TEST_CUDA_SHARED_MEMORY): + if "onnx" in BACKENDS: - def test_ensemble_label_lookup(self): - if all(x in BACKENDS for x in ["onnx", "plan"]): - # Ensemble needs to look up label from the actual model + def test_ensemble_mix_type(self): for bs in (1, 8): iu.infer_exact( self, - "mix_platform", + "mix_type", (bs, 16), bs, + np.int32, np.float32, np.float32, - np.float32, - output0_raw=False, - output1_raw=False, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - if all(x in BACKENDS for x in ["onnx", "plan"]): - # Label from the actual model will be passed along the nested ensemble + if all(x in BACKENDS for x in ["onnx", "libtorch"]): + + def test_ensemble_mix_ensemble(self): for bs in (1, 8): iu.infer_exact( self, @@ -1113,55 +1009,148 @@ def test_ensemble_label_lookup(self): np.int32, np.float32, np.float32, - output0_raw=False, - output1_raw=False, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - if "onnx" in BACKENDS: - # If label file is provided, it will use the provided label file directly - try: + if all( + x in BACKENDS + for x in [ + "onnx", + ] + ): + + def test_ensemble_mix_batch_nobatch(self): + base_names = ["batch_to_nobatch", "nobatch_to_batch"] + for name in base_names: + for bs in (1, 8): + iu.infer_exact( + self, + name, + (bs, 16), + bs, + np.float32, + np.float32, + np.float32, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) iu.infer_exact( self, - "wrong_label", - (1, 16), + name + "_nobatch", + (8, 16), 1, - np.int32, np.float32, np.float32, - output0_raw=False, - output1_raw=False, + np.float32, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - except AssertionError: - # Sanity check that infer_exact failed since this ensemble is provided - # with unexpected labels - pass - if "onnx" in BACKENDS: + # batch -> nobatch -> batch for bs in (1, 8): iu.infer_exact( self, - "label_override", + "mix_nobatch_batch", (bs, 16), bs, - np.int32, np.float32, np.float32, - output0_raw=False, - output1_raw=False, + np.float32, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) + if not (TEST_SYSTEM_SHARED_MEMORY or TEST_CUDA_SHARED_MEMORY): + + def test_ensemble_label_lookup(self): + if all(x in BACKENDS for x in ["onnx", "libtorch"]): + # Ensemble needs to look up label from the actual model + for bs in (1, 8): + iu.infer_exact( + self, + "mix_platform", + (bs, 16), + bs, + np.float32, + np.float32, + np.float32, + output0_raw=False, + output1_raw=False, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + + if all(x in BACKENDS for x in ["onnx", "libtorch"]): + # Label from the actual model will be passed along the nested ensemble + for bs in (1, 8): + iu.infer_exact( + self, + "mix_ensemble", + (bs, 16), + bs, + np.int32, + np.float32, + np.float32, + output0_raw=False, + output1_raw=False, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + + if "onnx" in BACKENDS: + # If label file is provided, it will use the provided label file directly + try: + iu.infer_exact( + self, + "wrong_label", + (1, 16), + 1, + np.int32, + np.float32, + np.float32, + output0_raw=False, + output1_raw=False, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + except AssertionError: + # Sanity check that infer_exact failed since this ensemble is provided + # with unexpected labels + pass + + if "onnx" in BACKENDS: + for bs in (1, 8): + iu.infer_exact( + self, + "label_override", + (bs, 16), + bs, + np.int32, + np.float32, + np.float32, + output0_raw=False, + output1_raw=False, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + if __name__ == "__main__": unittest.main() diff --git a/qa/L0_infer/test.sh b/qa/L0_infer/test.sh index 40625ae6bd..72df50bd41 100755 --- a/qa/L0_infer/test.sh +++ b/qa/L0_infer/test.sh @@ -349,7 +349,7 @@ done # separately to reduce the loading time. if [ "$TEST_VALGRIND" -eq 1 ]; then TESTING_BACKENDS="python python_dlpack onnx" - EXPECTED_NUM_TESTS=42 + EXPECTED_NUM_TESTS=36 if [[ "aarch64" != $(uname -m) ]] ; then pip3 install torch==2.3.1+cpu -f https://download.pytorch.org/whl/torch_stable.html else @@ -364,9 +364,11 @@ if [ "$TEST_VALGRIND" -eq 1 ]; then mkdir nobatch_models mv ./models/*nobatch_* ./nobatch_models/. cp -fr ./models/nop_* ./nobatch_models/. - # These two models are required by test_ensemble_mix_batch_nobatch test case. - cp -fr ./models/onnx_float32_float32_float32 ./nobatch_models/. - cp -fr ./models/custom_zero_1_float32 ./nobatch_models/. + if [[ $BACKENDS == *"onnx"* ]]; then + # These two models are required by test_ensemble_mix_batch_nobatch test case. + cp -fr ./models/onnx_float32_float32_float32 ./nobatch_models/. + cp -fr ./models/custom_zero_1_float32 ./nobatch_models/. + fi for BATCHING_MODE in batch nobatch; do if [ "$TRITON_SERVER_CPU_ONLY" == "1" ]; then @@ -406,7 +408,7 @@ if [ "$TEST_VALGRIND" -eq 1 ]; then set +e - python3 $INFER_TEST >$CLIENT_LOG 2>&1 + VALGRIND_TESTS="1" python3 $INFER_TEST >$CLIENT_LOG 2>&1 if [ $? -ne 0 ]; then cat $CLIENT_LOG RET=1 diff --git a/qa/L0_long_running_stress/scenarios.py b/qa/L0_long_running_stress/scenarios.py index 5b952f5b97..342b8a6937 100755 --- a/qa/L0_long_running_stress/scenarios.py +++ b/qa/L0_long_running_stress/scenarios.py @@ -99,7 +99,7 @@ def run(self, client_metadata): class PerfAnalyzerScenario(Scenario): # Some class static variables - command_ = "../clients/perf_analyzer" + command_ = "perf_analyzer" generation_mutex_ = threading.Lock() class ModelOption: diff --git a/qa/L0_memory_growth/test.sh b/qa/L0_memory_growth/test.sh index fe4911c9bc..c99e931175 100755 --- a/qa/L0_memory_growth/test.sh +++ b/qa/L0_memory_growth/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -42,7 +42,8 @@ fi export CUDA_VISIBLE_DEVICES=0 # Clients -PERF_ANALYZER=../clients/perf_analyzer +pip3 install perf_analyzer +PERF_ANALYZER=perf_analyzer IMAGE=../images/vulture.jpeg # Models @@ -101,7 +102,7 @@ export MAX_ALLOWED_ALLOC="100" # Create local model repository mkdir -p models/ -cp -r $DATADIR/perf_model_store/resnet50* models/ +cp -r $DATADIR/perf_model_store/resnet50_* models/ # Create the TensorRT plan from ONNX model rm -fr models/resnet50_fp32_plan && mkdir -p models/resnet50_fp32_plan/1 && \ diff --git a/qa/L0_mlflow/test.sh b/qa/L0_mlflow/test.sh index 4b5205ba25..1bf7bc18f8 100755 --- a/qa/L0_mlflow/test.sh +++ b/qa/L0_mlflow/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -39,7 +39,11 @@ rm -fr *.log *.json # install a higher version of python which uses blinker 1.6, # but it is unknown whether this test should rely on # the default installation of python. -apt remove -y python3-blinker + +apt update -qq && apt install python3-venv -y +python3 -m venv .venv + +source .venv/bin/activate RET=0 diff --git a/qa/L0_openai/generate_engine.py b/qa/L0_openai/generate_engine.py index b71e084b03..83ea35a88d 100644 --- a/qa/L0_openai/generate_engine.py +++ b/qa/L0_openai/generate_engine.py @@ -25,7 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from argparse import ArgumentParser -from tensorrt_llm import LLM, BuildConfig +from tensorrt_llm import BuildConfig +from tensorrt_llm._tensorrt_engine import LLM from tensorrt_llm.plugin import PluginConfig diff --git a/qa/L0_orca/test.sh b/qa/L0_orca/test.sh index 03f5118bd2..50b7f450fb 100755 --- a/qa/L0_orca/test.sh +++ b/qa/L0_orca/test.sh @@ -46,140 +46,7 @@ SERVER_LOG="${NAME}_server.log" SERVER_TIMEOUT=${SERVER_TIMEOUT:=120} CLIENT_PY=${BASE_DIR}/orca_http_test.py CLIENT_LOG="${NAME}_orca_http_test.log" -source ../common/util.sh - -function clone_tensorrt_llm_backend_repo { - rm -rf $TENSORRTLLM_BACKEND_DIR && mkdir $TENSORRTLLM_BACKEND_DIR - apt-get update && apt-get install git-lfs -y --no-install-recommends - git clone --single-branch --depth=1 -b ${TENSORRTLLM_BACKEND_REPO_TAG} ${TRITON_REPO_ORG}/tensorrtllm_backend.git $TENSORRTLLM_BACKEND_DIR - cd $TENSORRTLLM_BACKEND_DIR && git lfs install && git submodule update --init --recursive -} - -function build_gpt2_base_model { - # Download weights from HuggingFace Transformers - cd ${GPT_DIR} && rm -rf gpt2 && git clone https://huggingface.co/gpt2-medium gpt2 && cd gpt2 - rm pytorch_model.bin model.safetensors - if ! wget -q https://huggingface.co/gpt2-medium/resolve/main/pytorch_model.bin; then - echo "Downloading pytorch_model.bin failed." - exit 1 - fi - cd ${GPT_DIR} - - # Convert weights from HF Tranformers to FT format - python3 convert_checkpoint.py --model_dir gpt2 --dtype float16 --tp_size ${NUM_GPUS} --output_dir "./c-model/gpt2/${NUM_GPUS}-gpu/" - cd ${BASE_DIR} -} - -function build_gpt2_tensorrt_engine { - # Build TensorRT engines - cd ${GPT_DIR} - trtllm-build --checkpoint_dir "./c-model/gpt2/${NUM_GPUS}-gpu/" \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable \ - --paged_kv_cache enable \ - --gemm_plugin float16 \ - --workers "${NUM_GPUS}" \ - --output_dir "${ENGINES_DIR}" - - cd ${BASE_DIR} -} - -function replace_config_tags { - tag_to_replace="${1}" - new_value="${2}" - config_file_path="${3}" - sed -i "s|${tag_to_replace}|${new_value}|g" ${config_file_path} -} - -function prepare_model_repository { - rm -rf ${MODEL_REPOSITORY} && mkdir ${MODEL_REPOSITORY} - cp -r ${TENSORRTLLM_BACKEND_DIR}/tensorrt_llm/triton_backend/all_models/inflight_batcher_llm/* ${MODEL_REPOSITORY} - rm -rf ${MODEL_REPOSITORY}/tensorrt_llm_bls - mv "${MODEL_REPOSITORY}/ensemble" "${MODEL_REPOSITORY}/${MODEL_NAME}" - - replace_config_tags "model_version: -1" "model_version: 1" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" - replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" - replace_config_tags 'name: "ensemble"' "name: \"$MODEL_NAME\"" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" - replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" - - replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - replace_config_tags '${preprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - - replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" - replace_config_tags '${postprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" - replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" - replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" - - replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${decoupled_mode}' 'true' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${batching_strategy}' 'inflight_fused_batching' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${engine_dir}' "${ENGINES_DIR}" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${triton_backend}' "tensorrtllm" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${encoder_input_features_data_type}' "TYPE_FP32" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" -} - -# Wait until server health endpoint shows ready. Sets WAIT_RET to 0 on -# success, 1 on failure -function wait_for_server_ready() { - local wait_time_secs="${1:-30}" - shift - local spids=("$@") - - WAIT_RET=0 - - for _ in $(seq "$wait_time_secs"); do - for pid in "${spids[@]}"; do - if ! kill -0 "$pid" >/dev/null 2>&1; then - echo "=== Server not running." - WAIT_RET=1 - return - fi - done - - sleep 1 - - if curl -s --fail localhost:8000/v2/health/ready && - curl -s --fail -w "%{http_code}" -o /dev/null -d '{"log_verbose_level":1}' localhost:8000/v2/logging; then - return - fi - done - - echo "=== Timeout $wait_time_secs secs. Server not ready." - WAIT_RET=1 -} - -function run_server { - python3 ${TENSORRTLLM_BACKEND_DIR}/tensorrt_llm/triton_backend/scripts/launch_triton_server.py --world_size="${NUM_GPUS}" --model_repo="${MODEL_REPOSITORY}" >${SERVER_LOG} 2>&1 & - sleep 2 # allow time to obtain the pid(s) - # Read PIDs into an array, trimming whitespaces - readarray -t SERVER_PID < <(pgrep "tritonserver") - - wait_for_server_ready ${SERVER_TIMEOUT} "${SERVER_PID[@]}" - if [ "$WAIT_RET" != "0" ]; then - # Cleanup - kill "${SERVER_PID[@]}" >/dev/null 2>&1 || true - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi -} - -function kill_server { - pgrep tritonserver | xargs kill -SIGINT - for pid in "${SERVER_PID[@]}"; do - echo "Waiting for proc ${pid} to terminate..." - while kill -0 $pid >/dev/null 2>&1; do - sleep 1 - done - done -} +source ../common/trtllm_util.sh clone_tensorrt_llm_backend_repo build_gpt2_base_model diff --git a/qa/L0_passive_instance/test.sh b/qa/L0_passive_instance/test.sh index 8948434485..be56bb4185 100755 --- a/qa/L0_passive_instance/test.sh +++ b/qa/L0_passive_instance/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2021-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -45,7 +45,8 @@ CLIENT_LOG="./client.log" TEST_SCRIPT_PY=passive_instance_test.py EXPECTED_NUM_TESTS="1" -PERF_ANALYZER=../clients/perf_analyzer +pip3 install perf_analyzer +PERF_ANALYZER=perf_analyzer MODEL=distributed_int32_int32_int32 SERVER=/opt/tritonserver/bin/tritonserver diff --git a/qa/L0_perf_deeprecommender/run_test.sh b/qa/L0_perf_deeprecommender/run_test.sh index 434803e7a9..f973d7fea4 100755 --- a/qa/L0_perf_deeprecommender/run_test.sh +++ b/qa/L0_perf_deeprecommender/run_test.sh @@ -29,7 +29,9 @@ STATIC_BATCH_SIZES=${STATIC_BATCH_SIZES:=1} DYNAMIC_BATCH_SIZES=${DYNAMIC_BATCH_SIZES:=1} INSTANCE_COUNTS=${INSTANCE_COUNTS:=1} -PERF_CLIENT=../clients/perf_client +pip3 install perf_analyzer + +PERF_CLIENT=perf_analyzer REPORTER=../common/reporter.py SERVER=/opt/tritonserver/bin/tritonserver diff --git a/qa/L0_perf_nomodel/run_test.sh b/qa/L0_perf_nomodel/run_test.sh index fcfdf16e0e..6355b1f685 100755 --- a/qa/L0_perf_nomodel/run_test.sh +++ b/qa/L0_perf_nomodel/run_test.sh @@ -49,9 +49,10 @@ ARCH=${ARCH:="x86_64"} SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends MODEL_REPO="${PWD}/models" -PERF_CLIENT=../clients/perf_client +PERF_CLIENT=perf_analyzer SERVER_ARGS="--model-repository=${MODEL_REPO} --backend-directory=${BACKEND_DIR}" source ../common/util.sh +pip3 install perf_analyzer # DATADIR is already set in environment variable for aarch64 if [ "$ARCH" != "aarch64" ]; then diff --git a/qa/L0_perf_resnet/run_test.sh b/qa/L0_perf_resnet/run_test.sh index 3e7b048c40..6823525d82 100755 --- a/qa/L0_perf_resnet/run_test.sh +++ b/qa/L0_perf_resnet/run_test.sh @@ -53,8 +53,10 @@ rm -fr models && mkdir -p models && \ sed -i "s/^max_batch_size:.*/max_batch_size: ${MAX_BATCH}/" config.pbtxt && \ echo "instance_group [ { count: ${INSTANCE_CNT} }]") +pip3 install perf_analyzer + MEASUREMENT_WINDOW=5000 -PERF_CLIENT=../clients/perf_client +PERF_CLIENT=perf_analyzer # Onnx and onnx-trt models are very slow on Jetson. if [ "$ARCH" == "aarch64" ]; then if [ "$MODEL_FRAMEWORK" == "onnx" ] || [ "$MODEL_FRAMEWORK" == "onnx_trt" ]; then diff --git a/qa/L0_perf_tensorrt_llm/test.sh b/qa/L0_perf_tensorrt_llm/test.sh index 5cfb4ba595..5bf418c5c5 100755 --- a/qa/L0_perf_tensorrt_llm/test.sh +++ b/qa/L0_perf_tensorrt_llm/test.sh @@ -43,13 +43,7 @@ SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends SERVER_LOG="${NAME}_server.log" SERVER_TIMEOUT=${SERVER_TIMEOUT:=120} - -function clone_tensorrt_llm_backend_repo { - rm -rf $TENSORRTLLM_BACKEND_DIR && mkdir $TENSORRTLLM_BACKEND_DIR - apt-get update && apt-get install git-lfs -y --no-install-recommends - git clone --single-branch --depth=1 -b ${TENSORRTLLM_BACKEND_REPO_TAG} ${TRITON_REPO_ORG}/tensorrtllm_backend.git $TENSORRTLLM_BACKEND_DIR - cd $TENSORRTLLM_BACKEND_DIR && git lfs install && git submodule update --init --recursive -} +source ../common/trtllm_util.sh # Update Open MPI to a version compatible with SLURM. function upgrade_openmpi { @@ -95,125 +89,6 @@ function upgrade_openmpi { mpirun --version } -function build_gpt2_base_model { - # Download weights from HuggingFace Transformers - cd ${GPT_DIR} && rm -rf gpt2 && git clone https://huggingface.co/gpt2-medium gpt2 && cd gpt2 - rm pytorch_model.bin model.safetensors - if ! wget -q https://huggingface.co/gpt2-medium/resolve/main/pytorch_model.bin; then - echo "Downloading pytorch_model.bin failed." - exit 1 - fi - cd ${GPT_DIR} - - # Convert weights from HF Tranformers to FT format - python3 convert_checkpoint.py --model_dir gpt2 --dtype float16 --tp_size ${NUM_GPUS} --output_dir "./c-model/gpt2/${NUM_GPUS}-gpu/" - cd ${BASE_DIR} -} - -function build_gpt2_tensorrt_engine { - # Build TensorRT engines - cd ${GPT_DIR} - trtllm-build --checkpoint_dir "./c-model/gpt2/${NUM_GPUS}-gpu/" \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable \ - --paged_kv_cache enable \ - --gemm_plugin float16 \ - --workers "${NUM_GPUS}" \ - --output_dir "${ENGINES_DIR}" - - cd ${BASE_DIR} -} - -function replace_config_tags { - tag_to_replace="${1}" - new_value="${2}" - config_file_path="${3}" - sed -i "s|${tag_to_replace}|${new_value}|g" ${config_file_path} -} - -function prepare_model_repository { - rm -rf ${MODEL_REPOSITORY} && mkdir ${MODEL_REPOSITORY} - cp -r ${TENSORRTLLM_BACKEND_DIR}/tensorrt_llm/triton_backend/all_models/inflight_batcher_llm/* ${MODEL_REPOSITORY} - rm -rf ${MODEL_REPOSITORY}/tensorrt_llm_bls - mv "${MODEL_REPOSITORY}/ensemble" "${MODEL_REPOSITORY}/${MODEL_NAME}" - - replace_config_tags "model_version: -1" "model_version: 1" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" - replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" - replace_config_tags 'name: "ensemble"' "name: \"$MODEL_NAME\"" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" - - replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - replace_config_tags '${preprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - - replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" - replace_config_tags '${postprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" - replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" - - replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${decoupled_mode}' 'true' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${batching_strategy}' 'inflight_fused_batching' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${engine_dir}' "${ENGINES_DIR}" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${triton_backend}' "tensorrtllm" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" -} - -# Wait until server health endpoint shows ready. Sets WAIT_RET to 0 on -# success, 1 on failure -function wait_for_server_ready() { - local wait_time_secs="${1:-30}" - shift - local spids=("$@") - - WAIT_RET=0 - - for _ in $(seq "$wait_time_secs"); do - for pid in "${spids[@]}"; do - if ! kill -0 "$pid" >/dev/null 2>&1; then - echo "=== Server not running." - WAIT_RET=1 - return - fi - done - - sleep 1 - - if curl -s --fail localhost:8000/v2/health/ready && - curl -s --fail -w "%{http_code}" -o /dev/null -d '{"log_verbose_level":1}' localhost:8000/v2/logging; then - return - fi - done - - echo "=== Timeout $wait_time_secs secs. Server not ready." - WAIT_RET=1 -} - -function run_server { - python3 ${TENSORRTLLM_BACKEND_DIR}/tensorrt_llm/triton_backend/scripts/launch_triton_server.py --world_size="${NUM_GPUS}" --model_repo="${MODEL_REPOSITORY}" >${SERVER_LOG} 2>&1 & - sleep 2 # allow time to obtain the pid(s) - # Read PIDs into an array, trimming whitespaces - readarray -t SERVER_PID < <(pgrep "tritonserver") - - wait_for_server_ready ${SERVER_TIMEOUT} "${SERVER_PID[@]}" - if [ "$WAIT_RET" != "0" ]; then - # Cleanup - kill "${SERVER_PID[@]}" >/dev/null 2>&1 || true - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi -} - -function kill_server { - pgrep tritonserver | xargs kill -SIGINT - for pid in "${SERVER_PID[@]}"; do - echo "Waiting for proc ${pid} to terminate..." - while kill -0 $pid >/dev/null 2>&1; do - sleep 1 - done - done -} - upgrade_openmpi clone_tensorrt_llm_backend_repo build_gpt2_base_model diff --git a/qa/L0_perf_vllm/test.sh b/qa/L0_perf_vllm/test.sh index e1ce8cf2ed..2c9fc87cb4 100755 --- a/qa/L0_perf_vllm/test.sh +++ b/qa/L0_perf_vllm/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -47,7 +47,6 @@ rm -rf $MODEL_REPO $EXPORT_FILE *.tjson *.json *.csv mkdir -p $MODEL_REPO/$MODEL_NAME/1 echo '{ "model":"gpt2", - "disable_log_requests": "true", "gpu_memory_utilization": 0.5 }' >$MODEL_REPO/$MODEL_NAME/1/model.json diff --git a/qa/L0_pinned_memory/test.sh b/qa/L0_pinned_memory/test.sh index 48ec05b84c..beb16cd132 100755 --- a/qa/L0_pinned_memory/test.sh +++ b/qa/L0_pinned_memory/test.sh @@ -38,10 +38,12 @@ if [ ! -z "$TEST_REPO_ARCH" ]; then REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} fi +pip3 install perf_analyzer + # Use "--request-count" throughout the test to PA stability criteria and # reduce flaky failures from PA unstable measurements. REQUEST_COUNT=10 -CLIENT=../clients/perf_client +CLIENT=perf_analyzer # Only use libtorch as it accepts GPU I/O and it can handle variable shape BACKENDS=${BACKENDS:="libtorch"} diff --git a/qa/L0_response_cache/response_cache_test b/qa/L0_response_cache/response_cache_test deleted file mode 100755 index d3807fac01..0000000000 Binary files a/qa/L0_response_cache/response_cache_test and /dev/null differ diff --git a/qa/L0_response_cache/test.sh b/qa/L0_response_cache/test.sh index 7cf0f8fad6..bb372dab9f 100755 --- a/qa/L0_response_cache/test.sh +++ b/qa/L0_response_cache/test.sh @@ -426,12 +426,13 @@ if [ "$SERVER_PID" == "0" ]; then exit 1 fi +pip3 install perf_analyzer TEMP_RET=0 REPETITION=10 CONCURRENCY=20 CLIENT_BS=1 -PERF_ANALYZER=../clients/perf_analyzer +PERF_ANALYZER=perf_analyzer TEMP_CLIENT_LOG=temp_client.log set +e diff --git a/qa/L0_sdk/test.sh b/qa/L0_sdk/test.sh index 20baf31639..f309cf4c67 100755 --- a/qa/L0_sdk/test.sh +++ b/qa/L0_sdk/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -34,17 +34,13 @@ set +e RET=0 -# Check image_client and perf_client +# Check image_client and perf_analyzer if [[ ! -x "triton_client/bin/image_client" ]]; then echo -e "*** image_client executable not present\n" RET=1 fi -if [[ ! -x "triton_client/bin/perf_analyzer" ]]; then - echo -e "*** perf_analyzer executable is not present\n" - RET=1 -fi -if [[ ! -x "triton_client/bin/perf_client" ]]; then - echo -e "*** perf_client link is not present\n" +if ! command -v perf_analyzer >/dev/null 2>&1; then + echo -e "*** perf_analyzer is not installed\n" RET=1 fi @@ -179,7 +175,7 @@ python -c """import tritonclient; import tritonclient.grpc; import tritonclient. import tritonclient.utils.cuda_shared_memory; import tritonclient.utils.shared_memory""" RET=$(($RET+$?)) -EXECUTABLES="perf_analyzer perf_client" +EXECUTABLES="perf_analyzer" for l in $EXECUTABLES; do if [ $(which -a $l | grep "/usr/local/bin/$l" | wc -l) -ne 1 ]; then which -a $l diff --git a/qa/L0_trt_dynamic_shape/test.sh b/qa/L0_trt_dynamic_shape/test.sh index 43a39dd199..42928969fd 100755 --- a/qa/L0_trt_dynamic_shape/test.sh +++ b/qa/L0_trt_dynamic_shape/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -41,8 +41,10 @@ fi TEST_RESULT_FILE='test_results.txt' export CUDA_VISIBLE_DEVICES=0 +pip3 install perf_analyzer + CLIENT_LOG="./client.log" -PERF_CLIENT=../clients/perf_client +PERF_CLIENT=perf_analyzer TRT_OP_TEST=trt_dynamic_shape_test.py DATADIR="./models" @@ -70,11 +72,9 @@ fi # Shape beyond the limits of optimization profile set +e $PERF_CLIENT -v -i grpc -u localhost:8001 -m plan_float32_float32_float32-4-32 --shape INPUT0:33 --shape INPUT1:33 -t 1 -p2000 -b 1 > ${CLIENT_LOG}_max 2>&1 -if [ $? -eq 0 ]; then - cat ${CLIENT_LOG}_max - echo -e "\n***\n*** Test Failed\n***" - RET=1 -fi +EXIT_CODE=$? +echo "perf_analyzer exit code: ${EXIT_CODE}" >> "${CLIENT_LOG}_max" +"${PERF_CLIENT}" --version >> "${CLIENT_LOG}_max" 2>&1 || true EXPECTED_MESSAGE="model expected the shape of dimension 1 to be between 4 and 32 but received" if [ $(cat ${CLIENT_LOG}_max | grep "${EXPECTED_MESSAGE} 33" | wc -l) -eq 0 ]; then @@ -84,11 +84,10 @@ if [ $(cat ${CLIENT_LOG}_max | grep "${EXPECTED_MESSAGE} 33" | wc -l) -eq 0 ]; t fi $PERF_CLIENT -v -i grpc -u localhost:8001 -m plan_float32_float32_float32-4-32 --shape INPUT0:3 --shape INPUT1:3 -t 1 -p2000 -b 1 > ${CLIENT_LOG}_min 2>&1 -if [ $? -eq 0 ]; then - cat ${CLIENT_LOG}_min - echo -e "\n***\n*** Test Failed\n***" - RET=1 -fi +EXIT_CODE=$? +echo "perf_analyzer exit code: ${EXIT_CODE}" >> "${CLIENT_LOG}_min" +"${PERF_CLIENT}" --version >> "${CLIENT_LOG}_min" 2>&1 || true + if [ $(cat ${CLIENT_LOG}_min | grep "${EXPECTED_MESSAGE} 3" | wc -l) -eq 0 ]; then cat ${CLIENT_LOG}_min echo -e "\n***\n*** Test Failed\n***" @@ -331,11 +330,10 @@ if [ $? -ne 0 ]; then fi $PERF_CLIENT -v -i grpc -u localhost:8001 -m plan_float32_float32_float32 --shape INPUT0:33 --shape INPUT1:33 -t 1 -p2000 -b 6 > ${CLIENT_LOG}_static_fail 2>&1 -if [ $? -eq 0 ]; then - ${CLIENT_LOG}_static_fail - echo -e "\n***\n*** Test Failed\n***" - RET=1 -fi +EXIT_CODE=$? +echo "perf_analyzer exit code: ${EXIT_CODE}" >> "${CLIENT_LOG}_static_fail" +"${PERF_CLIENT}" --version >> "${CLIENT_LOG}_static_fail" 2>&1 || true + if [ $(cat ${CLIENT_LOG}_static_fail | grep "inference request batch-size must be <= 5" | wc -l) -eq 0 ]; then cat ${CLIENT_LOG}_static_fail echo -e "\n***\n*** Test Failed\n***" @@ -343,11 +341,10 @@ if [ $(cat ${CLIENT_LOG}_static_fail | grep "inference request batch-size must b fi $PERF_CLIENT -v -i grpc -u localhost:8001 -m plan_float32_float32_float32 --shape INPUT0:33 --shape INPUT1:33 -t 1 -p2000 -b 2 > ${CLIENT_LOG}_static_bs_2 2>&1 -if [ $? -eq 0 ]; then - ${CLIENT_LOG}_static_bs_2 - echo -e "\n***\n*** Test Failed\n***" - RET=1 -fi +EXIT_CODE=$? +echo "perf_analyzer exit code: ${EXIT_CODE}" >> "${CLIENT_LOG}_static_bs_2" +"${PERF_CLIENT}" --version >> "${CLIENT_LOG}_static_bs_2" 2>&1 || true + if [ $(cat ${CLIENT_LOG}_static_bs_2 | grep "model expected the shape of dimension 0 to be between 1 and 1 but received 2" | wc -l) -eq 0 ]; then cat ${CLIENT_LOG}_static_bs_2 echo -e "\n***\n*** Test Failed\n***" diff --git a/qa/common/gen_jetson_trt_models b/qa/common/gen_jetson_trt_models index bf3281bc9b..9b652a6977 100755 --- a/qa/common/gen_jetson_trt_models +++ b/qa/common/gen_jetson_trt_models @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -34,7 +34,7 @@ # Make all generated files accessible outside of container umask 0000 # Set the version of the models -TRITON_VERSION=${TRITON_VERSION:=25.08} +TRITON_VERSION=${TRITON_VERSION:=25.09} # Set the CUDA device to use CUDA_DEVICE=${RUNNER_ID:=0} # Set TensorRT image diff --git a/qa/common/gen_qa_custom_ops b/qa/common/gen_qa_custom_ops index a716c0e517..1d7f6db05a 100755 --- a/qa/common/gen_qa_custom_ops +++ b/qa/common/gen_qa_custom_ops @@ -37,7 +37,7 @@ ## ############################################################################ -TRITON_VERSION=${TRITON_VERSION:=25.08} +TRITON_VERSION=${TRITON_VERSION:=25.09} NVIDIA_UPSTREAM_VERSION=${NVIDIA_UPSTREAM_VERSION:=$TRITON_VERSION} PYTORCH_IMAGE=${PYTORCH_IMAGE:=nvcr.io/nvidia/pytorch:$NVIDIA_UPSTREAM_VERSION-py3} UBUNTU_IMAGE=${UBUNTU_IMAGE:=ubuntu:24.04} diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 67e2adb902..14d6b85011 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -48,7 +48,7 @@ ## ############################################################################ -TRITON_VERSION=${TRITON_VERSION:=25.08} +TRITON_VERSION=${TRITON_VERSION:=25.09} # ONNX. Use ONNX_OPSET 0 to use the default for ONNX version ONNX_VERSION=1.16.1 diff --git a/qa/common/trtllm_util.sh b/qa/common/trtllm_util.sh new file mode 100755 index 0000000000..81ecb2d770 --- /dev/null +++ b/qa/common/trtllm_util.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +function clone_tensorrt_llm_backend_repo { + rm -rf $TENSORRTLLM_BACKEND_DIR && mkdir $TENSORRTLLM_BACKEND_DIR + apt-get update && apt-get install git-lfs -y --no-install-recommends + git clone --single-branch --depth=1 -b ${TENSORRTLLM_BACKEND_REPO_TAG} ${TRITON_REPO_ORG}/tensorrtllm_backend.git $TENSORRTLLM_BACKEND_DIR + cd $TENSORRTLLM_BACKEND_DIR && git lfs install && git submodule update --init --recursive +} + +function build_gpt2_base_model { + # Download weights from HuggingFace Transformers + cd ${GPT_DIR} && rm -rf gpt2 && git clone https://huggingface.co/gpt2-medium gpt2 && cd gpt2 + rm pytorch_model.bin model.safetensors + if ! wget -q https://huggingface.co/gpt2-medium/resolve/main/pytorch_model.bin; then + echo "Downloading pytorch_model.bin failed." + exit 1 + fi + cd ${GPT_DIR} + + # Convert weights from HF Tranformers to FT format + python3 convert_checkpoint.py --model_dir gpt2 --dtype float16 --tp_size ${NUM_GPUS} --output_dir "./c-model/gpt2/${NUM_GPUS}-gpu/" + cd ${BASE_DIR} +} + +function build_gpt2_tensorrt_engine { + # Build TensorRT engines + cd ${GPT_DIR} + trtllm-build --checkpoint_dir "./c-model/gpt2/${NUM_GPUS}-gpu/" \ + --gpt_attention_plugin float16 \ + --remove_input_padding enable \ + --paged_kv_cache enable \ + --gemm_plugin float16 \ + --workers "${NUM_GPUS}" \ + --output_dir "${ENGINES_DIR}" + + cd ${BASE_DIR} +} + +function replace_config_tags { + tag_to_replace="${1}" + new_value="${2}" + config_file_path="${3}" + sed -i "s|${tag_to_replace}|${new_value}|g" ${config_file_path} +} + +function prepare_model_repository { + rm -rf ${MODEL_REPOSITORY} && mkdir ${MODEL_REPOSITORY} + cp -r ${TENSORRTLLM_BACKEND_DIR}/tensorrt_llm/triton_backend/all_models/inflight_batcher_llm/* ${MODEL_REPOSITORY} + rm -rf ${MODEL_REPOSITORY}/tensorrt_llm_bls + mv "${MODEL_REPOSITORY}/ensemble" "${MODEL_REPOSITORY}/${MODEL_NAME}" + + replace_config_tags "model_version: -1" "model_version: 1" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + replace_config_tags 'name: "ensemble"' "name: \"$MODEL_NAME\"" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${preprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + replace_config_tags '${postprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${decoupled_mode}' 'true' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${batching_strategy}' 'inflight_fused_batching' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${engine_dir}' "${ENGINES_DIR}" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${triton_backend}' "tensorrtllm" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${encoder_input_features_data_type}' "TYPE_FP32" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${prompt_embedding_table_data_type}' 'TYPE_FP16' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" +} + +# Wait until server health endpoint shows ready. Sets WAIT_RET to 0 on +# success, 1 on failure +function wait_for_server_ready() { + local wait_time_secs="${1:-30}" + shift + local spids=("$@") + + WAIT_RET=0 + + for _ in $(seq "$wait_time_secs"); do + for pid in "${spids[@]}"; do + if ! kill -0 "$pid" >/dev/null 2>&1; then + echo "=== Server not running." + WAIT_RET=1 + return + fi + done + + sleep 1 + + if curl -s --fail localhost:8000/v2/health/ready && + curl -s --fail -w "%{http_code}" -o /dev/null -d '{"log_verbose_level":1}' localhost:8000/v2/logging; then + return + fi + done + + echo "=== Timeout $wait_time_secs secs. Server not ready." + WAIT_RET=1 +} + +function run_server { + python3 ${TENSORRTLLM_BACKEND_DIR}/tensorrt_llm/triton_backend/scripts/launch_triton_server.py --world_size="${NUM_GPUS}" --model_repo="${MODEL_REPOSITORY}" >${SERVER_LOG} 2>&1 & + sleep 2 # allow time to obtain the pid(s) + # Read PIDs into an array, trimming whitespaces + readarray -t SERVER_PID < <(pgrep "tritonserver") + + wait_for_server_ready ${SERVER_TIMEOUT} "${SERVER_PID[@]}" + if [ "$WAIT_RET" != "0" ]; then + # Cleanup + kill "${SERVER_PID[@]}" >/dev/null 2>&1 || true + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 + fi +} + +function kill_server { + pgrep tritonserver | xargs kill -SIGINT + for pid in "${SERVER_PID[@]}"; do + echo "Waiting for proc ${pid} to terminate..." + while kill -0 $pid >/dev/null 2>&1; do + sleep 1 + done + done +} diff --git a/src/command_line_parser.cc b/src/command_line_parser.cc index 633eb5f980..0d7892efcb 100644 --- a/src/command_line_parser.cc +++ b/src/command_line_parser.cc @@ -506,7 +506,7 @@ TritonParser::SetupOptions() {OPTION_HTTP_RESTRICTED_API, "http-restricted-api", ":=", "Specify restricted HTTP api setting. The format of this " - "flag is --http-restricted-api=,=. Where " + "flag is --http-restricted-api=:=. Where " " is a comma-separated list of apis to be restricted. " " will be additional header key to be checked when a HTTP request " "is received, and is the value expected to be matched." @@ -630,7 +630,7 @@ TritonParser::SetupOptions() {OPTION_GRPC_RESTRICTED_PROTOCOL, "grpc-restricted-protocol", ":=", "Specify restricted GRPC protocol setting. The format of this " - "flag is --grpc-restricted-protocol=,=. Where " + "flag is --grpc-restricted-protocol=:=. Where " " is a comma-separated list of protocols to be restricted. " " will be additional header key to be checked when a GRPC request " "is received, and is the value expected to be matched." diff --git a/src/simple.cc b/src/simple.cc index c429861cc5..cf0cafeed5 100644 --- a/src/simple.cc +++ b/src/simple.cc @@ -1,4 +1,4 @@ -// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -876,6 +876,10 @@ main(int argc, char** argv) FAIL_IF_ERR( TRITONSERVER_InferenceResponseDelete(completed_response), "deleting inference response"); + + // We need to make sure that the previous request was released before + // reusing it. + request_release_future.get(); } // The TRITONSERVER_InferenceRequest object can be reused for @@ -906,10 +910,6 @@ main(int argc, char** argv) InferResponseComplete, reinterpret_cast(p)), "setting response callback"); - // We need to make sure that the previous request was released before - // reusing it. - request_release_future.get(); - // Register a new promise for the request callback barrier. barrier = std::make_unique>(); request_release_future = barrier->get_future(); @@ -936,6 +936,10 @@ main(int argc, char** argv) FAIL_IF_ERR( TRITONSERVER_InferenceResponseDelete(completed_response), "deleting inference response"); + + // We need to make sure that the previous request was released before + // reusing it. + request_release_future.get(); } // There are other TRITONSERVER_InferenceRequest APIs that allow @@ -967,7 +971,7 @@ main(int argc, char** argv) // Register a new promise for the request callback barrier. barrier = std::make_unique>(); - request_release_future.get(); + request_release_future = barrier->get_future(); FAIL_IF_ERR( TRITONSERVER_InferenceRequestSetReleaseCallback( @@ -993,6 +997,10 @@ main(int argc, char** argv) FAIL_IF_ERR( TRITONSERVER_InferenceResponseDelete(completed_response), "deleting inference response"); + + // Make sure the request is released before deleting it. If not, release + // callback will segfault after barrier is destructed. + request_release_future.get(); } FAIL_IF_ERR(