diff --git a/Dockerfile.sdk b/Dockerfile.sdk index 33414c3fe4..d1c4b5a189 100644 --- a/Dockerfile.sdk +++ b/Dockerfile.sdk @@ -29,7 +29,7 @@ # # Base image on the minimum Triton container -ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.05-py3-min +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.06-py3-min ARG TRITON_CLIENT_REPO_SUBDIR=clientrepo ARG TRITON_PA_REPO_SUBDIR=perfanalyzerrepo diff --git a/README.md b/README.md index 2de74ae96a..057e58e8f9 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ >[!WARNING] >You are currently on the `main` branch which tracks under-development progress ->towards the next release. The current release is version [2.58.0](https://github.com/triton-inference-server/server/releases/latest) ->and corresponds to the 25.05 container release on NVIDIA GPU Cloud (NGC). +>towards the next release. The current release is version [2.59.0](https://github.com/triton-inference-server/server/releases/latest) +>and corresponds to the 25.06 container release on NVIDIA GPU Cloud (NGC). # Triton Inference Server @@ -90,16 +90,16 @@ Inference Server with the ```bash # Step 1: Create the example model repository -git clone -b r25.05 https://github.com/triton-inference-server/server.git +git clone -b r25.06 https://github.com/triton-inference-server/server.git cd server/docs/examples ./fetch_models.sh # Step 2: Launch triton from the NGC Triton container -docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:25.05-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx +docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:25.06-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx # Step 3: Sending an Inference Request # In a separate console, launch the image_client example from the NGC Triton SDK container -docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:25.05-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg +docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:25.06-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg # Inference should return the following Image '/workspace/images/mug.jpg': diff --git a/build.py b/build.py index 3da4ad89e3..ff069c7d20 100755 --- a/build.py +++ b/build.py @@ -73,12 +73,12 @@ DEFAULT_TRITON_VERSION_MAP = { "release_version": "2.60.0dev", "triton_container_version": "25.07dev", - "upstream_container_version": "25.05", + "upstream_container_version": "25.06", "ort_version": "1.22.0", "ort_openvino_version": "2025.1.0", "standalone_openvino_version": "2025.1.0", "dcgm_version": "3.3.6", - "vllm_version": "0.8.4", + "vllm_version": "0.9.0.1", "rhel_py_version": "3.12.3", } diff --git a/deploy/aws/values.yaml b/deploy/aws/values.yaml index cddaaaf3a6..c24b0ed7b8 100644 --- a/deploy/aws/values.yaml +++ b/deploy/aws/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.05-py3 + imageName: nvcr.io/nvidia/tritonserver:25.06-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/fleetcommand/Chart.yaml b/deploy/fleetcommand/Chart.yaml index 0664a5d959..b819cbb4a5 100644 --- a/deploy/fleetcommand/Chart.yaml +++ b/deploy/fleetcommand/Chart.yaml @@ -26,7 +26,7 @@ apiVersion: v1 # appVersion is the Triton version; update when changing release -appVersion: "2.58.0" +appVersion: "2.59.0" description: Triton Inference Server (Fleet Command) name: triton-inference-server # version is the Chart version; update when changing anything in the chart diff --git a/deploy/fleetcommand/values.yaml b/deploy/fleetcommand/values.yaml index b4044b795a..d2300ef421 100644 --- a/deploy/fleetcommand/values.yaml +++ b/deploy/fleetcommand/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.05-py3 + imageName: nvcr.io/nvidia/tritonserver:25.06-py3 pullPolicy: IfNotPresent numGpus: 1 serverCommand: tritonserver @@ -47,13 +47,13 @@ image: # # To set model control mode, uncomment and configure below # TODO: Fix the following url, it is invalid - # See https://github.com/triton-inference-server/server/blob/r25.05/docs/user_guide/model_management.md + # See https://github.com/triton-inference-server/server/blob/r25.06/docs/user_guide/model_management.md # for more details #- --model-control-mode=explicit|poll|none # # Additional server args # - # see https://github.com/triton-inference-server/server/blob/r25.05/README.md + # see https://github.com/triton-inference-server/server/blob/r25.06/README.md # for more details service: diff --git a/deploy/gcp/values.yaml b/deploy/gcp/values.yaml index 3770934cc8..73948fb39d 100644 --- a/deploy/gcp/values.yaml +++ b/deploy/gcp/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.05-py3 + imageName: nvcr.io/nvidia/tritonserver:25.06-py3 pullPolicy: IfNotPresent modelRepositoryPath: gs://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml index 38a65ed8e8..8be23e0f2c 100644 --- a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml +++ b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml @@ -33,7 +33,7 @@ metadata: namespace: default spec: containers: - - image: nvcr.io/nvidia/tritonserver:25.05-py3-sdk + - image: nvcr.io/nvidia/tritonserver:25.06-py3-sdk imagePullPolicy: Always name: nv-triton-client securityContext: diff --git a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh index 6cb6f76e0a..b31804b9a2 100755 --- a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh +++ b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh @@ -27,9 +27,9 @@ export REGISTRY=gcr.io/$(gcloud config get-value project | tr ':' '/') export APP_NAME=tritonserver -export MAJOR_VERSION=2.58 -export MINOR_VERSION=2.58.0 -export NGC_VERSION=25.05-py3 +export MAJOR_VERSION=2.59 +export MINOR_VERSION=2.59.0 +export NGC_VERSION=25.06-py3 docker pull nvcr.io/nvidia/$APP_NAME:$NGC_VERSION diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml index 5011733bbc..3206248935 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. apiVersion: v1 -appVersion: "2.58" +appVersion: "2.59" description: Triton Inference Server name: triton-inference-server -version: 2.58.0 +version: 2.59.0 diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml index ed69180192..52b1ab2f21 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml @@ -31,14 +31,14 @@ maxReplicaCount: 3 tritonProtocol: HTTP # HPA GPU utilization autoscaling target HPATargetAverageValue: 85 -modelRepositoryPath: gs://triton_sample_models/25.05 -publishedVersion: '2.58.0' +modelRepositoryPath: gs://triton_sample_models/25.06 +publishedVersion: '2.59.0' gcpMarketplace: true image: registry: gcr.io repository: nvidia-ngc-public/tritonserver - tag: 25.05-py3 + tag: 25.06-py3 pullPolicy: IfNotPresent # modify the model repository here to match your GCP storage bucket numGpus: 1 diff --git a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml index 2c55441213..36ba6b6a63 100644 --- a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.58.0' + publishedVersion: '2.59.0' publishedVersionMetadata: releaseNote: >- Initial release. diff --git a/deploy/gke-marketplace-app/server-deployer/schema.yaml b/deploy/gke-marketplace-app/server-deployer/schema.yaml index 68c171d28a..629303952d 100644 --- a/deploy/gke-marketplace-app/server-deployer/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/schema.yaml @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.58.0' + publishedVersion: '2.59.0' publishedVersionMetadata: releaseNote: >- Initial release. @@ -89,7 +89,7 @@ properties: modelRepositoryPath: type: string title: Bucket where models are stored. Please make sure the user/service account to create the GKE app has permission to this GCS bucket. Read Triton documentation on configs and formatting details, supporting TensorRT, TensorFlow, Pytorch, Onnx ... etc. - default: gs://triton_sample_models/25.05 + default: gs://triton_sample_models/25.06 image.ldPreloadPath: type: string title: Leave this empty by default. Triton allows users to create custom layers for backend such as TensorRT plugin or Tensorflow custom ops, the compiled shared library must be provided via LD_PRELOAD environment variable. diff --git a/deploy/gke-marketplace-app/trt-engine/README.md b/deploy/gke-marketplace-app/trt-engine/README.md index 01475e551c..dba63cf63e 100644 --- a/deploy/gke-marketplace-app/trt-engine/README.md +++ b/deploy/gke-marketplace-app/trt-engine/README.md @@ -33,7 +33,7 @@ ``` docker run --gpus all -it --network host \ --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \ - -v ~:/scripts nvcr.io/nvidia/tensorrt:25.05-py3 + -v ~:/scripts nvcr.io/nvidia/tensorrt:25.06-py3 pip install onnx six torch tf2onnx tensorflow @@ -57,7 +57,7 @@ mkdir -p engines python3 builder.py -m models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/model.ckpt -o engines/bert_large_int8_bs1_s128.engine -b 1 -s 128 -c models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/ -v models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/vocab.txt --int8 --fp16 --strict --calib-num 1 -iln -imh -gsutil cp bert_large_int8_bs1_s128.engine gs://triton_sample_models/25.05/bert/1/model.plan +gsutil cp bert_large_int8_bs1_s128.engine gs://triton_sample_models/25.06/bert/1/model.plan ``` -For each Triton upgrade, container version used to generate the model, and the model path in GCS `gs://triton_sample_models/25.05/` should be updated accordingly with the correct version. +For each Triton upgrade, container version used to generate the model, and the model path in GCS `gs://triton_sample_models/25.06/` should be updated accordingly with the correct version. diff --git a/deploy/k8s-onprem/values.yaml b/deploy/k8s-onprem/values.yaml index 27cb9bf33b..ac6d7a3480 100644 --- a/deploy/k8s-onprem/values.yaml +++ b/deploy/k8s-onprem/values.yaml @@ -30,7 +30,7 @@ tags: openshift: false image: - imageName: nvcr.io/nvidia/tritonserver:25.05-py3 + imageName: nvcr.io/nvidia/tritonserver:25.06-py3 pullPolicy: IfNotPresent modelRepositoryServer: < Replace with the IP Address of your file server > modelRepositoryPath: /srv/models diff --git a/deploy/oci/values.yaml b/deploy/oci/values.yaml index b082864f3e..be6eb13a0e 100644 --- a/deploy/oci/values.yaml +++ b/deploy/oci/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.05-py3 + imageName: nvcr.io/nvidia/tritonserver:25.06-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://https://.compat.objectstorage..oraclecloud.com:443/triton-inference-server-repository numGpus: 1 diff --git a/docs/introduction/compatibility.md b/docs/introduction/compatibility.md index afa4ab0ac8..4332d4c8ae 100644 --- a/docs/introduction/compatibility.md +++ b/docs/introduction/compatibility.md @@ -38,20 +38,21 @@ | Triton release version | NGC Tag | Python version | Torch version | TensorRT version | TensorRT-LLM version | CUDA version | CUDA Driver version | Size | | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 25.06 | nvcr.io/nvidia/tritonserver:25.06-trtllm-python-py3 | Python 3.12.3 | 2.7.0a0+79aa17489c.nv25.4 | 10.10.0.31 | 0.20.0 | 12.9.0.036 | 575.51.03 | 18.3G | | 25.05 | nvcr.io/nvidia/tritonserver:25.05-trtllm-python-py3 | Python 3.12.3 | 2.7.0a0+7c8ec84dab.nv25.3 | 10.9.0.34 | 0.19.0 | 12.8.1.012 | 570.124.06 | 17G | | 25.04 | nvcr.io/nvidia/tritonserver:25.04-trtllm-python-py3 | Python 3.12.3 | 2.7.0a0+7c8ec84dab.nv25.3 | 10.9.0.34 | 0.18.2 | 12.8.1.012 | 570.124.06 | 17G | -| 25.03 | nvcr.io/nvidia/tritonserver:25.03-trtllm-python-py3 | Python 3.12.3 | 2.7.0a0%2B7c8ec84dab.nv25.3 | 10.9.0.34 | 0.18.0 | 12.8.1.012 | 570.124.06 | 28G | -| 25.02 | nvcr.io/nvidia/tritonserver:25.02-trtllm-python-py3 | Python 3.12.3 | 2.6.0a0%2Becf3bae40a.nv25.1 | 10.8.0.43 | 0.17.0.post1 | 12.8.0.038 | 570.86.10 | 28G | -| 25.01 | nvcr.io/nvidia/tritonserver:25.01-trtllm-python-py3 | Python 3.12.3 | 2.6.0a0%2Becf3bae40a.nv25.1 | 10.8.0.43 | 0.17.0 | 12.8.0.038 | 570.86.10 | 30G | -| 24.12 | nvcr.io/nvidia/tritonserver:24.12-trtllm-python-py3 | Python 3.12.3 | 2.6.0a0%2Bdf5bbc09d1.nv24.11 | 10.7.0 | 0.16.0 | 12.6.3 | 560.35.05 | 22G | -| 24.11 | nvcr.io/nvidia/tritonserver:24.11-trtllm-python-py3 | Python 3.10.12 | 2.5.0a0%2Be000cf0ad9.nv24.10 | 10.6.0 | 0.15.0 | 12.6.3 | 555.42.06 | 24.8G | -| 24.10 | nvcr.io/nvidia/tritonserver:24.10-trtllm-python-py3 | Python 3.10.12 | 2.4.0a0%2B3bcc3cddb5.nv24.7 | 10.4.0 | 0.14.0 | 12.5.1.007 | 555.42.06 | 23.3G | -| 24.09 | nvcr.io/nvidia/tritonserver:24.09-trtllm-python-py3 | Python 3.10.12 | 2.4.0a0%2B3bcc3cddb5.nv24.7 | 10.4.0 | 0.13.0 | 12.5.1.007 | 555.42.06 | 21G | -| 24.08 | nvcr.io/nvidia/tritonserver:24.08-trtllm-python-py3 | Python 3.10.12 | 2.4.0a0%2B3bcc3cddb5.nv24.7 | 10.3.0 | 0.12.0 | 12.5.1.007 | 555.42.06 | 21G | -| 24.07 | nvcr.io/nvidia/tritonserver:24.07-trtllm-python-py3 | Python 3.10.12 | 2.4.0a0%2B07cecf4168.nv24.5 | 10.1.0 | 0.11.0 | 12.4.1.003 | 550.54.15 | 23G | -| 24.06 | nvcr.io/nvidia/tritonserver:24.06-trtllm-python-py3 | Python 3.10.12 | 2.3.0a0%2B40ec155e58.nv24.3 | 10.0.1 | 0.10.0 | 12.4.0.041 | 550.54.14 | 31G | -| 24.05 | nvcr.io/nvidia/tritonserver:24.05-trtllm-python-py3 | Python 3.10.12 | 2.3.0a0%2Bebedce2 | 10.0.1.6 | 0.9.0 | 12.3.2.001 | 545.23.08 | 34G | -| 24.04 | nvcr.io/nvidia/tritonserver:24.04-trtllm-python-py3 | Python 3.10.12 | 2.3.0a0%2Bebedce2 | 9.3.0.post12.dev1 | 0.9.0 | 12.3.2.001 | 545.23.08 | 34G | +| 25.03 | nvcr.io/nvidia/tritonserver:25.03-trtllm-python-py3 | Python 3.12.3 | 2.7.0a0+7c8ec84dab.nv25.3 | 10.9.0.34 | 0.18.0 | 12.8.1.012 | 570.124.06 | 28G | +| 25.02 | nvcr.io/nvidia/tritonserver:25.02-trtllm-python-py3 | Python 3.12.3 | 2.6.0a0+ecf3bae40a.nv25.1 | 10.8.0.43 | 0.17.0.post1 | 12.8.0.038 | 570.86.10 | 28G | +| 25.01 | nvcr.io/nvidia/tritonserver:25.01-trtllm-python-py3 | Python 3.12.3 | 2.6.0a0+ecf3bae40a.nv25.1 | 10.8.0.43 | 0.17.0 | 12.8.0.038 | 570.86.10 | 30G | +| 24.12 | nvcr.io/nvidia/tritonserver:24.12-trtllm-python-py3 | Python 3.12.3 | 2.6.0a0+df5bbc09d1.nv24.11 | 10.7.0 | 0.16.0 | 12.6.3 | 560.35.05 | 22G | +| 24.11 | nvcr.io/nvidia/tritonserver:24.11-trtllm-python-py3 | Python 3.10.12 | 2.5.0a0+e000cf0ad9.nv24.10 | 10.6.0 | 0.15.0 | 12.6.3 | 555.42.06 | 24.8G | +| 24.10 | nvcr.io/nvidia/tritonserver:24.10-trtllm-python-py3 | Python 3.10.12 | 2.4.0a0+3bcc3cddb5.nv24.7 | 10.4.0 | 0.14.0 | 12.5.1.007 | 555.42.06 | 23.3G | +| 24.09 | nvcr.io/nvidia/tritonserver:24.09-trtllm-python-py3 | Python 3.10.12 | 2.4.0a0+3bcc3cddb5.nv24.7 | 10.4.0 | 0.13.0 | 12.5.1.007 | 555.42.06 | 21G | +| 24.08 | nvcr.io/nvidia/tritonserver:24.08-trtllm-python-py3 | Python 3.10.12 | 2.4.0a0+3bcc3cddb5.nv24.7 | 10.3.0 | 0.12.0 | 12.5.1.007 | 555.42.06 | 21G | +| 24.07 | nvcr.io/nvidia/tritonserver:24.07-trtllm-python-py3 | Python 3.10.12 | 2.4.0a0+07cecf4168.nv24.5 | 10.1.0 | 0.11.0 | 12.4.1.003 | 550.54.15 | 23G | +| 24.06 | nvcr.io/nvidia/tritonserver:24.06-trtllm-python-py3 | Python 3.10.12 | 2.3.0a0+40ec155e58.nv24.3 | 10.0.1 | 0.10.0 | 12.4.0.041 | 550.54.14 | 31G | +| 24.05 | nvcr.io/nvidia/tritonserver:24.05-trtllm-python-py3 | Python 3.10.12 | 2.3.0a0+ebedce2 | 10.0.1.6 | 0.9.0 | 12.3.2.001 | 545.23.08 | 34G | +| 24.04 | nvcr.io/nvidia/tritonserver:24.04-trtllm-python-py3 | Python 3.10.12 | 2.3.0a0+ebedce2 | 9.3.0.post12.dev1 | 0.9.0 | 12.3.2.001 | 545.23.08 | 34G | ## Container Name: vllm-python-py3 diff --git a/docs/introduction/release_notes.md b/docs/introduction/release_notes.md index f297ec3ac2..7038abbb01 100644 --- a/docs/introduction/release_notes.md +++ b/docs/introduction/release_notes.md @@ -25,9 +25,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> -# [Triton Inference Server Release 25.05](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-25-05.html#rel-25-05) +# [Triton Inference Server Release 25.06](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-25-06.html#rel-25-06) -The Triton Inference Server container image, release 25.05, is available +The Triton Inference Server container image, release 25.06, is available on [NGC](https://ngc.nvidia.com/catalog/containers/nvidia:tritonserver) and is open source on [GitHub](https://github.com/triton-inference-server/server). Release notes can diff --git a/python/openai/README.md b/python/openai/README.md index 3d770caa2e..e93c9c1f16 100644 --- a/python/openai/README.md +++ b/python/openai/README.md @@ -51,7 +51,7 @@ docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ - nvcr.io/nvidia/tritonserver:25.05-vllm-python-py3 + nvcr.io/nvidia/tritonserver:25.06-vllm-python-py3 ``` 2. Launch the OpenAI-compatible Triton Inference Server: diff --git a/qa/L0_client_nobatch/test.sh b/qa/L0_client_nobatch/test.sh index cb1a05d660..0c01bafb70 100755 --- a/qa/L0_client_nobatch/test.sh +++ b/qa/L0_client_nobatch/test.sh @@ -47,7 +47,7 @@ EXPECTED_NUM_TESTS="4" DATADIR=/data/inferenceserver/${REPO_VERSION} MODELDIR="${PWD}/qa_model_repository" -rm -rf ${MODELDIR} && cp -r "${DATADIR}/qa_model_repository/onnx_*" ${MODELDIR} # Note there is a coupling in ./client_test.py +rm -rf ${MODELDIR} && mkdir -p ${MODELDIR} && cp -r ${DATADIR}/qa_model_repository/onnx_* ${MODELDIR}/. # Note there is a coupling in ./client_test.py SERVER=/opt/tritonserver/bin/tritonserver SERVER_ARGS="--model-repository=${MODELDIR}" SERVER_LOG="./inference_server.log" diff --git a/qa/L0_long_running_stress/scenarios.py b/qa/L0_long_running_stress/scenarios.py index 650f228ab5..5b952f5b97 100755 --- a/qa/L0_long_running_stress/scenarios.py +++ b/qa/L0_long_running_stress/scenarios.py @@ -203,7 +203,7 @@ def __init__( # Add no validation models self.options_.append( PerfAnalyzerScenario.ModelOption( - "resnet_v1_50_def", 32, (1, 4, 1), queue_latency_range_us + "resnet_v1_50", 32, (1, 4, 1), queue_latency_range_us ) ) for trial in sequence_trials: @@ -334,7 +334,7 @@ def run(self, client_metadata): class ResNetScenario(Scenario): def __init__(self, name, batch_size=32, verbose=False, out_stream=sys.stdout): super().__init__(name, [], verbose, out_stream) - self.model_name_ = "resnet_v1_50_def" + self.model_name_ = "resnet_v1_50" self.batch_size_ = batch_size img = self.preprocess("../images/vulture.jpeg") @@ -353,7 +353,7 @@ def preprocess(self, filename): return scaled def postprocess(self, results): - output_array = results.as_numpy("resnet_v1_50/predictions/Softmax") + output_array = results.as_numpy("resnet_v1_50/predictions/Softmax:0") if len(output_array) != self.batch_size_: raise Exception( "expected {} results, got {}".format( @@ -377,12 +377,12 @@ def postprocess(self, results): def run(self, client_metadata): triton_client = client_metadata[0] - inputs = [grpcclient.InferInput("input", self.image_data_.shape, "FP32")] + inputs = [grpcclient.InferInput("input:0", self.image_data_.shape, "FP32")] inputs[0].set_data_from_numpy(self.image_data_) outputs = [ grpcclient.InferRequestedOutput( - "resnet_v1_50/predictions/Softmax", class_count=1 + "resnet_v1_50/predictions/Softmax:0", class_count=1 ) ] res = triton_client.infer(self.model_name_, inputs, outputs=outputs) diff --git a/qa/L0_long_running_stress/test.sh b/qa/L0_long_running_stress/test.sh index 1973028bcf..92d8c09b0d 100755 --- a/qa/L0_long_running_stress/test.sh +++ b/qa/L0_long_running_stress/test.sh @@ -130,9 +130,8 @@ cp -r ../custom_models/custom_zero_1_float32 $MODEL_DIR/custom_zero_1_float32 && echo "{ key: \"execute_delay_ms\"; value: { string_value: \"10000\" }}" >> config.pbtxt && \ echo "]" >> config.pbtxt) -cp -r $DATADIR/onnx_model_store/resnet_v1_50 $MODEL_DIR/resnet_v1_50_def && \ - (cd $MODEL_DIR/resnet_v1_50_def && \ - sed -i 's/^name: "resnet_v1_50"/name: "resnet_v1_50_def"/' config.pbtxt && \ +cp -r $DATADIR/onnx_model_store/resnet_v1_50 $MODEL_DIR/. && \ + (cd $MODEL_DIR/resnet_v1_50 && \ echo "optimization { }" >> config.pbtxt) SERVER_ARGS="--model-repository=`pwd`/$MODEL_DIR" diff --git a/qa/common/gen_jetson_trt_models b/qa/common/gen_jetson_trt_models index 8726e7c271..b081fe9c4e 100755 --- a/qa/common/gen_jetson_trt_models +++ b/qa/common/gen_jetson_trt_models @@ -34,7 +34,7 @@ # Make all generated files accessible outside of container umask 0000 # Set the version of the models -TRITON_VERSION=${TRITON_VERSION:=25.05} +TRITON_VERSION=${TRITON_VERSION:=25.06} # Set the CUDA device to use CUDA_DEVICE=${RUNNER_ID:=0} # Set TensorRT image diff --git a/qa/common/gen_qa_custom_ops b/qa/common/gen_qa_custom_ops index 96801e4470..108b1daa53 100755 --- a/qa/common/gen_qa_custom_ops +++ b/qa/common/gen_qa_custom_ops @@ -37,7 +37,7 @@ ## ############################################################################ -TRITON_VERSION=${TRITON_VERSION:=25.05} +TRITON_VERSION=${TRITON_VERSION:=25.06} NVIDIA_UPSTREAM_VERSION=${NVIDIA_UPSTREAM_VERSION:=$TRITON_VERSION} PYTORCH_IMAGE=${PYTORCH_IMAGE:=nvcr.io/nvidia/pytorch:$NVIDIA_UPSTREAM_VERSION-py3} UBUNTU_IMAGE=${UBUNTU_IMAGE:=ubuntu:24.04} diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 7f4741d1c7..2deeb97da5 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -48,7 +48,7 @@ ## ############################################################################ -TRITON_VERSION=${TRITON_VERSION:=25.05} +TRITON_VERSION=${TRITON_VERSION:=25.06} # ONNX. Use ONNX_OPSET 0 to use the default for ONNX version ONNX_VERSION=1.16.1