From 2f642d154057ad647f3ff3f6a65e018574b4a9e1 Mon Sep 17 00:00:00 2001 From: Misha Chornyi Date: Thu, 4 Dec 2025 14:32:46 -0800 Subject: [PATCH 1/7] temp: condition bypass. --- qa/common/gen_qa_model_repository | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 2edfcf325d..bff205110b 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -345,7 +345,7 @@ if [ $? -ne 0 ]; then fi log_message.status "check: engine installation" -if which docker ; then +if which docker-remove-upon-merge ; then log_message.info "Docker is installed." if ! docker volume inspect $DOCKER_VOLUME > /dev/null 2>&1; then From 3ec431351ac1fd2cd72d569e1e9c936c5ae9f73f Mon Sep 17 00:00:00 2001 From: Misha Chornyi Date: Thu, 4 Dec 2025 15:14:07 -0800 Subject: [PATCH 2/7] feat: Convert script generation into functions --- qa/common/gen_qa_model_repository | 158 ++++++++++++++++-------------- 1 file changed, 84 insertions(+), 74 deletions(-) diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index bff205110b..0711385395 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -80,50 +80,54 @@ DOCKER_VOLUME=${DOCKER_VOLUME:=volume.gen_qa_model_repository.${CI_JOB_ID}} DOCKER_VOLUME_CONTAINER=${DOCKER_VOLUME}.gen_qa_model_repository.${CI_JOB_ID} DOCKER_GPU_ARGS=${DOCKER_GPU_ARGS:-$([[ $RUNNER_GPUS =~ ^[0-9] ]] && eval $NV_DOCKER_ARGS || echo "--gpus device=$CUDA_DEVICE" )} -log_message.status "define: model output directories" -TRITON_MDLS_BLD_DIR=${TRITON_MDLS_BLD_DIR:=/mnt/$CI_JOB_ID} -TRITON_MDLS_SRC_DIR=${TRITON_MDLS_SRC_DIR:=$TRITON_MDLS_BLD_DIR/gen_srcdir} -TRITON_MDLS_QA_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_model_repository -TRITON_MDLS_QA_VARIABLE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_variable_model_repository -TRITON_MDLS_QA_IDENTITY_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_identity_model_repository -TRITON_MDLS_QA_IDENTITY_BIG_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_identity_big_model_repository -TRITON_MDLS_QA_SHAPETENSOR_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_shapetensor_model_repository -TRITON_MDLS_QA_RESHAPE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_reshape_model_repository -TRITON_MDLS_QA_SEQUENCE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_sequence_model_repository -TRITON_MDLS_QA_DYNA_SEQUENCE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_dyna_sequence_model_repository -TRITON_MDLS_QA_DYNA_SEQUENCE_IMPLICIT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_dyna_sequence_implicit_model_repository -TRITON_MDLS_QA_VARIABLE_SEQUENCE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_variable_sequence_model_repository -TRITON_MDLS_QA_ENSEMBLE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_ensemble_model_repository -TRITON_MDLS_QA_NOSHAPE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_noshape_model_repository -TRITON_MDLS_QA_TRT_PLUGIN_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_trt_plugin_model_repository -TRITON_MDLS_QA_RAGGED_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_ragged_model_repository -TRITON_MDLS_QA_TRT_FORMAT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_trt_format_model_repository -TRITON_MDLS_QA_TRT_DATA_DEPENDENT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_trt_data_dependent_model_repository -TRITON_MDLS_QA_SEQUENCE_IMPLICIT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_sequence_implicit_model_repository -TRITON_MDLS_QA_VARIABLE_SEQUENCE_IMPLICIT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_variable_sequence_implicit_model_repository -TRITON_MDLS_QA_SEQUENCE_INITIAL_STATE_IMPLICIT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_sequence_initial_state_implicit_model_repository -TRITON_MDLS_QA_VARIABLE_SEQUENCE_INITIAL_STATE_IMPLICIT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_variable_sequence_initial_state_implicit_model_repository -TRITON_MDLS_QA_TORCHTRT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/torchtrt_model_store -TRITON_MDLS_QA_SCALAR_MODELS=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_scalar_models -TRITON_MDLS_QA_DYNAMIC_BATCH_IMAGE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_dynamic_batch_image_model_repository -TRITON_MDLS_QA_CUSTOM_OPS=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_custom_ops/libtorch_custom_ops - -log_message.status "define: script names" - -OPENVINOSCRIPT=gen.OpenVINO.gen_qa_model_repository.v2.sh -log_message.info "OpenVINO script: " "${OPENVINOSCRIPT}" - -ONNXSCRIPT=gen.ONNXRuntime.gen_qa_model_repository.v2.sh -log_message.info "ONNX script: " "${ONNXSCRIPT}" - -TORCHSCRIPT=gen.PyTorch.gen_qa_model_repository.v2.sh -log_message.info "PyTorch script: " "${TORCHSCRIPT}" - -TRTSCRIPT=gen.TensorRT.gen_qa_model_repository.v2.sh -log_message.info "TensorRT script: " "${TRTSCRIPT}" - -log_message.status "create: OpenVINO script - ${OPENVINOSCRIPT}" -cat > $OPENVINOSCRIPT < $OPENVINOSCRIPT < $ONNXSCRIPT < $ONNXSCRIPT < $TORCHSCRIPT < $TORCHSCRIPT < $TRTSCRIPT < $TRTSCRIPT < Date: Thu, 4 Dec 2025 17:35:33 -0800 Subject: [PATCH 3/7] models: PyTorch + enroot --- qa/common/gen_qa_model_repository | 65 ++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 0711385395..5451b9a0c5 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -135,13 +135,6 @@ nvidia-smi --query-gpu=compute_cap,compute_mode,driver_version,name,index --form nvidia-smi || true set -e set -x -export DEBIAN_FRONTEND=noninteractive -apt-get update && \ - apt-get install -y --no-install-recommends build-essential cmake libprotobuf-dev \ - protobuf-compiler python3 python3-dev python3-pip wget gnupg2 \ - software-properties-common -ln -s /usr/bin/python3 /usr/bin/python - pip3 install "numpy<=1.23.5" setuptools pip3 install openvino==$OPENVINO_VERSION @@ -269,7 +262,8 @@ chmod -R 777 $TRITON_MDLS_QA_RAGGED_MODEL python3 $TRITON_MDLS_SRC_DIR/gen_qa_image_models.py --resnet50 --resnet152 --vgg19 --models_dir=$TRITON_MDLS_QA_DYNAMIC_BATCH_IMAGE_MODEL chmod -R 777 $TRITON_MDLS_QA_DYNAMIC_BATCH_IMAGE_MODEL -export TORCH_EXTENSIONS_DIR="/root/.cache/torch_extensions/" +export TORCH_EXTENSIONS_DIR=/tmp/.cache/torch_extensions/ +mkdir -p \${TORCH_EXTENSIONS_DIR} python3 $TRITON_MDLS_SRC_DIR/gen_qa_custom_ops_models.py --libtorch --models_dir=$TRITON_MDLS_QA_CUSTOM_OPS mkdir -p $TRITON_MDLS_QA_CUSTOM_OPS/libtorch_modulo/ cp \${TORCH_EXTENSIONS_DIR}/custom_modulo/custom_modulo.so $TRITON_MDLS_QA_CUSTOM_OPS/libtorch_modulo/. @@ -354,6 +348,10 @@ log_message.status "check: engine installation" if which docker-remove-upon-merge ; then log_message.info "Docker is installed." + define_model_output_directories + + SCRIPT_NAME_SUFFIX=docker.v2 define_models_generation_scripts + if ! docker volume inspect $DOCKER_VOLUME > /dev/null 2>&1; then log_message.status "docker volume: $DOCKER_VOLUME does not exist. Creating..." docker volume create $DOCKER_VOLUME --label RUNNER_ID=$RUNNER_ID --label PROJECT_NAME=$PROJECT_NAME @@ -506,7 +504,56 @@ if which docker-remove-upon-merge ; then elif which enroot ; then log_message.info "NVIDIA Enroot is installed." ; - enroot import docker://$UBUNTU_IMAGE + + TRITON_MDLS_BLD_DIR="/tmp/$CI_JOB_ID" define_model_output_directories + + SCRIPT_NAME_SUFFIX=enroot.v1 define_models_generation_scripts + + log_message.status "cleanup models folder if exists: $TRITON_MDLS_BLD_DIR" + rm -rf $TRITON_MDLS_BLD_DIR + + log_message.status "create models directory structure in: $TRITON_MDLS_BLD_DIR" + mkdir -p \ + $TRITON_MDLS_BLD_DIR \ + $TRITON_MDLS_SRC_DIR \ + $TRITON_MDLS_QA_MODEL \ + $TRITON_MDLS_QA_VARIABLE_MODEL \ + $TRITON_MDLS_QA_IDENTITY_MODEL \ + $TRITON_MDLS_QA_IDENTITY_BIG_MODEL \ + $TRITON_MDLS_QA_SHAPETENSOR_MODEL \ + $TRITON_MDLS_QA_RESHAPE_MODEL \ + $TRITON_MDLS_QA_SEQUENCE_MODEL \ + $TRITON_MDLS_QA_DYNA_SEQUENCE_MODEL \ + $TRITON_MDLS_QA_DYNA_SEQUENCE_IMPLICIT_MODEL \ + $TRITON_MDLS_QA_VARIABLE_SEQUENCE_MODEL \ + $TRITON_MDLS_QA_ENSEMBLE_MODEL \ + $TRITON_MDLS_QA_NOSHAPE_MODEL \ + $TRITON_MDLS_QA_TRT_PLUGIN_MODEL \ + $TRITON_MDLS_QA_RAGGED_MODEL \ + $TRITON_MDLS_QA_TRT_FORMAT_MODEL \ + $TRITON_MDLS_QA_TRT_DATA_DEPENDENT_MODEL \ + $TRITON_MDLS_QA_SEQUENCE_IMPLICIT_MODEL \ + $TRITON_MDLS_QA_VARIABLE_SEQUENCE_IMPLICIT_MODEL \ + $TRITON_MDLS_QA_SEQUENCE_INITIAL_STATE_IMPLICIT_MODEL \ + $TRITON_MDLS_QA_VARIABLE_SEQUENCE_INITIAL_STATE_IMPLICIT_MODEL \ + $TRITON_MDLS_QA_TORCHTRT_MODEL \ + $TRITON_MDLS_QA_SCALAR_MODELS \ + $TRITON_MDLS_QA_DYNAMIC_BATCH_IMAGE_MODEL + + log_message.status "copy scripts to: $TRITON_MDLS_SRC_DIR" + cp -rv $TRITON_MDLS_BASE_SCRIPT_DIR/* $TRITON_MDLS_SRC_DIR/ + + # log_message.status "enroot import: $UBUNTU_IMAGE to openvino.ubuntu.$CI_JOB_ID.enroot.sqsh" + # enroot import --output openvino.ubuntu.$CI_JOB_ID.enroot.sqsh docker://$UBUNTU_IMAGE + # enroot create --name openvino.ubuntu.$CI_JOB_ID openvino.ubuntu.$CI_JOB_ID.enroot.sqsh + # enroot start -m /tmp/$CI_JOB_ID:/tmp/$CI_JOB_ID openvino.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT + + log_message.status "enroot import: $PYTORCH_IMAGE to /tmp/pytorch.$CI_JOB_ID.enroot.sqsh" + enroot import --output /tmp/pytorch.$CI_JOB_ID.enroot.sqsh docker://$PYTORCH_IMAGE + log_message.status "enroot create: pytorch.$CI_JOB_ID" + enroot create --name pytorch.$CI_JOB_ID /tmp/pytorch.$CI_JOB_ID.enroot.sqsh + log_message.status "enroot start: pytorch.$CI_JOB_ID" + enroot start --rw -m /tmp:/tmp pytorch.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT From 5f049179d0998b5c1e24e6b2faf439abe3f3e5c7 Mon Sep 17 00:00:00 2001 From: Misha Chornyi Date: Thu, 4 Dec 2025 17:51:10 -0800 Subject: [PATCH 4/7] models: OpenVino + ONNX Runtime --- qa/common/gen_qa_model_repository | 36 +++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 5451b9a0c5..e6a4f1c404 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -135,6 +135,22 @@ nvidia-smi --query-gpu=compute_cap,compute_mode,driver_version,name,index --form nvidia-smi || true set -e set -x +export DEBIAN_FRONTEND=noninteractive +apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + libprotobuf-dev \ + protobuf-compiler \ + python3 \ + python3-dev \ + python3-pip \ + wget \ + gnupg2 \ + software-properties-common + +ln -s /usr/bin/python3 /usr/bin/python + pip3 install "numpy<=1.23.5" setuptools pip3 install openvino==$OPENVINO_VERSION @@ -543,13 +559,25 @@ elif which enroot ; then log_message.status "copy scripts to: $TRITON_MDLS_SRC_DIR" cp -rv $TRITON_MDLS_BASE_SCRIPT_DIR/* $TRITON_MDLS_SRC_DIR/ - # log_message.status "enroot import: $UBUNTU_IMAGE to openvino.ubuntu.$CI_JOB_ID.enroot.sqsh" - # enroot import --output openvino.ubuntu.$CI_JOB_ID.enroot.sqsh docker://$UBUNTU_IMAGE - # enroot create --name openvino.ubuntu.$CI_JOB_ID openvino.ubuntu.$CI_JOB_ID.enroot.sqsh - # enroot start -m /tmp/$CI_JOB_ID:/tmp/$CI_JOB_ID openvino.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT + log_message.status "enroot import: $UBUNTU_IMAGE to ubuntu.$CI_JOB_ID.enroot.sqsh" + enroot import --output /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh docker://$UBUNTU_IMAGE + + + log_message.status "enroot create: openvino.ubuntu.$CI_JOB_ID" + enroot create --name openvino.ubuntu.$CI_JOB_ID /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh + log_message.info "enroot start: openvino.ubuntu.$CI_JOB_ID" + enroot start --root --rw -m /tmp:/tmp openvino.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT + + + log_message.status "enroot create: onnxruntime.ubuntu.$CI_JOB_ID" + enroot create --name onnxruntime.ubuntu.$CI_JOB_ID /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh + log_message.info "enroot start: onnxruntime.ubuntu.$CI_JOB_ID" + enroot start --root --rw -m /tmp:/tmp onnxruntime.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT + log_message.status "enroot import: $PYTORCH_IMAGE to /tmp/pytorch.$CI_JOB_ID.enroot.sqsh" enroot import --output /tmp/pytorch.$CI_JOB_ID.enroot.sqsh docker://$PYTORCH_IMAGE + log_message.status "enroot create: pytorch.$CI_JOB_ID" enroot create --name pytorch.$CI_JOB_ID /tmp/pytorch.$CI_JOB_ID.enroot.sqsh log_message.status "enroot start: pytorch.$CI_JOB_ID" From b0964c9b6e35fd38d42add327eb6d48a4eeda94e Mon Sep 17 00:00:00 2001 From: Misha Chornyi Date: Thu, 4 Dec 2025 18:39:19 -0800 Subject: [PATCH 5/7] models: Enroot enabled --- qa/common/gen_qa_model_repository | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index e6a4f1c404..125fe9e4a0 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -583,9 +583,14 @@ elif which enroot ; then log_message.status "enroot start: pytorch.$CI_JOB_ID" enroot start --rw -m /tmp:/tmp pytorch.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT - + log_message.status "enroot import: $TENSORRT_IMAGE to /tmp/tensorrt.$CI_JOB_ID.enroot.sqsh" + enroot import --output /tmp/tensorrt.$CI_JOB_ID.enroot.sqsh docker://$TENSORRT_IMAGE + log_message.status "enroot create: tensorrt.$CI_JOB_ID" + enroot create --name tensorrt.$CI_JOB_ID /tmp/tensorrt.$CI_JOB_ID.enroot.sqsh + log_message.status "enroot start: tensorrt.$CI_JOB_ID" + enroot start --rw -m /tmp:/tmp tensorrt.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TRTSCRIPT else log_message.warning "Neither Docker nor NVIDIA Enroot is installed." ; - log_message.warning "Please install either Docker or NVIDIA Enroot to generate the models." ; + log_message.warning "Please install Docker or NVIDIA Enroot to generate the models." ; fi From bd3e553b6212e92f3c00d4a9a7fff1c600c70bd2 Mon Sep 17 00:00:00 2001 From: Misha Chornyi Date: Thu, 4 Dec 2025 19:05:47 -0800 Subject: [PATCH 6/7] Revert "temp: condition bypass." This reverts commit 2f642d154057ad647f3ff3f6a65e018574b4a9e1. --- qa/common/gen_qa_model_repository | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 125fe9e4a0..c29bba7514 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -361,7 +361,7 @@ EOF } log_message.status "check: engine installation" -if which docker-remove-upon-merge ; then +if which docker ; then log_message.info "Docker is installed." define_model_output_directories From a1bb13cf8d8bffeb903440f49a0252d117a8a29b Mon Sep 17 00:00:00 2001 From: Misha Chornyi Date: Thu, 4 Dec 2025 20:25:56 -0800 Subject: [PATCH 7/7] add conditional variable --- qa/common/gen_qa_model_repository | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index c29bba7514..7d106fcae7 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -69,6 +69,8 @@ PYTORCH_IMAGE=${PYTORCH_IMAGE:=nvcr.io/nvidia/pytorch:$TRITON_VERSION-py3} TENSORRT_IMAGE=${TENSORRT_IMAGE:=nvcr.io/nvidia/tensorrt:$TRITON_VERSION-py3} CUDA_DEVICE=${NV_GPU:=0} MODEL_TYPE=${MODEL_TYPE:-""} +TRITON_MODELS_USE_DOCKER=${TRITON_MODELS_USE_DOCKER:-"1"} +TRITON_MODELS_USE_ENROOT=${TRITON_MODELS_USE_ENROOT:-"1"} log_message.status "define: CI related parameters" CI_JOB_ID=${CI_JOB_ID:=$(date +%Y%m%d_%H%M)} @@ -361,7 +363,7 @@ EOF } log_message.status "check: engine installation" -if which docker ; then +if [ "$TRITON_MODELS_USE_DOCKER" -eq 1 ] && which docker ; then log_message.info "Docker is installed." define_model_output_directories @@ -518,7 +520,7 @@ if which docker ; then docker volume rm $DOCKER_VOLUME fi # [ -z $CI ] -elif which enroot ; then +elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.info "NVIDIA Enroot is installed." ; TRITON_MDLS_BLD_DIR="/tmp/$CI_JOB_ID" define_model_output_directories