diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 2edfcf325d..7d106fcae7 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -69,6 +69,8 @@ PYTORCH_IMAGE=${PYTORCH_IMAGE:=nvcr.io/nvidia/pytorch:$TRITON_VERSION-py3} TENSORRT_IMAGE=${TENSORRT_IMAGE:=nvcr.io/nvidia/tensorrt:$TRITON_VERSION-py3} CUDA_DEVICE=${NV_GPU:=0} MODEL_TYPE=${MODEL_TYPE:-""} +TRITON_MODELS_USE_DOCKER=${TRITON_MODELS_USE_DOCKER:-"1"} +TRITON_MODELS_USE_ENROOT=${TRITON_MODELS_USE_ENROOT:-"1"} log_message.status "define: CI related parameters" CI_JOB_ID=${CI_JOB_ID:=$(date +%Y%m%d_%H%M)} @@ -80,50 +82,54 @@ DOCKER_VOLUME=${DOCKER_VOLUME:=volume.gen_qa_model_repository.${CI_JOB_ID}} DOCKER_VOLUME_CONTAINER=${DOCKER_VOLUME}.gen_qa_model_repository.${CI_JOB_ID} DOCKER_GPU_ARGS=${DOCKER_GPU_ARGS:-$([[ $RUNNER_GPUS =~ ^[0-9] ]] && eval $NV_DOCKER_ARGS || echo "--gpus device=$CUDA_DEVICE" )} -log_message.status "define: model output directories" -TRITON_MDLS_BLD_DIR=${TRITON_MDLS_BLD_DIR:=/mnt/$CI_JOB_ID} -TRITON_MDLS_SRC_DIR=${TRITON_MDLS_SRC_DIR:=$TRITON_MDLS_BLD_DIR/gen_srcdir} -TRITON_MDLS_QA_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_model_repository -TRITON_MDLS_QA_VARIABLE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_variable_model_repository -TRITON_MDLS_QA_IDENTITY_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_identity_model_repository -TRITON_MDLS_QA_IDENTITY_BIG_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_identity_big_model_repository -TRITON_MDLS_QA_SHAPETENSOR_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_shapetensor_model_repository -TRITON_MDLS_QA_RESHAPE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_reshape_model_repository -TRITON_MDLS_QA_SEQUENCE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_sequence_model_repository -TRITON_MDLS_QA_DYNA_SEQUENCE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_dyna_sequence_model_repository -TRITON_MDLS_QA_DYNA_SEQUENCE_IMPLICIT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_dyna_sequence_implicit_model_repository -TRITON_MDLS_QA_VARIABLE_SEQUENCE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_variable_sequence_model_repository -TRITON_MDLS_QA_ENSEMBLE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_ensemble_model_repository -TRITON_MDLS_QA_NOSHAPE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_noshape_model_repository -TRITON_MDLS_QA_TRT_PLUGIN_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_trt_plugin_model_repository -TRITON_MDLS_QA_RAGGED_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_ragged_model_repository -TRITON_MDLS_QA_TRT_FORMAT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_trt_format_model_repository -TRITON_MDLS_QA_TRT_DATA_DEPENDENT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_trt_data_dependent_model_repository -TRITON_MDLS_QA_SEQUENCE_IMPLICIT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_sequence_implicit_model_repository -TRITON_MDLS_QA_VARIABLE_SEQUENCE_IMPLICIT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_variable_sequence_implicit_model_repository -TRITON_MDLS_QA_SEQUENCE_INITIAL_STATE_IMPLICIT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_sequence_initial_state_implicit_model_repository -TRITON_MDLS_QA_VARIABLE_SEQUENCE_INITIAL_STATE_IMPLICIT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_variable_sequence_initial_state_implicit_model_repository -TRITON_MDLS_QA_TORCHTRT_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/torchtrt_model_store -TRITON_MDLS_QA_SCALAR_MODELS=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_scalar_models -TRITON_MDLS_QA_DYNAMIC_BATCH_IMAGE_MODEL=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_dynamic_batch_image_model_repository -TRITON_MDLS_QA_CUSTOM_OPS=$TRITON_MDLS_BLD_DIR/$TRITON_VERSION/qa_custom_ops/libtorch_custom_ops - -log_message.status "define: script names" - -OPENVINOSCRIPT=gen.OpenVINO.gen_qa_model_repository.v2.sh -log_message.info "OpenVINO script: " "${OPENVINOSCRIPT}" - -ONNXSCRIPT=gen.ONNXRuntime.gen_qa_model_repository.v2.sh -log_message.info "ONNX script: " "${ONNXSCRIPT}" - -TORCHSCRIPT=gen.PyTorch.gen_qa_model_repository.v2.sh -log_message.info "PyTorch script: " "${TORCHSCRIPT}" - -TRTSCRIPT=gen.TensorRT.gen_qa_model_repository.v2.sh -log_message.info "TensorRT script: " "${TRTSCRIPT}" - -log_message.status "create: OpenVINO script - ${OPENVINOSCRIPT}" -cat > $OPENVINOSCRIPT < $OPENVINOSCRIPT < $ONNXSCRIPT < $ONNXSCRIPT < $TORCHSCRIPT < $TORCHSCRIPT < $TRTSCRIPT < $TRTSCRIPT < /dev/null 2>&1; then log_message.status "docker volume: $DOCKER_VOLUME does not exist. Creating..." docker volume create $DOCKER_VOLUME --label RUNNER_ID=$RUNNER_ID --label PROJECT_NAME=$PROJECT_NAME @@ -498,9 +520,79 @@ if which docker ; then docker volume rm $DOCKER_VOLUME fi # [ -z $CI ] -elif which enroot ; then +elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.info "NVIDIA Enroot is installed." ; + + TRITON_MDLS_BLD_DIR="/tmp/$CI_JOB_ID" define_model_output_directories + + SCRIPT_NAME_SUFFIX=enroot.v1 define_models_generation_scripts + + log_message.status "cleanup models folder if exists: $TRITON_MDLS_BLD_DIR" + rm -rf $TRITON_MDLS_BLD_DIR + + log_message.status "create models directory structure in: $TRITON_MDLS_BLD_DIR" + mkdir -p \ + $TRITON_MDLS_BLD_DIR \ + $TRITON_MDLS_SRC_DIR \ + $TRITON_MDLS_QA_MODEL \ + $TRITON_MDLS_QA_VARIABLE_MODEL \ + $TRITON_MDLS_QA_IDENTITY_MODEL \ + $TRITON_MDLS_QA_IDENTITY_BIG_MODEL \ + $TRITON_MDLS_QA_SHAPETENSOR_MODEL \ + $TRITON_MDLS_QA_RESHAPE_MODEL \ + $TRITON_MDLS_QA_SEQUENCE_MODEL \ + $TRITON_MDLS_QA_DYNA_SEQUENCE_MODEL \ + $TRITON_MDLS_QA_DYNA_SEQUENCE_IMPLICIT_MODEL \ + $TRITON_MDLS_QA_VARIABLE_SEQUENCE_MODEL \ + $TRITON_MDLS_QA_ENSEMBLE_MODEL \ + $TRITON_MDLS_QA_NOSHAPE_MODEL \ + $TRITON_MDLS_QA_TRT_PLUGIN_MODEL \ + $TRITON_MDLS_QA_RAGGED_MODEL \ + $TRITON_MDLS_QA_TRT_FORMAT_MODEL \ + $TRITON_MDLS_QA_TRT_DATA_DEPENDENT_MODEL \ + $TRITON_MDLS_QA_SEQUENCE_IMPLICIT_MODEL \ + $TRITON_MDLS_QA_VARIABLE_SEQUENCE_IMPLICIT_MODEL \ + $TRITON_MDLS_QA_SEQUENCE_INITIAL_STATE_IMPLICIT_MODEL \ + $TRITON_MDLS_QA_VARIABLE_SEQUENCE_INITIAL_STATE_IMPLICIT_MODEL \ + $TRITON_MDLS_QA_TORCHTRT_MODEL \ + $TRITON_MDLS_QA_SCALAR_MODELS \ + $TRITON_MDLS_QA_DYNAMIC_BATCH_IMAGE_MODEL + + log_message.status "copy scripts to: $TRITON_MDLS_SRC_DIR" + cp -rv $TRITON_MDLS_BASE_SCRIPT_DIR/* $TRITON_MDLS_SRC_DIR/ + + log_message.status "enroot import: $UBUNTU_IMAGE to ubuntu.$CI_JOB_ID.enroot.sqsh" + enroot import --output /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh docker://$UBUNTU_IMAGE + + + log_message.status "enroot create: openvino.ubuntu.$CI_JOB_ID" + enroot create --name openvino.ubuntu.$CI_JOB_ID /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh + log_message.info "enroot start: openvino.ubuntu.$CI_JOB_ID" + enroot start --root --rw -m /tmp:/tmp openvino.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT + + + log_message.status "enroot create: onnxruntime.ubuntu.$CI_JOB_ID" + enroot create --name onnxruntime.ubuntu.$CI_JOB_ID /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh + log_message.info "enroot start: onnxruntime.ubuntu.$CI_JOB_ID" + enroot start --root --rw -m /tmp:/tmp onnxruntime.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT + + + log_message.status "enroot import: $PYTORCH_IMAGE to /tmp/pytorch.$CI_JOB_ID.enroot.sqsh" + enroot import --output /tmp/pytorch.$CI_JOB_ID.enroot.sqsh docker://$PYTORCH_IMAGE + + log_message.status "enroot create: pytorch.$CI_JOB_ID" + enroot create --name pytorch.$CI_JOB_ID /tmp/pytorch.$CI_JOB_ID.enroot.sqsh + log_message.status "enroot start: pytorch.$CI_JOB_ID" + enroot start --rw -m /tmp:/tmp pytorch.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT + + log_message.status "enroot import: $TENSORRT_IMAGE to /tmp/tensorrt.$CI_JOB_ID.enroot.sqsh" + enroot import --output /tmp/tensorrt.$CI_JOB_ID.enroot.sqsh docker://$TENSORRT_IMAGE + log_message.status "enroot create: tensorrt.$CI_JOB_ID" + enroot create --name tensorrt.$CI_JOB_ID /tmp/tensorrt.$CI_JOB_ID.enroot.sqsh + log_message.status "enroot start: tensorrt.$CI_JOB_ID" + enroot start --rw -m /tmp:/tmp tensorrt.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TRTSCRIPT + else log_message.warning "Neither Docker nor NVIDIA Enroot is installed." ; - log_message.warning "Please install either Docker or NVIDIA Enroot to generate the models." ; + log_message.warning "Please install Docker or NVIDIA Enroot to generate the models." ; fi