From d3baf30d03d4040120b2d1ec2626702214a06a69 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Tue, 25 Nov 2025 15:59:49 -0500 Subject: [PATCH 1/4] feat: PyTorch PT2 Model Generation This change adds the generation of PT2 format serialized PyTorch model files to the QA model generation scripts. add backend: pytorch to model config generation fix pedantic style complaints and very valid copyright mistakes pedantic style cop fix fix variable name error after rebase rename model add nvcc to pytorch path remove unnecessary changes add torch.export.export generation support to libtorch remove set -x from generated scripts improving gen_qa_models.py print statements fix color add print based on flags improve color visibility remove IDE help (caused errors) pedantic style cop fixes specify cuda fix libtorch pt2 generation add torchvision generation fix aoti model generation output types --- qa/common/gen_qa_model_repository | 89 ++- qa/common/gen_qa_models.py | 495 ++++++++++++-- qa/common/resnet50_labels.txt | 1000 +++++++++++++++++++++++++++++ qa/common/test_util.py | 15 +- 4 files changed, 1518 insertions(+), 81 deletions(-) create mode 100644 qa/common/resnet50_labels.txt diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index bcd00c90d8..38e00ee1e9 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -50,10 +50,16 @@ TRITON_MDLS_BASE_SCRIPT_DIR="$(dirname $(readlink -f $0))" TRITON_MDLS_BASE_SCRIPT_FILE="$(readlink -f $0)" -log_message.info() { local message=$@ ; echo -e "\033[34m$(date +"%Y-%m-%d %H:%M:%S") - [ INFO ] - ${message} \033[0m"; } ; -log_message.status() { local message=$@ ; echo -e "\033[37m$(date +"%Y-%m-%d %H:%M:%S") - [ STATUS ] - ${message} \033[0m"; } ; -log_message.warning() { local message=$@ ; echo -e "\033[33m$(date +"%Y-%m-%d %H:%M:%S") - [ WARNING ] - ${message} \033[0m"; } ; -log_message.error() { local message=$@ ; echo -e "\033[31m$(date +"%Y-%m-%d %H:%M:%S") - [ ERROR ] - ${message} \033[0m"; } ; +COLOR_ERROR="\033[31m" +COLOR_INFO="\033[94m" +COLOR_RESET="\033[0m" +COLOR_STATUS="\033[36m" +COLOR_WARNING="\033[33m" + +log_message.info() { local message=$@ ; echo -e "${COLOR_INFO}$(date +"%Y-%m-%d %H:%M:%S") - [ INFO ] - ${message} ${COLOR_RESET}"; } ; +log_message.status() { local message=$@ ; echo -e "${COLOR_STATUS}$(date +"%Y-%m-%d %H:%M:%S") - [ STATUS ] - ${message} ${COLOR_RESET}"; } ; +log_message.warning() { local message=$@ ; echo -e "${COLOR_WARNING}$(date +"%Y-%m-%d %H:%M:%S") - [ WARNING ] - ${message} ${COLOR_RESET}"; } ; +log_message.error() { local message=$@ ; echo -e "${COLOR_ERROR}$(date +"%Y-%m-%d %H:%M:%S") - [ ERROR ] - ${message} ${COLOR_RESET}"; } ; log_message.status "Changing working directory to the script directory to: " "${TRITON_MDLS_BASE_SCRIPT_DIR}" @@ -135,8 +141,8 @@ function define_models_generation_scripts() { umask 0000 nvidia-smi --query-gpu=compute_cap,compute_mode,driver_version,name,index --format=csv || true nvidia-smi || true +echo -e "${COLOR_INFO}Generating OpenVINO models${COLOR_RESET}" set -e -set -x export DEBIAN_FRONTEND=noninteractive apt-get update && \ apt-get install -y --no-install-recommends \ @@ -170,6 +176,7 @@ chmod -R 777 $TRITON_MDLS_QA_RESHAPE_MODEL # chmod -R 777 $SVOLUME_EQDESTDIR # python3 $TRITON_MDLS_SRC_DIR/gen_qa_dyna_sequence_models.py --openvino --models_dir=$TRITON_MDLS_QA_DYNA_SEQUENCE_MODEL # chmod -R 777 $TRITON_MDLS_QA_DYNA_SEQUENCE_MODEL +exit 0 EOF log_message.status "run: chmod a+x ${OPENVINOSCRIPT}" @@ -187,8 +194,8 @@ EOF umask 0000 nvidia-smi --query-gpu=compute_cap,compute_mode,driver_version,name,index --format=csv || true nvidia-smi || true +echo -e "${COLOR_INFO}Generating ONNX models${COLOR_RESET}" set -e -set -x export DEBIAN_FRONTEND=noninteractive apt-get update && \ apt-get install -y --no-install-recommends build-essential cmake libprotobuf-dev \ @@ -233,6 +240,7 @@ python3 $TRITON_MDLS_SRC_DIR/gen_qa_identity_models.py --ensemble --models_dir=$ python3 $TRITON_MDLS_SRC_DIR/gen_qa_sequence_models.py --ensemble --models_dir=$TRITON_MDLS_QA_ENSEMBLE_MODEL/qa_sequence_model_repository python3 $TRITON_MDLS_SRC_DIR/gen_qa_sequence_models.py --ensemble --variable --models_dir=$TRITON_MDLS_QA_ENSEMBLE_MODEL/qa_variable_sequence_model_repository chmod -R 777 $TRITON_MDLS_QA_ENSEMBLE_MODEL +exit 0 EOF log_message.status "run: chmod a+x ${ONNXSCRIPT}" @@ -249,12 +257,15 @@ EOF umask 0000 nvidia-smi --query-gpu=compute_cap,compute_mode,driver_version,name,index --format=csv || true nvidia-smi || true +echo -e "${COLOR_INFO}Generating PyTorch models${COLOR_RESET}" pip3 install onnxscript set -e -set -x +PATH=$PATH:/usr/local/cuda-13.0/bin python3 $TRITON_MDLS_SRC_DIR/gen_qa_models.py --libtorch --models_dir=$TRITON_MDLS_QA_MODEL +python3 $TRITON_MDLS_SRC_DIR/gen_qa_models.py --torch-aoti --models_dir=$TRITON_MDLS_QA_MODEL chmod -R 777 $TRITON_MDLS_QA_MODEL python3 $TRITON_MDLS_SRC_DIR/gen_qa_models.py --libtorch --variable --models_dir=$TRITON_MDLS_QA_VARIABLE_MODEL +python3 $TRITON_MDLS_SRC_DIR/gen_qa_models.py --torch-aoti --variable --models_dir=$TRITON_MDLS_QA_VARIABLE_MODEL chmod -R 777 $TRITON_MDLS_QA_VARIABLE_MODEL python3 $TRITON_MDLS_SRC_DIR/gen_qa_identity_models.py --libtorch --models_dir=$TRITON_MDLS_QA_IDENTITY_MODEL chmod -R 777 $TRITON_MDLS_QA_IDENTITY_MODEL @@ -271,7 +282,7 @@ chmod -R 777 $TRITON_MDLS_QA_VARIABLE_SEQUENCE_IMPLICIT_MODEL python3 $TRITON_MDLS_SRC_DIR/gen_qa_dyna_sequence_models.py --libtorch --models_dir=$TRITON_MDLS_QA_DYNA_SEQUENCE_MODEL chmod -R 777 $TRITON_MDLS_QA_DYNA_SEQUENCE_MODEL if [ -z "$MODEL_TYPE" ] || [ "$MODEL_TYPE" != "igpu" ]; then - nvidia-smi --query-gpu=compute_cap | grep -qz 12.1 && echo -e '\033[33m[WARNING]\033[0m Skipping model generation for Torch TensorRT' || python3 $TRITON_MDLS_SRC_DIR/gen_qa_torchtrt_models.py --models_dir=$TRITON_MDLS_QA_TORCHTRT_MODEL + nvidia-smi --query-gpu=compute_cap | grep -qz 12.1 && echo -e '${COLOR_WARNING}[WARNING]${COLOR_RESET} Skipping model generation for Torch TensorRT${COLOR_RESET}' || python3 $TRITON_MDLS_SRC_DIR/gen_qa_torchtrt_models.py --models_dir=$TRITON_MDLS_QA_TORCHTRT_MODEL chmod -R 777 $TRITON_MDLS_QA_TORCHTRT_MODEL fi python3 $TRITON_MDLS_SRC_DIR/gen_qa_ragged_models.py --libtorch --models_dir=$TRITON_MDLS_QA_RAGGED_MODEL @@ -286,6 +297,7 @@ python3 $TRITON_MDLS_SRC_DIR/gen_qa_custom_ops_models.py --libtorch --models_dir mkdir -p $TRITON_MDLS_QA_CUSTOM_OPS/libtorch_modulo/ cp \${TORCH_EXTENSIONS_DIR}/custom_modulo/custom_modulo.so $TRITON_MDLS_QA_CUSTOM_OPS/libtorch_modulo/. chmod -R 777 $TRITON_MDLS_QA_CUSTOM_OPS +exit 0 EOF log_message.status "run: chmod a+x ${TORCHSCRIPT}" @@ -302,8 +314,8 @@ EOF umask 0000 nvidia-smi --query-gpu=compute_cap,compute_mode,driver_version,name,index --format=csv || true nvidia-smi || true +echo -e "${COLOR_INFO}Generating TensorRT models${COLOR_RESET}" set -e -set -x dpkg -l | grep TensorRT export TRT_SUPPRESS_DEPRECATION_WARNINGS=1 # Models using shape tensor i/o @@ -338,7 +350,7 @@ python3 $TRITON_MDLS_SRC_DIR/gen_qa_ragged_models.py --tensorrt --models_dir=$TR chmod -R 777 $TRITON_MDLS_QA_RAGGED_MODEL python3 $TRITON_MDLS_SRC_DIR/gen_qa_trt_format_models.py --models_dir=$TRITON_MDLS_QA_TRT_FORMAT_MODEL chmod -R 777 $TRITON_MDLS_QA_TRT_FORMAT_MODEL -nvidia-smi --query-gpu=compute_cap | grep -qz 11.0 && echo -e '\033[33m[WARNING]\033[0m Skipping model generation for data dependent shape' || python3 $TRITON_MDLS_SRC_DIR/gen_qa_trt_data_dependent_shape.py --models_dir=$TRITON_MDLS_QA_TRT_DATA_DEPENDENT_MODEL +nvidia-smi --query-gpu=compute_cap | grep -qz 11.0 && echo -e '${COLOR_WARNING}[WARNING]${COLOR_RESET} Skipping model generation for data dependent shape${COLOR_RESET}' || python3 $TRITON_MDLS_SRC_DIR/gen_qa_trt_data_dependent_shape.py --models_dir=$TRITON_MDLS_QA_TRT_DATA_DEPENDENT_MODEL chmod -R 777 $TRITON_MDLS_QA_TRT_DATA_DEPENDENT_MODEL # Make shared library for custom Hardmax plugin. if [ -d "/usr/src/tensorrt/samples/python/onnx_custom_plugin" ]; then @@ -362,6 +374,7 @@ rm -rf build && mkdir build && \ cd build && cmake .. && make -j && cp libcustomHardmaxPlugin.so $TRITON_MDLS_QA_TRT_PLUGIN_MODEL/. LD_PRELOAD=$TRITON_MDLS_QA_TRT_PLUGIN_MODEL/libcustomHardmaxPlugin.so python3 $TRITON_MDLS_SRC_DIR/gen_qa_trt_plugin_models.py --models_dir=$TRITON_MDLS_QA_TRT_PLUGIN_MODEL chmod -R 777 $TRITON_MDLS_QA_TRT_PLUGIN_MODEL +exit 0 EOF log_message.status "run: chmod a+x ${TRTSCRIPT}" @@ -395,6 +408,7 @@ if [ "$TRITON_MODELS_USE_DOCKER" -eq 1 ] && which docker ; then docker pull $UBUNTU_IMAGE log_message.status "docker volume: create destination directory on volume" + log_message.info "docker run -v $DOCKER_VOLUME:/mnt -w /mnt/$CI_JOB_ID $UBUNTU_IMAGE mkdir -p gen_srcdir ${TRITON_VERSION}" docker run \ --rm \ --label RUNNER_ID=$RUNNER_ID \ @@ -404,6 +418,7 @@ if [ "$TRITON_MODELS_USE_DOCKER" -eq 1 ] && which docker ; then $UBUNTU_IMAGE \ mkdir -p gen_srcdir ${TRITON_VERSION} + log_message.status "docker volume: create model directories on volume" docker run \ --rm \ --label RUNNER_ID=$RUNNER_ID \ @@ -439,6 +454,7 @@ if [ "$TRITON_MODELS_USE_DOCKER" -eq 1 ] && which docker ; then $TRITON_MDLS_QA_DYNAMIC_BATCH_IMAGE_MODEL log_message.status "docker container: create container $DOCKER_VOLUME_CONTAINER" + log_message.info "docker create --name $DOCKER_VOLUME_CONTAINER -v $DOCKER_VOLUME:/mnt -w /mnt/$CI_JOB_ID $UBUNTU_IMAGE" docker create \ --label RUNNER_ID=$RUNNER_ID \ --label PROJECT_NAME=$PROJECT_NAME \ @@ -452,74 +468,103 @@ if [ "$TRITON_MODELS_USE_DOCKER" -eq 1 ] && which docker ; then if [[ "aarch64" != $(uname -m) ]] ; then - log_message.info "docker run: $OPENVINOSCRIPT" + log_message.status "docker run: $OPENVINOSCRIPT" + log_message.info "docker run $DOCKER_GPU_ARGS -v $DOCKER_VOLUME:/mnt $UBUNTU_IMAGE bash -e $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT" docker run \ --rm \ + -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR \ --label RUNNER_ID=$RUNNER_ID \ --label PROJECT_NAME=$PROJECT_NAME \ $DOCKER_GPU_ARGS \ -v $DOCKER_VOLUME:/mnt \ + -t \ $UBUNTU_IMAGE \ - bash -xe $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT + bash -e $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT + + exit_code=$? - if [ $? -ne 0 ]; then + if [ $exit_code -ne 0 ]; then log_message.error "docker run: ${OPENVINOSCRIPT} failed" exit 1 fi + + rm $OPENVINOSCRIPT fi # [[ "aarch64" != $(uname -m) ]] - log_message.info "docker run: $ONNXSCRIPT" + log_message.status "docker run: $ONNXSCRIPT" + log_message.info "docker run $DOCKER_GPU_ARGS -v $DOCKER_VOLUME:/mnt $UBUNTU_IMAGE bash -e $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT" docker run \ --rm \ + -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR \ --label RUNNER_ID=$RUNNER_ID \ --label PROJECT_NAME=$PROJECT_NAME \ $DOCKER_GPU_ARGS \ -v $DOCKER_VOLUME:/mnt \ + -t \ $UBUNTU_IMAGE \ - bash -xe $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT + bash -e $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT - if [ $? -ne 0 ]; then + exit_code=$? + + if [ $exit_code -ne 0 ]; then log_message.error "docker run: ${ONNXSCRIPT} failed" exit 1 fi + rm $ONNXSCRIPT + log_message.status "docker pull: $PYTORCH_IMAGE" + log_message.info "docker pull $PYTORCH_IMAGE" docker pull $PYTORCH_IMAGE - log_message.info "docker run: $TORCHSCRIPT" + log_message.status "docker run: $TORCHSCRIPT" + log_message.info "docker run $DOCKER_GPU_ARGS -v $DOCKER_VOLUME:/mnt $PYTORCH_IMAGE bash -e $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT" docker run \ --rm \ + -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR \ --label RUNNER_ID=$RUNNER_ID \ --label PROJECT_NAME=$PROJECT_NAME \ $DOCKER_GPU_ARGS \ -v $DOCKER_VOLUME:/mnt \ + -t \ $PYTORCH_IMAGE \ - bash -xe $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT + bash -e $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT - if [ $? -ne 0 ]; then + exit_code=$? + + if [ $exit_code -ne 0 ]; then log_message.error "docker run: ${TORCHSCRIPT} failed" exit 1 fi + rm $TORCHSCRIPT + if [ "$MODEL_TYPE" != "igpu" ] ; then log_message.status "docker pull: $TENSORRT_IMAGE" docker pull $TENSORRT_IMAGE - log_message.info "docker run: $TRTSCRIPT" + log_message.status "docker run: $TRTSCRIPT" + log_message.info "docker run $DOCKER_GPU_ARGS -v $DOCKER_VOLUME:/mnt $TENSORRT_IMAGE bash -e $TRITON_MDLS_SRC_DIR/$TRTSCRIPT" docker run \ --rm \ + -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR \ --label RUNNER_ID=$RUNNER_ID \ --label PROJECT_NAME=$PROJECT_NAME \ $DOCKER_GPU_ARGS \ -v $DOCKER_VOLUME:/mnt \ + -t \ -e TRT_VERBOSE \ $TENSORRT_IMAGE \ - bash -xe $TRITON_MDLS_SRC_DIR/$TRTSCRIPT + bash -e $TRITON_MDLS_SRC_DIR/$TRTSCRIPT - if [ $? -ne 0 ]; then + exit_code=$? + + if [ $exit_code -ne 0 ]; then log_message.error "docker run: ${TRTSCRIPT} failed" exit 1 fi + + rm $TRTSCRIPT fi # [ "$MODEL_TYPE" != "igpu" ] if [ -z $CI ] ; then diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py index cfce75be39..27c2db51ea 100755 --- a/qa/common/gen_qa_models.py +++ b/qa/common/gen_qa_models.py @@ -27,8 +27,10 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse +from cProfile import label import os from builtins import range +import sys import gen_ensemble_model_utils as emu import numpy as np @@ -46,6 +48,14 @@ from typing import List, Tuple +_color_blue = "\033[94m" +_color_green = "\033[32m" +_color_magenta = "\033[35m" +_color_red = "\033[31m" +_color_reset = "\033[0m" +_color_yellow = "\033[33m" + + def create_plan_dynamic_rf_modelfile( models_dir, max_batch, @@ -348,6 +358,7 @@ def create_plan_dynamic_modelfile( output0_dtype, output1_dtype, ) + print(f"{_color_green}Creating model {model_name}{_color_reset}") if min_dim != 1 or max_dim != 32: model_name = "{}-{}-{}".format(model_name, min_dim, max_dim) @@ -467,6 +478,7 @@ def create_plan_fixed_rf_modelfile( output0_dtype, output1_dtype, ) + print(f"{_color_green}Creating model {model_name}{_color_reset}") model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) try: @@ -554,6 +566,7 @@ def create_plan_fixed_modelfile( output0_dtype, output1_dtype, ) + print(f"{_color_green}Creating model {model_name}{_color_reset}") model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) try: @@ -726,6 +739,7 @@ def create_plan_modelconfig( output0_dtype, output1_dtype, ) + print(f"{_color_green}Creating config for {model_name}{_color_reset}") if min_dim != 1 or max_dim != 32: model_name = "{}-{}-{}".format(model_name, min_dim, max_dim) @@ -831,7 +845,7 @@ def create_plan_modelconfig( try: os.makedirs(config_dir) - except OSError as ex: + except OSError: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -879,6 +893,7 @@ def create_onnx_modelfile( output0_dtype, output1_dtype, ) + print(f"{_color_green}Creating model {model_name}{_color_reset}") model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) batch_dim = [] if max_batch == 0 else [None] @@ -984,6 +999,9 @@ def create_onnx_modelconfig( output0_dtype, output1_dtype, ) + + print(f"{_color_green}Creating config for {model_name}{_color_reset}") + config_dir = models_dir + "/" + model_name # [TODO] move create_general_modelconfig() out of emu as it is general @@ -1005,15 +1023,15 @@ def create_onnx_modelconfig( try: os.makedirs(config_dir) - except OSError as ex: + except OSError: pass # ignore existing dir - with open(config_dir + "/config.pbtxt", "w") as cfile: - cfile.write(config) + with open(config_dir + "/config.pbtxt", "w") as file: + file.write(config) - with open(config_dir + "/output0_labels.txt", "w") as lfile: + with open(config_dir + "/output0_labels.txt", "w") as file: for l in range(output0_label_cnt): - lfile.write("label" + str(l) + "\n") + file.write("label" + str(l) + "\n") def create_libtorch_modelfile( @@ -1048,6 +1066,9 @@ def create_libtorch_modelfile( output0_dtype, output1_dtype, ) + + print(f"{_color_green}Creating model {model_name}{_color_reset}") + # handle for -1 (when variable) since can't create tensor with shape of [-1] input_shape = [abs(ips) for ips in input_shape] @@ -1057,7 +1078,6 @@ def create_libtorch_modelfile( and (output0_dtype != np_dtype_string) and (output1_dtype != np_dtype_string) ): - class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1085,7 +1105,6 @@ def forward(self, INPUT0: List[str], INPUT1: List[str]): and (output0_dtype == np_dtype_string) and (output1_dtype == np_dtype_string) ): - class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1121,7 +1140,6 @@ def forward( and (output0_dtype == np_dtype_string) and (output1_dtype != np_dtype_string) ): - class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1154,7 +1172,6 @@ def forward( and (output0_dtype != np_dtype_string) and (output1_dtype == np_dtype_string) ): - class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1187,7 +1204,6 @@ def forward( and (output0_dtype == np_dtype_string) and (output1_dtype == np_dtype_string) ): - class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1211,7 +1227,6 @@ def forward(self, INPUT0, INPUT1) -> Tuple[List[str], List[str]]: and (output0_dtype != np_dtype_string) and (output1_dtype == np_dtype_string) ): - class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1234,7 +1249,6 @@ def forward(self, INPUT0, INPUT1) -> Tuple[torch.Tensor, List[str]]: and (output0_dtype == np_dtype_string) and (output1_dtype != np_dtype_string) ): - class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1253,7 +1267,6 @@ def forward(self, INPUT0, INPUT1) -> Tuple[List[str], torch.Tensor]: return op0, op1 else: - class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1273,14 +1286,216 @@ def forward(self, INPUT0, INPUT1): addSubModel = AddSubNet((torch_output0_dtype, torch_output1_dtype, swap)) traced = torch.jit.script(addSubModel) - model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) + model_version_dir = f"{models_dir}/{model_name}/{model_version}" try: os.makedirs(model_version_dir) - except OSError as ex: + except OSError: + pass # ignore existing dir + + traced.save(f"{model_version_dir}/model.pt") + + +def generate_sample_inputs( + input_shape, + input_dtype, + device, +): + # handle for -1 (when variable) since can't create tensor with shape of [-1] + input_shape = [abs(ips) for ips in input_shape] + + if input_dtype == np.int8: + input0 = torch.randint(-128, 127, input_shape, dtype=torch.int8, device=device) + input1 = torch.randint(-128, 127, input_shape, dtype=torch.int8, device=device) + elif input_dtype == np.int16: + input0 = torch.randint(-32768, 32767, input_shape, dtype=torch.int16, device=device) + input1 = torch.randint(-32768, 32767, input_shape, dtype=torch.int16, device=device) + elif input_dtype == np.int32: + input0 = torch.randint(-2147483648, 2147483647, input_shape, dtype=torch.int32, device=device) + input1 = torch.randint(-2147483648, 2147483647, input_shape, dtype=torch.int32, device=device) + elif input_dtype == np.int64: + input0 = torch.randint(-9223372036854775808, 9223372036854775807, input_shape, dtype=torch.int64, device=device) + input1 = torch.randint(-9223372036854775808, 9223372036854775807, input_shape, dtype=torch.int64, device=device) + elif input_dtype == np.float16: + input0 = torch.randn(*input_shape, dtype=torch.float16, device=device) + input1 = torch.randn(*input_shape, dtype=torch.float16, device=device) + elif input_dtype == np.float32: + input0 = torch.randn(*input_shape, dtype=torch.float32, device=device) + input1 = torch.randn(*input_shape, dtype=torch.float32, device=device) + elif input_dtype == np.float64: + input0 = torch.randn(*input_shape, dtype=torch.float64, device=device) + input1 = torch.randn(*input_shape, dtype=torch.float64, device=device) + elif input_dtype == np.uint8: + input0 = torch.randint(0, 255, input_shape, dtype=torch.uint8, device=device) + input1 = torch.randint(0, 255, input_shape, dtype=torch.uint8, device=device) + elif input_dtype == np.uint16: + input0 = torch.randint(0, 65535, input_shape, dtype=torch.uint16, device=device) + input1 = torch.randint(0, 65535, input_shape, dtype=torch.uint16, device=device) + elif input_dtype == np.uint32: + input0 = torch.randint(0, 4294967295, input_shape, dtype=torch.uint32, device=device) + input1 = torch.randint(0, 4294967295, input_shape, dtype=torch.uint32, device=device) + elif input_dtype == np.uint64: + input0 = torch.randint(0, 18446744073709551615, input_shape, dtype=torch.uint64, device=device) + input1 = torch.randint(0, 18446744073709551615, input_shape, dtype=torch.uint64, device=device) + else: + input0 = torch.randn(*input_shape, device=device) + input1 = torch.randn(*input_shape, device=device) + + return (input0, input1) + + +def np_to_dtype(np_dtype): + if np_dtype == np.int8: + return torch.int8 + elif np_dtype == np.int16: + return torch.int16 + elif np_dtype == np.int32: + return torch.int32 + elif np_dtype == np.int64: + return torch.int64 + elif np_dtype == np.float16: + return torch.float16 + elif np_dtype == np.float32: + return torch.float32 + elif np_dtype == np.float64: + return torch.float64 + elif np_dtype == np.uint8: + return torch.uint8 + elif np_dtype == np.uint16: + return torch.uint16 + elif np_dtype == np.uint32: + return torch.uint32 + elif np_dtype == np.uint64: + return torch.uint64 + else: + print( + f"{_color_yellow}warning: dtype {np_dtype} is unsupported; falling back to torch.int32{_color_reset}" + ) + return torch.int32 + +def create_torch_aoti_modelfile( + models_dir, + model_version, + input_shape, + input_dtype, + output_dtype, + swap=False, +): + model_name = tu.get_model_name( + "torch_aoti", + input_dtype, + output_dtype, + None, + ) + model_version_dir = f"{models_dir}/{model_name}/{model_version}" + + print(f"{_color_green}Creating model {model_name}{_color_reset}") + + torch_input_dtype: torch.dtype = np_to_dtype(input_dtype) + torch_output_dtype: torch.dtype = np_to_dtype(output_dtype) + + print(f"{model_name}({torch_input_dtype}) -> {torch_output_dtype}") + + # handle for -1 (when variable) since can't create tensor with shape of [-1] + input_shape = [abs(ips) for ips in input_shape] + + try: + os.makedirs(model_version_dir) + except OSError: pass # ignore existing dir - traced.save(model_version_dir + "/model.pt") + class AddSubNet(nn.Module): + def __init__( + self, + swap: bool, + input_dtype: torch.dtype, + output_dtype: torch.dtype, + ) -> None: + self.swap = swap + self.input_dtype = input_dtype + self.output_dtype = output_dtype + super(AddSubNet, self).__init__() + + def forward( + self, + INPUT0: torch.Tensor, + INPUT1: torch.Tensor + ) -> torch.Tensor: + if INPUT0.dtype != self.input_dtype: + raise TypeError( + f"INPUT0 expected {self.input_dtype} vs. actual {INPUT0.dtype} type." + ) + if INPUT1.dtype != self.input_dtype: + raise TypeError( + f"INPUT1 expected {self.input_dtype} vs. actual {INPUT1.dtype} type." + ) + return (INPUT0 - INPUT1 if self.swap else INPUT0 + INPUT1).to( + self.output_dtype, + ) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = AddSubNet(swap, torch_input_dtype, torch_output_dtype) + model.to(device) + model = model.eval() + + sample_input = generate_sample_inputs(input_shape, input_dtype, device) + + try: + ep = torch.export.export(model, sample_input) + torch._inductor.aoti_compile_and_package( + ep, + package_path=f"{model_version_dir}/model.pt2", + ) + except Exception as e: + print( + f"{_color_red}error: Failed to create model {model_name}{_color_reset}", + file=sys.stderr, + ) + print(f"\n{_color_red}{e}{_color_reset}\n", file=sys.stderr) + return False + + return True + + +def create_torchvision_aoti_modelfile( + models_dir: str, + max_batch: int, + model_version: int, +): + model_name = "torchvision_aoti" + model_version_dir = f"{models_dir}/{model_name}/{model_version}" + + try: + os.makedirs(model_version_dir) + except OSError: + pass # ignore existing dir + + print(f"{_color_green}Creating model {model_name}{_color_reset}") + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = models.resnet50(weights=models.ResNet50_Weights.DEFAULT) + model = model.to(device) + model = model.eval() + + # Example input tensor with batch size 1 and 3 color channels (RGB), height and width of 224 + input_tensor = torch.randn(max_batch, 3, 224, 224, device=device) + + try: + ep = torch.export.export(model, (input_tensor,)) + + torch._inductor.aoti_compile_and_package( + ep, + package_path=f"{model_version_dir}/model.pt2", + ) + except Exception as e: + print( + f"{_color_red}error: Failed to create model {model_name}{_color_reset}", + file=sys.stderr, + ) + print(f"\n{_color_red}{e}{_color_reset}\n", file=sys.stderr) + return False + + return True def create_libtorch_modelconfig( @@ -1312,9 +1527,9 @@ def create_libtorch_modelconfig( if version_policy is not None: type, val = version_policy if type == "latest": - version_policy_str = "{{ latest {{ num_versions: {} }}}}".format(val) + version_policy_str = f"{{ latest {{ num_versions: {val} }} }}" elif type == "specific": - version_policy_str = "{{ specific {{ versions: {} }}}}".format(val) + version_policy_str = f"{{ specific {{ versions: {val} }} }}" else: version_policy_str = "{ all { }}" @@ -1325,62 +1540,182 @@ def create_libtorch_modelconfig( output0_dtype, output1_dtype, ) - config_dir = models_dir + "/" + model_name - config = """ -name: "{}" + + print(f"{_color_green}Creating config for {model_name}{_color_reset}") + + label_filename = "output0_labels.txt" + config_dir = f"{models_dir}/{model_name}" + config = f""" +backend: "pytorch" +name: "{model_name}" platform: "pytorch_libtorch" -max_batch_size: {} -version_policy: {} +max_batch_size: {max_batch} +version_policy: {version_policy_str} input [ {{ name: "INPUT0" - data_type: {} - dims: [ {} ] + data_type: {np_to_model_dtype(input_dtype)} + dims: [ {tu.shape_to_dims_str(input_shape)} ] }}, {{ name: "INPUT1" - data_type: {} - dims: [ {} ] + data_type: {np_to_model_dtype(input_dtype)} + dims: [ {tu.shape_to_dims_str(input_shape)} ] }} ] output [ {{ name: "OUTPUT__0" - data_type: {} - dims: [ {} ] - label_filename: "output0_labels.txt" + data_type: {np_to_model_dtype(output0_dtype)} + dims: [ {tu.shape_to_dims_str(output0_shape)} ] + label_filename: "{label_filename}" }}, {{ name: "OUTPUT__1" - data_type: {} - dims: [ {} ] + data_type: {np_to_model_dtype(output1_dtype)} + dims: [ {tu.shape_to_dims_str(output1_shape)} ] }} ] -""".format( - model_name, - max_batch, - version_policy_str, - np_to_model_dtype(input_dtype), - tu.shape_to_dims_str(input_shape), - np_to_model_dtype(input_dtype), - tu.shape_to_dims_str(input_shape), - np_to_model_dtype(output0_dtype), - tu.shape_to_dims_str(output0_shape), - np_to_model_dtype(output1_dtype), - tu.shape_to_dims_str(output1_shape), - ) +""" try: os.makedirs(config_dir) - except OSError as ex: + except OSError: pass # ignore existing dir - with open(config_dir + "/config.pbtxt", "w") as cfile: - cfile.write(config) + with open(f"{config_dir}/config.pbtxt", "w") as file: + file.write(config) + print(f"Created {config_dir}/config.pbtxt") - with open(config_dir + "/output0_labels.txt", "w") as lfile: + with open(f"{config_dir}/{label_filename}", "w") as file: for l in range(output0_label_cnt): - lfile.write("label" + str(l) + "\n") + file.write("label" + str(l) + "\n") + print(f"Created {config_dir}/{label_filename}") + + +def create_torch_aoti_modelconfig( + models_dir, + input_shape, + output_shape, + input_dtype, + output_dtype, + output_label_cnt, + version_policy, +): + # Unpack version policy + version_policy_str = "{ latest { num_versions: 1 }}" + if version_policy is not None: + type, val = version_policy + if type == "latest": + version_policy_str = f"{{ latest {{ num_versions: {val} }} }}" + elif type == "specific": + version_policy_str = f"{{ specific {{ versions: {val} }} }}" + else: + version_policy_str = "{ all { }}" + + # Use a different model name for the non-batching variant + model_name = tu.get_model_name( + "torch_aoti", + input_dtype, + output_dtype, + None, + ) + + print(f"{_color_green}Creating config for {model_name}{_color_reset}") + + label_filename = "output_labels.txt" + config_dir = f"{models_dir}/{model_name}" + config = f""" +backend: "pytorch" +name: "{model_name}" +platform: "torch_aoti" +max_batch_size: 8 +version_policy: {version_policy_str} +input [ + {{ + name: "INPUT0" + data_type: {np_to_model_dtype(input_dtype)} + dims: [ {tu.shape_to_dims_str(input_shape)} ] + }}, + {{ + name: "INPUT1" + data_type: {np_to_model_dtype(input_dtype)} + dims: [ {tu.shape_to_dims_str(input_shape)} ] + }} +] +output [ + {{ + name: "OUTPUT__0" + data_type: {np_to_model_dtype(output_dtype)} + dims: [ {tu.shape_to_dims_str(output_shape)} ] + label_filename: "{label_filename}" + }} +] +instance_group [{{ kind: {"KIND_GPU" if torch.cuda.is_available() else "KIND_CPU"} }}] +""" + + try: + os.makedirs(config_dir) + except OSError: + pass # ignore existing dir + + with open(f"{config_dir}/config.pbtxt", "w") as file: + file.write(config) + print(f"Created {config_dir}/config.pbtxt") + + with open(f"{config_dir}/{label_filename}", "w") as file: + for l in range(output_label_cnt): + file.write(f"label{l}\n") + print(f"Created {config_dir}/{label_filename}") + + +def create_torchvision_aoti_modelconfig( + models_dir: str, + max_batch: int, +): + model_name = "torchvision_aoti" + label_filename = "resnet50_labels.txt" + + print(f"{_color_green}Creating config for {model_name}{_color_reset}") + + config_dir = f"{models_dir}/{model_name}" + config = f""" +backend: "pytorch" +name: "{model_name}" +platform: "torch_aoti" +max_batch_size: {max_batch} +input [ + {{ + name: "INPUT__0" + data_type: TYPE_FP32 + format: FORMAT_NCHW + dims: [ 3, 224, 224 ] + }}] +output [ + {{ + name: "OUTPUT__0" + data_type: TYPE_FP32 + dims: [ 1000 ] + label_filename: "{label_filename}" + }} +] +instance_group [{{ kind: {"KIND_GPU" if torch.cuda.is_available() else "KIND_CPU"} }}] +""" + + try: + os.makedirs(config_dir) + except OSError: + pass # ignore existing dir + + with open(f"{config_dir}/config.pbtxt", "w") as file: + file.write(config) + print(f"Created {config_dir}/config.pbtxt") + + source_path = os.environ.get("TRITON_GENSRCDIR", default="gen_srcdir") + source_filename = os.path.join(source_path, RESNET50_LABEL_FILE) + + shutil.copyfile(source_filename, f"{config_dir}/{label_filename}") + print(f"Created {config_dir}/{label_filename}") def create_openvino_modelfile( @@ -1413,6 +1748,7 @@ def create_openvino_modelfile( output0_dtype, output1_dtype, ) + print(f"{_color_green}Creating model {model_name}{_color_reset}") model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) in0 = ov.opset1.parameter( @@ -1477,6 +1813,7 @@ def create_openvino_modelconfig( output0_dtype, output1_dtype, ) + print(f"{_color_green}Creating config for {model_name}{_color_reset}") config_dir = models_dir + "/" + model_name # platform is empty and backend is 'openvino' for openvino model @@ -1526,7 +1863,7 @@ def create_openvino_modelconfig( try: os.makedirs(config_dir) - except OSError as ex: + except OSError: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -1548,8 +1885,10 @@ def create_models( output0_label_cnt, version_policy=None, ): + print(f"{_color_blue}Creating models in {models_dir}{_color_reset}") model_version = 1 if FLAGS.tensorrt: + print(f"{_color_magenta}TensorRT model generation requested{_color_reset}") # max-batch 8 suffix = () if ( @@ -1640,6 +1979,7 @@ def create_models( ) if FLAGS.onnx: + print(f"{_color_magenta}ONNX model generation requested{_color_reset}") # max-batch 8 create_onnx_modelconfig( models_dir, @@ -1692,6 +2032,7 @@ def create_models( ) if FLAGS.libtorch: + print(f"{_color_magenta}PyTorch: PT model generation requested{_color_reset}") # max-batch 8 create_libtorch_modelconfig( models_dir, @@ -1743,7 +2084,30 @@ def create_models( output1_dtype, ) + if FLAGS.torch_aoti: + if output0_dtype == output1_dtype: + print(f"{_color_magenta}PyTorch: AOTI model generation requested{_color_reset}") + # max-batch 8 + if create_torch_aoti_modelfile( + models_dir, + model_version, + input_shape, + input_dtype, + output0_dtype, + ): + create_torch_aoti_modelconfig( + models_dir, + input_shape, + output0_shape, + input_dtype, + output0_dtype, + output0_label_cnt, + version_policy, + ) + + if FLAGS.openvino: + print(f"{_color_magenta}OpenVINO model generation requested{_color_reset}") # max-batch 8 create_openvino_modelconfig( models_dir, @@ -1796,6 +2160,7 @@ def create_models( ) if FLAGS.ensemble: + print(f"{_color_magenta}Ensemble model generation requested{_color_reset}") for pair in emu.platform_types_and_validation(): if not pair[1]( input_dtype, @@ -1933,6 +2298,18 @@ def create_fixed_models( action="store_true", help="Generate Pytorch LibTorch models", ) + parser.add_argument( + "--torch-aoti", + required=False, + action="store_true", + help="Generate Pytorch LibTorch models using PT2", + ) + parser.add_argument( + "--torchvision-aoti", + required=False, + action="store_true", + help="Generate Pytorch Torchvision models using PT2", + ) parser.add_argument( "--openvino", required=False, @@ -1959,9 +2336,14 @@ def create_fixed_models( import tensorrt as trt if FLAGS.onnx: import onnx - if FLAGS.libtorch: + if FLAGS.libtorch or FLAGS.torch_aoti: import torch from torch import nn + if FLAGS.torchvision_aoti: + import torch + import torchvision.models as models + import shutil + RESNET50_LABEL_FILE = "resnet50_labels.txt" if FLAGS.openvino: import openvino.runtime as ov @@ -2116,6 +2498,7 @@ def create_fixed_models( create_onnx_modelfile( FLAGS.models_dir, 0, 3, (16,), (16,), (16,), vt, vt, vt, swap=True ) + if FLAGS.libtorch: for vt in [np.float32, np.int32, np.int16, np.int8]: create_libtorch_modelfile( @@ -2130,6 +2513,7 @@ def create_fixed_models( create_libtorch_modelfile( FLAGS.models_dir, 0, 3, (16,), (16,), (16,), vt, vt, vt, swap=True ) + if FLAGS.openvino: for vt in [np.float16, np.float32, np.int8, np.int16, np.int32]: create_openvino_modelfile( @@ -2345,3 +2729,8 @@ def create_fixed_models( # to fixed size model is not safe but doable for model_shape in [(-1,), (-1, -1), (-1, -1, -1)]: emu.create_nop_modelconfig(FLAGS.models_dir, model_shape, model_dtype) + + if FLAGS.torchvision_aoti: + print(f"{_color_blue}TorchVision AOTI model generation requested{_color_reset}") + if create_torchvision_aoti_modelfile(FLAGS.models_dir, 1, 1): + create_torchvision_aoti_modelconfig(FLAGS.models_dir, 1) diff --git a/qa/common/resnet50_labels.txt b/qa/common/resnet50_labels.txt new file mode 100644 index 0000000000..f40829ed0f --- /dev/null +++ b/qa/common/resnet50_labels.txt @@ -0,0 +1,1000 @@ +tench +goldfish +great white shark +tiger shark +hammerhead +electric ray +stingray +cock +hen +ostrich +brambling +goldfinch +house finch +junco +indigo bunting +robin +bulbul +jay +magpie +chickadee +water ouzel +kite +bald eagle +vulture +great grey owl +European fire salamander +common newt +eft +spotted salamander +axolotl +bullfrog +tree frog +tailed frog +loggerhead +leatherback turtle +mud turtle +terrapin +box turtle +banded gecko +common iguana +American chameleon +whiptail +agama +frilled lizard +alligator lizard +Gila monster +green lizard +African chameleon +Komodo dragon +African crocodile +American alligator +triceratops +thunder snake +ringneck snake +hognose snake +green snake +king snake +garter snake +water snake +vine snake +night snake +boa constrictor +rock python +Indian cobra +green mamba +sea snake +horned viper +diamondback +sidewinder +trilobite +harvestman +scorpion +black and gold garden spider +barn spider +garden spider +black widow +tarantula +wolf spider +tick +centipede +black grouse +ptarmigan +ruffed grouse +prairie chicken +peacock +quail +partridge +African grey +macaw +sulphur-crested cockatoo +lorikeet +coucal +bee eater +hornbill +hummingbird +jacamar +toucan +drake +red-breasted merganser +goose +black swan +tusker +echidna +platypus +wallaby +koala +wombat +jellyfish +sea anemone +brain coral +flatworm +nematode +conch +snail +slug +sea slug +chiton +chambered nautilus +Dungeness crab +rock crab +fiddler crab +king crab +American lobster +spiny lobster +crayfish +hermit crab +isopod +white stork +black stork +spoonbill +flamingo +little blue heron +American egret +bittern +crane +limpkin +European gallinule +American coot +bustard +ruddy turnstone +red-backed sandpiper +redshank +dowitcher +oystercatcher +pelican +king penguin +albatross +grey whale +killer whale +dugong +sea lion +Chihuahua +Japanese spaniel +Maltese dog +Pekinese +Shih-Tzu +Blenheim spaniel +papillon +toy terrier +Rhodesian ridgeback +Afghan hound +basset +beagle +bloodhound +bluetick +black-and-tan coonhound +Walker hound +English foxhound +redbone +borzoi +Irish wolfhound +Italian greyhound +whippet +Ibizan hound +Norwegian elkhound +otterhound +Saluki +Scottish deerhound +Weimaraner +Staffordshire bullterrier +American Staffordshire terrier +Bedlington terrier +Border terrier +Kerry blue terrier +Irish terrier +Norfolk terrier +Norwich terrier +Yorkshire terrier +wire-haired fox terrier +Lakeland terrier +Sealyham terrier +Airedale +cairn +Australian terrier +Dandie Dinmont +Boston bull +miniature schnauzer +giant schnauzer +standard schnauzer +Scotch terrier +Tibetan terrier +silky terrier +soft-coated wheaten terrier +West Highland white terrier +Lhasa +flat-coated retriever +curly-coated retriever +golden retriever +Labrador retriever +Chesapeake Bay retriever +German short-haired pointer +vizsla +English setter +Irish setter +Gordon setter +Brittany spaniel +clumber +English springer +Welsh springer spaniel +cocker spaniel +Sussex spaniel +Irish water spaniel +kuvasz +schipperke +groenendael +malinois +briard +kelpie +komondor +Old English sheepdog +Shetland sheepdog +collie +Border collie +Bouvier des Flandres +Rottweiler +German shepherd +Doberman +miniature pinscher +Greater Swiss Mountain dog +Bernese mountain dog +Appenzeller +EntleBucher +boxer +bull mastiff +Tibetan mastiff +French bulldog +Great Dane +Saint Bernard +Eskimo dog +malamute +Siberian husky +dalmatian +affenpinscher +basenji +pug +Leonberg +Newfoundland +Great Pyrenees +Samoyed +Pomeranian +chow +keeshond +Brabancon griffon +Pembroke +Cardigan +toy poodle +miniature poodle +standard poodle +Mexican hairless +timber wolf +white wolf +red wolf +coyote +dingo +dhole +African hunting dog +hyena +red fox +kit fox +Arctic fox +grey fox +tabby +tiger cat +Persian cat +Siamese cat +Egyptian cat +cougar +lynx +leopard +snow leopard +jaguar +lion +tiger +cheetah +brown bear +American black bear +ice bear +sloth bear +mongoose +meerkat +tiger beetle +ladybug +ground beetle +long-horned beetle +leaf beetle +dung beetle +rhinoceros beetle +weevil +fly +bee +ant +grasshopper +cricket +walking stick +cockroach +mantis +cicada +leafhopper +lacewing +dragonfly +damselfly +admiral +ringlet +monarch +cabbage butterfly +sulphur butterfly +lycaenid +starfish +sea urchin +sea cucumber +wood rabbit +hare +Angora +hamster +porcupine +fox squirrel +marmot +beaver +guinea pig +sorrel +zebra +hog +wild boar +warthog +hippopotamus +ox +water buffalo +bison +ram +bighorn +ibex +hartebeest +impala +gazelle +Arabian camel +llama +weasel +mink +polecat +black-footed ferret +otter +skunk +badger +armadillo +three-toed sloth +orangutan +gorilla +chimpanzee +gibbon +siamang +guenon +patas +baboon +macaque +langur +colobus +proboscis monkey +marmoset +capuchin +howler monkey +titi +spider monkey +squirrel monkey +Madagascar cat +indri +Indian elephant +African elephant +lesser panda +giant panda +barracouta +eel +coho +rock beauty +anemone fish +sturgeon +gar +lionfish +puffer +abacus +abaya +academic gown +accordion +acoustic guitar +aircraft carrier +airliner +airship +altar +ambulance +amphibian +analog clock +apiary +apron +ashcan +assault rifle +backpack +bakery +balance beam +balloon +ballpoint +Band Aid +banjo +bannister +barbell +barber chair +barbershop +barn +barometer +barrel +barrow +baseball +basketball +bassinet +bassoon +bathing cap +bath towel +bathtub +beach wagon +beacon +beaker +bearskin +beer bottle +beer glass +bell cote +bib +bicycle-built-for-two +bikini +binder +binoculars +birdhouse +boathouse +bobsled +bolo tie +bonnet +bookcase +bookshop +bottlecap +bow +bow tie +brass +brassiere +breakwater +breastplate +broom +bucket +buckle +bulletproof vest +bullet train +butcher shop +cab +caldron +candle +cannon +canoe +can opener +cardigan +car mirror +carousel +carpenter's kit +carton +car wheel +cash machine +cassette +cassette player +castle +catamaran +CD player +cello +cellular telephone +chain +chainlink fence +chain mail +chain saw +chest +chiffonier +chime +china cabinet +Christmas stocking +church +cinema +cleaver +cliff dwelling +cloak +clog +cocktail shaker +coffee mug +coffeepot +coil +combination lock +computer keyboard +confectionery +container ship +convertible +corkscrew +cornet +cowboy boot +cowboy hat +cradle +crane +crash helmet +crate +crib +Crock Pot +croquet ball +crutch +cuirass +dam +desk +desktop computer +dial telephone +diaper +digital clock +digital watch +dining table +dishrag +dishwasher +disk brake +dock +dogsled +dome +doormat +drilling platform +drum +drumstick +dumbbell +Dutch oven +electric fan +electric guitar +electric locomotive +entertainment center +envelope +espresso maker +face powder +feather boa +file +fireboat +fire engine +fire screen +flagpole +flute +folding chair +football helmet +forklift +fountain +fountain pen +four-poster +freight car +French horn +frying pan +fur coat +garbage truck +gasmask +gas pump +goblet +go-kart +golf ball +golfcart +gondola +gong +gown +grand piano +greenhouse +grille +grocery store +guillotine +hair slide +hair spray +half track +hammer +hamper +hand blower +hand-held computer +handkerchief +hard disc +harmonica +harp +harvester +hatchet +holster +home theater +honeycomb +hook +hoopskirt +horizontal bar +horse cart +hourglass +iPod +iron +jack-o'-lantern +jean +jeep +jersey +jigsaw puzzle +jinrikisha +joystick +kimono +knee pad +knot +lab coat +ladle +lampshade +laptop +lawn mower +lens cap +letter opener +library +lifeboat +lighter +limousine +liner +lipstick +Loafer +lotion +loudspeaker +loupe +lumbermill +magnetic compass +mailbag +mailbox +maillot +maillot +manhole cover +maraca +marimba +mask +matchstick +maypole +maze +measuring cup +medicine chest +megalith +microphone +microwave +military uniform +milk can +minibus +miniskirt +minivan +missile +mitten +mixing bowl +mobile home +Model T +modem +monastery +monitor +moped +mortar +mortarboard +mosque +mosquito net +motor scooter +mountain bike +mountain tent +mouse +mousetrap +moving van +muzzle +nail +neck brace +necklace +nipple +notebook +obelisk +oboe +ocarina +odometer +oil filter +organ +oscilloscope +overskirt +oxcart +oxygen mask +packet +paddle +paddlewheel +padlock +paintbrush +pajama +palace +panpipe +paper towel +parachute +parallel bars +park bench +parking meter +passenger car +patio +pay-phone +pedestal +pencil box +pencil sharpener +perfume +Petri dish +photocopier +pick +pickelhaube +picket fence +pickup +pier +piggy bank +pill bottle +pillow +ping-pong ball +pinwheel +pirate +pitcher +plane +planetarium +plastic bag +plate rack +plow +plunger +Polaroid camera +pole +police van +poncho +pool table +pop bottle +pot +potter's wheel +power drill +prayer rug +printer +prison +projectile +projector +puck +punching bag +purse +quill +quilt +racer +racket +radiator +radio +radio telescope +rain barrel +recreational vehicle +reel +reflex camera +refrigerator +remote control +restaurant +revolver +rifle +rocking chair +rotisserie +rubber eraser +rugby ball +rule +running shoe +safe +safety pin +saltshaker +sandal +sarong +sax +scabbard +scale +school bus +schooner +scoreboard +screen +screw +screwdriver +seat belt +sewing machine +shield +shoe shop +shoji +shopping basket +shopping cart +shovel +shower cap +shower curtain +ski +ski mask +sleeping bag +slide rule +sliding door +slot +snorkel +snowmobile +snowplow +soap dispenser +soccer ball +sock +solar dish +sombrero +soup bowl +space bar +space heater +space shuttle +spatula +speedboat +spider web +spindle +sports car +spotlight +stage +steam locomotive +steel arch bridge +steel drum +stethoscope +stole +stone wall +stopwatch +stove +strainer +streetcar +stretcher +studio couch +stupa +submarine +suit +sundial +sunglass +sunglasses +sunscreen +suspension bridge +swab +sweatshirt +swimming trunks +swing +switch +syringe +table lamp +tank +tape player +teapot +teddy +television +tennis ball +thatch +theater curtain +thimble +thresher +throne +tile roof +toaster +tobacco shop +toilet seat +torch +totem pole +tow truck +toyshop +tractor +trailer truck +tray +trench coat +tricycle +trimaran +tripod +triumphal arch +trolleybus +trombone +tub +turnstile +typewriter keyboard +umbrella +unicycle +upright +vacuum +vase +vault +velvet +vending machine +vestment +viaduct +violin +volleyball +waffle iron +wall clock +wallet +wardrobe +warplane +washbasin +washer +water bottle +water jug +water tower +whiskey jug +whistle +wig +window screen +window shade +Windsor tie +wine bottle +wing +wok +wooden spoon +wool +worm fence +wreck +yawl +yurt +web site +comic book +crossword puzzle +street sign +traffic light +book jacket +menu +plate +guacamole +consomme +hot pot +trifle +ice cream +ice lolly +French loaf +bagel +pretzel +cheeseburger +hotdog +mashed potato +head cabbage +broccoli +cauliflower +zucchini +spaghetti squash +acorn squash +butternut squash +cucumber +artichoke +bell pepper +cardoon +mushroom +Granny Smith +strawberry +orange +lemon +fig +pineapple +banana +jackfruit +custard apple +pomegranate +hay +carbonara +chocolate sauce +dough +meat loaf +pizza +potpie +burrito +red wine +espresso +cup +eggnog +alp +bubble +cliff +coral reef +geyser +lakeside +promontory +sandbar +seashore +valley +volcano +ballplayer +groom +scuba diver +rapeseed +daisy +yellow lady's slipper +corn +acorn +hip +buckeye +coral fungus +agaric +gyromitra +stinkhorn +earthstar +hen-of-the-woods +bolete +ear +toilet tissue diff --git a/qa/common/test_util.py b/qa/common/test_util.py index 3eee61ce52..ceda056089 100755 --- a/qa/common/test_util.py +++ b/qa/common/test_util.py @@ -266,12 +266,15 @@ def get_dtype_name(dtype): def get_model_name(pf, input_dtype, output0_dtype, output1_dtype): - return "{}_{}_{}_{}".format( - pf, - get_dtype_name(input_dtype), - get_dtype_name(output0_dtype), - get_dtype_name(output1_dtype), - ) + if output1_dtype is None: + return f"{pf}_{get_dtype_name(input_dtype)}_{get_dtype_name(output0_dtype)}" + else: + return "{}_{}_{}_{}".format( + pf, + get_dtype_name(input_dtype), + get_dtype_name(output0_dtype), + get_dtype_name(output1_dtype), + ) def get_sequence_model_name(pf, dtype): From d716408961246be75ea28716e12f9d2a3ae81316 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Thu, 19 Feb 2026 12:14:04 -0500 Subject: [PATCH 2/4] fixup pre-commit complaints --- qa/common/gen_qa_models.py | 83 +++++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 24 deletions(-) diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py index 27c2db51ea..8823b77a41 100755 --- a/qa/common/gen_qa_models.py +++ b/qa/common/gen_qa_models.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,10 +27,10 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse -from cProfile import label import os -from builtins import range import sys +from builtins import range +from cProfile import label import gen_ensemble_model_utils as emu import numpy as np @@ -47,7 +47,6 @@ np_dtype_string = np.dtype(object) from typing import List, Tuple - _color_blue = "\033[94m" _color_green = "\033[32m" _color_magenta = "\033[35m" @@ -1078,6 +1077,7 @@ def create_libtorch_modelfile( and (output0_dtype != np_dtype_string) and (output1_dtype != np_dtype_string) ): + class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1105,6 +1105,7 @@ def forward(self, INPUT0: List[str], INPUT1: List[str]): and (output0_dtype == np_dtype_string) and (output1_dtype == np_dtype_string) ): + class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1140,6 +1141,7 @@ def forward( and (output0_dtype == np_dtype_string) and (output1_dtype != np_dtype_string) ): + class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1172,6 +1174,7 @@ def forward( and (output0_dtype != np_dtype_string) and (output1_dtype == np_dtype_string) ): + class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1204,6 +1207,7 @@ def forward( and (output0_dtype == np_dtype_string) and (output1_dtype == np_dtype_string) ): + class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1227,6 +1231,7 @@ def forward(self, INPUT0, INPUT1) -> Tuple[List[str], List[str]]: and (output0_dtype != np_dtype_string) and (output1_dtype == np_dtype_string) ): + class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1249,6 +1254,7 @@ def forward(self, INPUT0, INPUT1) -> Tuple[torch.Tensor, List[str]]: and (output0_dtype == np_dtype_string) and (output1_dtype != np_dtype_string) ): + class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1267,6 +1273,7 @@ def forward(self, INPUT0, INPUT1) -> Tuple[List[str], torch.Tensor]: return op0, op1 else: + class AddSubNet(nn.Module): def __init__(self, *args): self.output0_dtype = args[0][0] @@ -1308,14 +1315,34 @@ def generate_sample_inputs( input0 = torch.randint(-128, 127, input_shape, dtype=torch.int8, device=device) input1 = torch.randint(-128, 127, input_shape, dtype=torch.int8, device=device) elif input_dtype == np.int16: - input0 = torch.randint(-32768, 32767, input_shape, dtype=torch.int16, device=device) - input1 = torch.randint(-32768, 32767, input_shape, dtype=torch.int16, device=device) + input0 = torch.randint( + -32768, 32767, input_shape, dtype=torch.int16, device=device + ) + input1 = torch.randint( + -32768, 32767, input_shape, dtype=torch.int16, device=device + ) elif input_dtype == np.int32: - input0 = torch.randint(-2147483648, 2147483647, input_shape, dtype=torch.int32, device=device) - input1 = torch.randint(-2147483648, 2147483647, input_shape, dtype=torch.int32, device=device) + input0 = torch.randint( + -2147483648, 2147483647, input_shape, dtype=torch.int32, device=device + ) + input1 = torch.randint( + -2147483648, 2147483647, input_shape, dtype=torch.int32, device=device + ) elif input_dtype == np.int64: - input0 = torch.randint(-9223372036854775808, 9223372036854775807, input_shape, dtype=torch.int64, device=device) - input1 = torch.randint(-9223372036854775808, 9223372036854775807, input_shape, dtype=torch.int64, device=device) + input0 = torch.randint( + -9223372036854775808, + 9223372036854775807, + input_shape, + dtype=torch.int64, + device=device, + ) + input1 = torch.randint( + -9223372036854775808, + 9223372036854775807, + input_shape, + dtype=torch.int64, + device=device, + ) elif input_dtype == np.float16: input0 = torch.randn(*input_shape, dtype=torch.float16, device=device) input1 = torch.randn(*input_shape, dtype=torch.float16, device=device) @@ -1332,11 +1359,19 @@ def generate_sample_inputs( input0 = torch.randint(0, 65535, input_shape, dtype=torch.uint16, device=device) input1 = torch.randint(0, 65535, input_shape, dtype=torch.uint16, device=device) elif input_dtype == np.uint32: - input0 = torch.randint(0, 4294967295, input_shape, dtype=torch.uint32, device=device) - input1 = torch.randint(0, 4294967295, input_shape, dtype=torch.uint32, device=device) + input0 = torch.randint( + 0, 4294967295, input_shape, dtype=torch.uint32, device=device + ) + input1 = torch.randint( + 0, 4294967295, input_shape, dtype=torch.uint32, device=device + ) elif input_dtype == np.uint64: - input0 = torch.randint(0, 18446744073709551615, input_shape, dtype=torch.uint64, device=device) - input1 = torch.randint(0, 18446744073709551615, input_shape, dtype=torch.uint64, device=device) + input0 = torch.randint( + 0, 18446744073709551615, input_shape, dtype=torch.uint64, device=device + ) + input1 = torch.randint( + 0, 18446744073709551615, input_shape, dtype=torch.uint64, device=device + ) else: input0 = torch.randn(*input_shape, device=device) input1 = torch.randn(*input_shape, device=device) @@ -1373,6 +1408,7 @@ def np_to_dtype(np_dtype): ) return torch.int32 + def create_torch_aoti_modelfile( models_dir, model_version, @@ -1416,11 +1452,7 @@ def __init__( self.output_dtype = output_dtype super(AddSubNet, self).__init__() - def forward( - self, - INPUT0: torch.Tensor, - INPUT1: torch.Tensor - ) -> torch.Tensor: + def forward(self, INPUT0: torch.Tensor, INPUT1: torch.Tensor) -> torch.Tensor: if INPUT0.dtype != self.input_dtype: raise TypeError( f"INPUT0 expected {self.input_dtype} vs. actual {INPUT0.dtype} type." @@ -1484,8 +1516,8 @@ def create_torchvision_aoti_modelfile( ep = torch.export.export(model, (input_tensor,)) torch._inductor.aoti_compile_and_package( - ep, - package_path=f"{model_version_dir}/model.pt2", + ep, + package_path=f"{model_version_dir}/model.pt2", ) except Exception as e: print( @@ -2086,7 +2118,9 @@ def create_models( if FLAGS.torch_aoti: if output0_dtype == output1_dtype: - print(f"{_color_magenta}PyTorch: AOTI model generation requested{_color_reset}") + print( + f"{_color_magenta}PyTorch: AOTI model generation requested{_color_reset}" + ) # max-batch 8 if create_torch_aoti_modelfile( models_dir, @@ -2105,7 +2139,6 @@ def create_models( version_policy, ) - if FLAGS.openvino: print(f"{_color_magenta}OpenVINO model generation requested{_color_reset}") # max-batch 8 @@ -2340,9 +2373,11 @@ def create_fixed_models( import torch from torch import nn if FLAGS.torchvision_aoti: + import shutil + import torch import torchvision.models as models - import shutil + RESNET50_LABEL_FILE = "resnet50_labels.txt" if FLAGS.openvino: import openvino.runtime as ov From f9fdaea7ae999eec406bf8887230ddec4e1ce2c8 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Mon, 23 Feb 2026 12:15:39 -0500 Subject: [PATCH 3/4] Potential fix for code scanning alert no. 1026: Unused import Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- qa/common/gen_qa_models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py index 8823b77a41..a6d196310d 100755 --- a/qa/common/gen_qa_models.py +++ b/qa/common/gen_qa_models.py @@ -30,7 +30,6 @@ import os import sys from builtins import range -from cProfile import label import gen_ensemble_model_utils as emu import numpy as np From 283d1e0bf11f2f0f24eed036e1e07fc06b9f9ae7 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Wed, 11 Mar 2026 17:02:06 -0400 Subject: [PATCH 4/4] remove batch_size from aoti model generation --- qa/common/gen_qa_models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py index a6d196310d..3ec33dbc60 100755 --- a/qa/common/gen_qa_models.py +++ b/qa/common/gen_qa_models.py @@ -1660,7 +1660,6 @@ def create_torch_aoti_modelconfig( backend: "pytorch" name: "{model_name}" platform: "torch_aoti" -max_batch_size: 8 version_policy: {version_policy_str} input [ {{