From c2a214f6cfed14ca00ad0e9330266aa301264bdb Mon Sep 17 00:00:00 2001 From: Alexander Dokuchaev Date: Wed, 1 Jul 2026 12:06:12 +0300 Subject: [PATCH 1/4] Updated ref --- tests/cross_fw/examples/example_scope.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cross_fw/examples/example_scope.json b/tests/cross_fw/examples/example_scope.json index 11857c49c16..dbc9501189c 100644 --- a/tests/cross_fw/examples/example_scope.json +++ b/tests/cross_fw/examples/example_scope.json @@ -200,7 +200,7 @@ "requirements": "examples/llm_compression/openvino/tiny_llama/requirements.txt", "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", "accuracy_metrics": { - "word_count": 86 + "word_count": 74 } }, "llm_compression_fx": { @@ -208,7 +208,7 @@ "requirements": "examples/llm_compression/torch_fx/tiny_llama/requirements.txt", "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", "accuracy_metrics": { - "word_count": 56 + "word_count": 69 } }, "llm_compression_onnx": { From 8ab8bc44a55fee255a07a94888640e4b7768641e Mon Sep 17 00:00:00 2001 From: Alexander Dokuchaev Date: Wed, 1 Jul 2026 17:09:20 +0300 Subject: [PATCH 2/4] test --- .github/workflows/examples.yml | 194 +----------- tests/cross_fw/examples/example_scope.json | 350 --------------------- 2 files changed, 2 insertions(+), 542 deletions(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index ee7a6ae5143..fc4006dc2f3 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -1,49 +1,14 @@ name: Test examples -run-name: "Test examples${{ inputs.pr_num != '' && format(' PR#{0}', inputs.pr_num) || '' }}${{ inputs.pytest_args != '' && format(' [{0}]', inputs.pytest_args) || '' }}" permissions: read-all on: - workflow_call: - inputs: - pr_num: - description: 'The pull request number' - type: string - default: '' - pytest_args: - description: 'Pytest arguments' - type: string - default: '' - skip_windows: - description: 'Skip tests on Windows' - type: boolean - default: false - skip_gpu: - description: 'Skip tests on GPU' - type: boolean - default: false - workflow_dispatch: - inputs: - pr_num: - description: 'The pull request number' - default: '' - pytest_args: - description: 'Pytest arguments' - default: '' - skip_windows: - description: 'Skip tests on Windows' - type: boolean - default: false - skip_gpu: - description: 'Skip tests on GPU' - type: boolean - default: false + pull_request: defaults: run: shell: bash env: - PYTEST_ARGS: ${{ inputs.pytest_args || '' }} TQDM_DISABLE: 1 RESULT_XML: "pytest-results.xml" PYTHON_VERSION: "3.12" @@ -59,67 +24,6 @@ jobs: group: [1, 2, 3, 4, 5, 6] steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ inputs.pr_num != '' && format('refs/pull/{0}/head', inputs.pr_num) || github.ref }} - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: cpuinfo - run: cat /proc/cpuinfo - - name: Install test requirements - run: | - pip install -r tests/cross_fw/examples/requirements.txt - - name: Print installed modules - run: pip list - - name: Run examples test scope - run: | - set +e - python -m pytest -s -ra tests/cross_fw/examples \ - -m 'not cuda' \ - --junit-xml=${{ env.RESULT_XML }} \ - --durations-path=tests/cross_fw/examples/test_durations.json \ - --splitting-algorithm=least_duration \ - --splits 6 \ - --group ${{ matrix.group }} \ - ${{ env.PYTEST_ARGS }} - ret=$? - [ $ret -eq 5 ] && [ -n "${{ inputs.pytest_args || '' }}" ] && exit 0 || exit $ret - - name: Upload test results - if: ${{ !cancelled() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: examples-cpu-pytest-results-${{ matrix.group }} - path: ${{ env.RESULT_XML }} - retention-days: 7 - - examples-cuda: - name: Test examples CUDA [${{ matrix.group }}/1] - runs-on: aks-linux-6-cores-55gb-gpu-a10 - timeout-minutes: 40 - if: ${{ !inputs.skip_gpu }} - strategy: - fail-fast: false - matrix: - group: [1] - env: - DEBIAN_FRONTEND: noninteractive - steps: - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get --assume-yes install build-essential ninja-build libgl1-mesa-dev libglib2.0-0 wget make - - name: Download CUDA - run: | - wget -q https://developer.download.nvidia.com/compute/cuda/12.6.3/local_installers/cuda_12.6.3_560.35.05_linux.run - sudo sh cuda_12.6.3_560.35.05_linux.run --toolkit --silent - - name: Runner info - continue-on-error: true - run: | - nvidia-smi - cat /proc/cpuinfo - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ inputs.pr_num != '' && format('refs/pull/{0}/head', inputs.pr_num) || github.ref }} - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ env.PYTHON_VERSION }} @@ -133,98 +37,4 @@ jobs: - name: Run examples test scope run: | set +e - python -m pytest -s -ra tests/cross_fw/examples \ - -m cuda \ - --junit-xml=${{ env.RESULT_XML }} \ - --durations-path=tests/cross_fw/examples/test_durations.json \ - --splitting-algorithm=least_duration \ - --splits 1 \ - --group ${{ matrix.group }} \ - ${{ env.PYTEST_ARGS }} - ret=$? - [ $ret -eq 5 ] && [ -n "${{ inputs.pytest_args || '' }}" ] && exit 0 || exit $ret - - name: Upload test results - if: ${{ !cancelled() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: examples-cuda-pytest-results-${{ matrix.group }} - path: ${{ env.RESULT_XML }} - retention-days: 7 - - examples-win-cpu: - name: Test examples CPU Windows [${{ matrix.group }}/6] - runs-on: windows-2025-16-core - timeout-minutes: 80 - if: ${{ !inputs.skip_windows }} - strategy: - fail-fast: false - matrix: - group: [1, 2, 3, 4, 5, 6] - env: - PYTHONIOENCODING: utf-8 - NNCF_EXTENSION_LOAD_TIMEOUT: 180 - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ inputs.pr_num != '' && format('refs/pull/{0}/head', inputs.pr_num) || github.ref }} - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - - name: Install NNCF and test requirements - run: | - pip install -r tests/cross_fw/examples/requirements.txt - - name: Print installed modules - run: pip list - - name: Run examples test scope - run: | - set +e - - sys_lib="$(python -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))")" - prefix_lib="$(python -c "import sys; print(sys.prefix + '/libs')")" - include_dir="$(python -c "import sysconfig; print(sysconfig.get_path('include'))")" - - export LIB="${LIB};${sys_lib};${prefix_lib}" - export INCLUDE="${INCLUDE};${include_dir}" - - python -m pytest -s -ra tests/cross_fw/examples \ - -m 'not cuda' \ - --junit-xml=${{ env.RESULT_XML }} \ - --durations-path=tests/cross_fw/examples/test_durations.json \ - --splitting-algorithm=least_duration \ - --splits 6 \ - --group ${{ matrix.group }} \ - ${{ env.PYTEST_ARGS }} - ret=$? - [ $ret -eq 5 ] && [ -n "${{ inputs.pytest_args || '' }}" ] && exit 0 || exit $ret - - name: Upload test results - if: ${{ !cancelled() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: examples-win-cpu-pytest-results-${{ matrix.group }} - path: ${{ env.RESULT_XML }} - retention-days: 7 - - examples-cpu-summary: - name: Summary Examples CPU - if: ${{ !cancelled() }} - needs: examples-cpu - uses: ./.github/workflows/call_summary.yml - with: - artifact_pattern: examples-cpu-pytest-results-* - - examples-cuda-summary: - name: Summary Examples CUDA - if: ${{ !cancelled() && !inputs.skip_gpu }} - needs: examples-cuda - uses: ./.github/workflows/call_summary.yml - with: - artifact_pattern: examples-cuda-pytest-results-* - - examples-win-cpu-summary: - name: Summary Examples Windows CPU - if: ${{ !cancelled() && !inputs.skip_windows }} - needs: examples-win-cpu - uses: ./.github/workflows/call_summary.yml - with: - artifact_pattern: examples-win-cpu-pytest-results-* + python -m pytest -s -ra tests/cross_fw/examples diff --git a/tests/cross_fw/examples/example_scope.json b/tests/cross_fw/examples/example_scope.json index dbc9501189c..be9041362ce 100644 --- a/tests/cross_fw/examples/example_scope.json +++ b/tests/cross_fw/examples/example_scope.json @@ -1,200 +1,4 @@ { - "post_training_quantization_onnx_mobilenet_v2": { - "backend": "onnx", - "requirements": "examples/post_training_quantization/onnx/mobilenet_v2/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_metrics": { - "fp32_top1": 0.9864968152866243, - "int8_top1": 0.9844585987261146, - "accuracy_drop": 0.0020382165605096203 - }, - "performance_metrics": { - "fp32_fps": 1732.8, - "int8_fps": 6013.68, - "performance_speed_up": 3.470498614958449 - }, - "model_size_metrics": { - "fp32_model_size": 9.01850414276123, - "int8_model_size": 2.9374523162841797, - "model_compression_rate": 3.0701789073360906 - } - }, - "post_training_quantization_openvino_mobilenet_v2_quantize": { - "backend": "openvino", - "requirements": "examples/post_training_quantization/openvino/mobilenet_v2/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_metrics": { - "fp32_top1": 0.9864968152866243, - "int8_top1": 0.9844585987261146, - "accuracy_drop": 0.0020382165605096203 - }, - "performance_metrics": { - "fp32_fps": 1725.97, - "int8_fps": 6003.05, - "performance_speed_up": 3.478073199418298 - }, - "model_size_metrics": { - "fp32_model_size": 8.605232238769531, - "int8_model_size": 2.6038694381713867, - "model_compression_rate": 3.30478637393306 - } - }, - "post_training_quantization_openvino_yolo26_quantize": { - "backend": "openvino", - "requirements": "examples/post_training_quantization/openvino/yolo26/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_tolerance": 0.3, - "accuracy_metrics": { - "fp32_mAP": 0.4796109388386565, - "int8_mAP": 0.47337998186573793, - "accuracy_drop": 0.006230956972918589 - }, - "performance_metrics": { - "fp32_fps": 201.03, - "int8_fps": 423.12, - "performance_speed_up": 2.20937173556 - } - }, - "post_training_quantization_openvino_anomaly_stfpm_quantize_with_accuracy_control": { - "backend": "openvino", - "requirements": "examples/post_training_quantization/openvino/anomaly_stfpm_quantize_with_accuracy_control/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_metrics": { - "fp32_top1": 0.9680365324020386, - "int8_top1": 0.9636363983154297, - "accuracy_drop": 0.004400134086608887 - }, - "performance_metrics": { - "fp32_fps": 330.17, - "int8_fps": 838.9, - "performance_speed_up": 2.5408123088106125 - }, - "model_size_metrics": { - "fp32_model_size": 21.33917808532715, - "int8_model_size": 5.742419242858887, - "model_compression_rate": 3.7160606327836403 - } - }, - "post_training_quantization_openvino_yolo8_quantize_with_accuracy_control": { - "backend": "openvino", - "requirements": "examples/post_training_quantization/openvino/yolov8_quantize_with_accuracy_control/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_metrics": { - "fp32_mAP": 0.44964819083938, - "int8_mAP": 0.4507030984145927, - "accuracy_drop": -0.0009941002422954037 - }, - "performance_metrics": { - "fp32_fps": 134.48, - "int8_fps": 274.58, - "performance_speed_up": 2.0417906008328375 - } - }, - "post_training_quantization_onnx_yolo8_quantize_with_accuracy_control": { - "backend": "onnx", - "requirements": "examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_tolerance": 0.3, - "accuracy_metrics": { - "onnx_fp32_box_mAP": 0.4496481927421265, - "onnx_fp32_mask_mAP": 0.36509979887281135, - "onnx_int8_box_mAP": 0.4509292170587964, - "onnx_int8_mask_mAP": 0.36710935370534864, - "onnx_drop_box_mAP": -0.001281024316669932, - "onnx_drop_mask_mAP": -0.002009554832537286, - "ov_fp32_box_mAP": 0.4496481927421265, - "ov_fp32_mask_mAP": 0.36509979887281135, - "ov_int8_box_mAP": 0.4494792029705020, - "ov_int8_mask_mAP": 0.3666179206494213, - "ov_drop_box_mAP": 0.000168989771624439, - "ov_drop_mask_mAP": -0.0015181217766099264 - }, - "performance_metrics": { - "ov_fp32_fps": 139.32, - "ov_int8_fps": 269.16, - "performance_speed_up": 1.9319552110249787 - } - }, - "post_training_quantization_torch_mobilenet_v2": { - "backend": "torch", - "requirements": "examples/post_training_quantization/torch/mobilenet_v2/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_metrics": { - "fp32_top1": 0.9864968152866243, - "int8_top1": 0.9849681528662421, - "accuracy_drop": 0.0015286624203821875 - }, - "performance_metrics": { - "fp32_fps": 1624.67, - "int8_fps": 5981.95, - "performance_speed_up": 3.681947718613626 - }, - "model_size_metrics": { - "fp32_model_size": 8.610844612121582, - "int8_model_size": 2.637709617614746, - "model_compression_rate": 3.264515758147889 - } - }, - "post_training_quantization_torch_ssd300_vgg16": { - "backend": "torch", - "requirements": "examples/post_training_quantization/torch/ssd300_vgg16/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_metrics": { - "fp32_mAP": 0.5228869318962097, - "int8_mAP": 0.5145288109779358, - "accuracy_drop": 0.0080191493 - }, - "performance_metrics": { - "fp32_fps": 36.3, - "int8_fps": 145.76, - "performance_speed_up": 4.01487603306 - }, - "model_size_metrics": { - "fp32_model_size": 137.17038249969482, - "int8_model_size": 35.38157653808594, - "model_compression_rate": 3.8768872368375087 - } - }, - "post_training_quantization_torch_fx_resnet18": { - "backend": "torch", - "requirements": "examples/post_training_quantization/torch_fx/resnet18/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_tolerance": 0.2, - "accuracy_metrics": { - "fp32_top1": 55.52, - "int8_top1": 55.27 - }, - "performance_metrics": { - "fp32_latency": 3.3447, - "fp32_ov_latency": 1.401, - "int8_latency": 0.6003 - } - }, - "quantization_aware_training_torch_resnet18": { - "backend": "torch", - "requirements": "examples/quantization_aware_training/torch/resnet18/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_tolerance_after_training": 1.0, - "accuracy_tolerance": 0.2, - "accuracy_metrics": { - "fp32_top1": 55.52000045776367, - "int8_init_top1": 55.279998779296875 - }, - "accuracy_metrics_after_training": { - "int8_top1": 57.08069620253165, - "accuracy_drop": -1.5606957447679761 - }, - "performance_metrics": { - "fp32_fps": 3646.13, - "int8_fps": 15301.98, - "performance_speed_up": 4.196773016869941 - }, - "model_size_metrics": { - "fp32_model_size": 43.06997013092041, - "int8_model_size": 11.01035213470459, - "model_compression_rate": 3.9117704505711512 - } - }, "llm_compression": { "backend": "openvino", "requirements": "examples/llm_compression/openvino/tiny_llama/requirements.txt", @@ -210,159 +14,5 @@ "accuracy_metrics": { "word_count": 69 } - }, - "llm_compression_onnx": { - "backend": "onnx", - "requirements": "examples/llm_compression/onnx/tiny_llama/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_metrics": { - "word_count": 75 - } - }, - "llm_compression_scale_estimation_onnx": { - "backend": "onnx", - "requirements": "examples/llm_compression/onnx/tiny_llama_scale_estimation/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_metrics": { - "word_count": 81 - } - }, - "llm_tune_params": { - "backend": "openvino", - "requirements": "examples/llm_compression/openvino/tiny_llama_find_hyperparams/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_metrics": { - "awq": true, - "ratio": 1.0, - "group_size": 128 - } - }, - "llm_compression_synthetic": { - "xfail": "Differ output on windows: ticket-187175", - "backend": "openvino", - "requirements": "examples/llm_compression/openvino/tiny_llama_synthetic_data/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_metrics": { - "word_count": 84 - } - }, - "quantization_aware_training_torch_anomalib": { - "backend": "torch", - "requirements": "examples/quantization_aware_training/torch/anomalib/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_tolerance_after_training": 0.02, - "accuracy_tolerance": 0.08, - "accuracy_metrics": { - "fp32_f1score": 0.9045643210411072, - "int8_init_f1score": 0.9045643210411072 - }, - "accuracy_metrics_after_training": { - "int8_f1score": 0.9152542352676392 - }, - "performance_metrics": { - "fp32_fps": 316.28, - "int8_fps": 879.84, - "performance_speed_up": 2.7818388769444797 - }, - "model_size_metrics": { - "fp32_model_size": 21.37990665435791, - "int8_model_size": 5.677968978881836, - "model_compression_rate": 3.7654144877995197 - } - }, - "fp8_llm_quantization": { - "backend": "openvino", - "requirements": "examples/llm_compression/openvino/smollm2_360m_fp8/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_metrics": { - "answers": [ - "Paris.", - "Mont Blanc.", - "Toronto.", - "Tokyo." - ] - } - }, - "fp8_llm_quantization_onnx": { - "backend": "onnx", - "requirements": "examples/llm_compression/onnx/smollm2_360m_fp8/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_metrics": { - "answers": [ - "Paris.", - "Mont Blanc.", - "Toronto.", - "Tokyo." - ] - } - }, - "codebook_llm_compression": { - "backend": "openvino", - "requirements": "examples/llm_compression/openvino/smollm2_360m_codebook/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_metrics": { - "answers": [ - "Paris.", - "Mont Blanc.", - "Toronto.", - "Tokyo.", - "Paris.", - "Mont Blanc.", - "Montauin.", - "Fukuoka." - ] - } - }, - "adaptive_codebook_llm_compression": { - "backend": "openvino", - "requirements": "examples/llm_compression/openvino/smollm2_360m_adaptive_codebook/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_metrics": { - "answers": [ - "Paris.", - "Mount Blanc.", - "Toronto.", - "Fukuoka." - ] - } - }, - "llm_compression_distillation_qat_with_lora": { - "backend": "torch", - "device": "cuda", - "CUDA_VISIBLE_DEVICES": "0", - "requirements": "examples/llm_compression/torch/distillation_qat_with_lora/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_tolerance": 0.1, - "accuracy_metrics": { - "best_ov_perplexity": 32.37 - } - }, - "llm_compression_qat_with_nls": { - "backend": "torch", - "device": "cuda", - "requirements": "examples/llm_compression/torch/downstream_qat_with_nls/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_tolerance": 0.02, - "accuracy_metrics": { - "accuracy_diff": 0.03 - } - }, - "magnitude_pruning_torch_resnet18": { - "backend": "torch", - "requirements": "examples/pruning/torch/resnet18/requirements.txt", - "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", - "accuracy_tolerance": 0.2, - "accuracy_metrics": { - "acc1": 54.96 - } - }, - "llm_compression_torch_gptqmodel_convertor": { - "backend": "torch", - "device": "cuda", - "requirements": "examples/llm_compression/torch/gptq_model_convertor/requirements.txt", - "extra_requirements": "examples/llm_compression/torch/gptq_model_convertor/requirements_extra.txt", - "accuracy_metrics": { - "word_count": 28 - } } } From 85453b5f052187546ac65f742cb610d520a14e47 Mon Sep 17 00:00:00 2001 From: Alexander Dokuchaev Date: Wed, 1 Jul 2026 18:07:06 +0300 Subject: [PATCH 3/4] Revert "test" This reverts commit 8ab8bc44a55fee255a07a94888640e4b7768641e. --- .github/workflows/examples.yml | 194 +++++++++++- tests/cross_fw/examples/example_scope.json | 350 +++++++++++++++++++++ 2 files changed, 542 insertions(+), 2 deletions(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index fc4006dc2f3..ee7a6ae5143 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -1,14 +1,49 @@ name: Test examples +run-name: "Test examples${{ inputs.pr_num != '' && format(' PR#{0}', inputs.pr_num) || '' }}${{ inputs.pytest_args != '' && format(' [{0}]', inputs.pytest_args) || '' }}" permissions: read-all on: - pull_request: + workflow_call: + inputs: + pr_num: + description: 'The pull request number' + type: string + default: '' + pytest_args: + description: 'Pytest arguments' + type: string + default: '' + skip_windows: + description: 'Skip tests on Windows' + type: boolean + default: false + skip_gpu: + description: 'Skip tests on GPU' + type: boolean + default: false + workflow_dispatch: + inputs: + pr_num: + description: 'The pull request number' + default: '' + pytest_args: + description: 'Pytest arguments' + default: '' + skip_windows: + description: 'Skip tests on Windows' + type: boolean + default: false + skip_gpu: + description: 'Skip tests on GPU' + type: boolean + default: false defaults: run: shell: bash env: + PYTEST_ARGS: ${{ inputs.pytest_args || '' }} TQDM_DISABLE: 1 RESULT_XML: "pytest-results.xml" PYTHON_VERSION: "3.12" @@ -24,6 +59,67 @@ jobs: group: [1, 2, 3, 4, 5, 6] steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.pr_num != '' && format('refs/pull/{0}/head', inputs.pr_num) || github.ref }} + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: cpuinfo + run: cat /proc/cpuinfo + - name: Install test requirements + run: | + pip install -r tests/cross_fw/examples/requirements.txt + - name: Print installed modules + run: pip list + - name: Run examples test scope + run: | + set +e + python -m pytest -s -ra tests/cross_fw/examples \ + -m 'not cuda' \ + --junit-xml=${{ env.RESULT_XML }} \ + --durations-path=tests/cross_fw/examples/test_durations.json \ + --splitting-algorithm=least_duration \ + --splits 6 \ + --group ${{ matrix.group }} \ + ${{ env.PYTEST_ARGS }} + ret=$? + [ $ret -eq 5 ] && [ -n "${{ inputs.pytest_args || '' }}" ] && exit 0 || exit $ret + - name: Upload test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: examples-cpu-pytest-results-${{ matrix.group }} + path: ${{ env.RESULT_XML }} + retention-days: 7 + + examples-cuda: + name: Test examples CUDA [${{ matrix.group }}/1] + runs-on: aks-linux-6-cores-55gb-gpu-a10 + timeout-minutes: 40 + if: ${{ !inputs.skip_gpu }} + strategy: + fail-fast: false + matrix: + group: [1] + env: + DEBIAN_FRONTEND: noninteractive + steps: + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get --assume-yes install build-essential ninja-build libgl1-mesa-dev libglib2.0-0 wget make + - name: Download CUDA + run: | + wget -q https://developer.download.nvidia.com/compute/cuda/12.6.3/local_installers/cuda_12.6.3_560.35.05_linux.run + sudo sh cuda_12.6.3_560.35.05_linux.run --toolkit --silent + - name: Runner info + continue-on-error: true + run: | + nvidia-smi + cat /proc/cpuinfo + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.pr_num != '' && format('refs/pull/{0}/head', inputs.pr_num) || github.ref }} - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ env.PYTHON_VERSION }} @@ -37,4 +133,98 @@ jobs: - name: Run examples test scope run: | set +e - python -m pytest -s -ra tests/cross_fw/examples + python -m pytest -s -ra tests/cross_fw/examples \ + -m cuda \ + --junit-xml=${{ env.RESULT_XML }} \ + --durations-path=tests/cross_fw/examples/test_durations.json \ + --splitting-algorithm=least_duration \ + --splits 1 \ + --group ${{ matrix.group }} \ + ${{ env.PYTEST_ARGS }} + ret=$? + [ $ret -eq 5 ] && [ -n "${{ inputs.pytest_args || '' }}" ] && exit 0 || exit $ret + - name: Upload test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: examples-cuda-pytest-results-${{ matrix.group }} + path: ${{ env.RESULT_XML }} + retention-days: 7 + + examples-win-cpu: + name: Test examples CPU Windows [${{ matrix.group }}/6] + runs-on: windows-2025-16-core + timeout-minutes: 80 + if: ${{ !inputs.skip_windows }} + strategy: + fail-fast: false + matrix: + group: [1, 2, 3, 4, 5, 6] + env: + PYTHONIOENCODING: utf-8 + NNCF_EXTENSION_LOAD_TIMEOUT: 180 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.pr_num != '' && format('refs/pull/{0}/head', inputs.pr_num) || github.ref }} + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + - name: Install NNCF and test requirements + run: | + pip install -r tests/cross_fw/examples/requirements.txt + - name: Print installed modules + run: pip list + - name: Run examples test scope + run: | + set +e + + sys_lib="$(python -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))")" + prefix_lib="$(python -c "import sys; print(sys.prefix + '/libs')")" + include_dir="$(python -c "import sysconfig; print(sysconfig.get_path('include'))")" + + export LIB="${LIB};${sys_lib};${prefix_lib}" + export INCLUDE="${INCLUDE};${include_dir}" + + python -m pytest -s -ra tests/cross_fw/examples \ + -m 'not cuda' \ + --junit-xml=${{ env.RESULT_XML }} \ + --durations-path=tests/cross_fw/examples/test_durations.json \ + --splitting-algorithm=least_duration \ + --splits 6 \ + --group ${{ matrix.group }} \ + ${{ env.PYTEST_ARGS }} + ret=$? + [ $ret -eq 5 ] && [ -n "${{ inputs.pytest_args || '' }}" ] && exit 0 || exit $ret + - name: Upload test results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: examples-win-cpu-pytest-results-${{ matrix.group }} + path: ${{ env.RESULT_XML }} + retention-days: 7 + + examples-cpu-summary: + name: Summary Examples CPU + if: ${{ !cancelled() }} + needs: examples-cpu + uses: ./.github/workflows/call_summary.yml + with: + artifact_pattern: examples-cpu-pytest-results-* + + examples-cuda-summary: + name: Summary Examples CUDA + if: ${{ !cancelled() && !inputs.skip_gpu }} + needs: examples-cuda + uses: ./.github/workflows/call_summary.yml + with: + artifact_pattern: examples-cuda-pytest-results-* + + examples-win-cpu-summary: + name: Summary Examples Windows CPU + if: ${{ !cancelled() && !inputs.skip_windows }} + needs: examples-win-cpu + uses: ./.github/workflows/call_summary.yml + with: + artifact_pattern: examples-win-cpu-pytest-results-* diff --git a/tests/cross_fw/examples/example_scope.json b/tests/cross_fw/examples/example_scope.json index be9041362ce..dbc9501189c 100644 --- a/tests/cross_fw/examples/example_scope.json +++ b/tests/cross_fw/examples/example_scope.json @@ -1,4 +1,200 @@ { + "post_training_quantization_onnx_mobilenet_v2": { + "backend": "onnx", + "requirements": "examples/post_training_quantization/onnx/mobilenet_v2/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_metrics": { + "fp32_top1": 0.9864968152866243, + "int8_top1": 0.9844585987261146, + "accuracy_drop": 0.0020382165605096203 + }, + "performance_metrics": { + "fp32_fps": 1732.8, + "int8_fps": 6013.68, + "performance_speed_up": 3.470498614958449 + }, + "model_size_metrics": { + "fp32_model_size": 9.01850414276123, + "int8_model_size": 2.9374523162841797, + "model_compression_rate": 3.0701789073360906 + } + }, + "post_training_quantization_openvino_mobilenet_v2_quantize": { + "backend": "openvino", + "requirements": "examples/post_training_quantization/openvino/mobilenet_v2/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_metrics": { + "fp32_top1": 0.9864968152866243, + "int8_top1": 0.9844585987261146, + "accuracy_drop": 0.0020382165605096203 + }, + "performance_metrics": { + "fp32_fps": 1725.97, + "int8_fps": 6003.05, + "performance_speed_up": 3.478073199418298 + }, + "model_size_metrics": { + "fp32_model_size": 8.605232238769531, + "int8_model_size": 2.6038694381713867, + "model_compression_rate": 3.30478637393306 + } + }, + "post_training_quantization_openvino_yolo26_quantize": { + "backend": "openvino", + "requirements": "examples/post_training_quantization/openvino/yolo26/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_tolerance": 0.3, + "accuracy_metrics": { + "fp32_mAP": 0.4796109388386565, + "int8_mAP": 0.47337998186573793, + "accuracy_drop": 0.006230956972918589 + }, + "performance_metrics": { + "fp32_fps": 201.03, + "int8_fps": 423.12, + "performance_speed_up": 2.20937173556 + } + }, + "post_training_quantization_openvino_anomaly_stfpm_quantize_with_accuracy_control": { + "backend": "openvino", + "requirements": "examples/post_training_quantization/openvino/anomaly_stfpm_quantize_with_accuracy_control/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_metrics": { + "fp32_top1": 0.9680365324020386, + "int8_top1": 0.9636363983154297, + "accuracy_drop": 0.004400134086608887 + }, + "performance_metrics": { + "fp32_fps": 330.17, + "int8_fps": 838.9, + "performance_speed_up": 2.5408123088106125 + }, + "model_size_metrics": { + "fp32_model_size": 21.33917808532715, + "int8_model_size": 5.742419242858887, + "model_compression_rate": 3.7160606327836403 + } + }, + "post_training_quantization_openvino_yolo8_quantize_with_accuracy_control": { + "backend": "openvino", + "requirements": "examples/post_training_quantization/openvino/yolov8_quantize_with_accuracy_control/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_metrics": { + "fp32_mAP": 0.44964819083938, + "int8_mAP": 0.4507030984145927, + "accuracy_drop": -0.0009941002422954037 + }, + "performance_metrics": { + "fp32_fps": 134.48, + "int8_fps": 274.58, + "performance_speed_up": 2.0417906008328375 + } + }, + "post_training_quantization_onnx_yolo8_quantize_with_accuracy_control": { + "backend": "onnx", + "requirements": "examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_tolerance": 0.3, + "accuracy_metrics": { + "onnx_fp32_box_mAP": 0.4496481927421265, + "onnx_fp32_mask_mAP": 0.36509979887281135, + "onnx_int8_box_mAP": 0.4509292170587964, + "onnx_int8_mask_mAP": 0.36710935370534864, + "onnx_drop_box_mAP": -0.001281024316669932, + "onnx_drop_mask_mAP": -0.002009554832537286, + "ov_fp32_box_mAP": 0.4496481927421265, + "ov_fp32_mask_mAP": 0.36509979887281135, + "ov_int8_box_mAP": 0.4494792029705020, + "ov_int8_mask_mAP": 0.3666179206494213, + "ov_drop_box_mAP": 0.000168989771624439, + "ov_drop_mask_mAP": -0.0015181217766099264 + }, + "performance_metrics": { + "ov_fp32_fps": 139.32, + "ov_int8_fps": 269.16, + "performance_speed_up": 1.9319552110249787 + } + }, + "post_training_quantization_torch_mobilenet_v2": { + "backend": "torch", + "requirements": "examples/post_training_quantization/torch/mobilenet_v2/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_metrics": { + "fp32_top1": 0.9864968152866243, + "int8_top1": 0.9849681528662421, + "accuracy_drop": 0.0015286624203821875 + }, + "performance_metrics": { + "fp32_fps": 1624.67, + "int8_fps": 5981.95, + "performance_speed_up": 3.681947718613626 + }, + "model_size_metrics": { + "fp32_model_size": 8.610844612121582, + "int8_model_size": 2.637709617614746, + "model_compression_rate": 3.264515758147889 + } + }, + "post_training_quantization_torch_ssd300_vgg16": { + "backend": "torch", + "requirements": "examples/post_training_quantization/torch/ssd300_vgg16/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_metrics": { + "fp32_mAP": 0.5228869318962097, + "int8_mAP": 0.5145288109779358, + "accuracy_drop": 0.0080191493 + }, + "performance_metrics": { + "fp32_fps": 36.3, + "int8_fps": 145.76, + "performance_speed_up": 4.01487603306 + }, + "model_size_metrics": { + "fp32_model_size": 137.17038249969482, + "int8_model_size": 35.38157653808594, + "model_compression_rate": 3.8768872368375087 + } + }, + "post_training_quantization_torch_fx_resnet18": { + "backend": "torch", + "requirements": "examples/post_training_quantization/torch_fx/resnet18/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_tolerance": 0.2, + "accuracy_metrics": { + "fp32_top1": 55.52, + "int8_top1": 55.27 + }, + "performance_metrics": { + "fp32_latency": 3.3447, + "fp32_ov_latency": 1.401, + "int8_latency": 0.6003 + } + }, + "quantization_aware_training_torch_resnet18": { + "backend": "torch", + "requirements": "examples/quantization_aware_training/torch/resnet18/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_tolerance_after_training": 1.0, + "accuracy_tolerance": 0.2, + "accuracy_metrics": { + "fp32_top1": 55.52000045776367, + "int8_init_top1": 55.279998779296875 + }, + "accuracy_metrics_after_training": { + "int8_top1": 57.08069620253165, + "accuracy_drop": -1.5606957447679761 + }, + "performance_metrics": { + "fp32_fps": 3646.13, + "int8_fps": 15301.98, + "performance_speed_up": 4.196773016869941 + }, + "model_size_metrics": { + "fp32_model_size": 43.06997013092041, + "int8_model_size": 11.01035213470459, + "model_compression_rate": 3.9117704505711512 + } + }, "llm_compression": { "backend": "openvino", "requirements": "examples/llm_compression/openvino/tiny_llama/requirements.txt", @@ -14,5 +210,159 @@ "accuracy_metrics": { "word_count": 69 } + }, + "llm_compression_onnx": { + "backend": "onnx", + "requirements": "examples/llm_compression/onnx/tiny_llama/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_metrics": { + "word_count": 75 + } + }, + "llm_compression_scale_estimation_onnx": { + "backend": "onnx", + "requirements": "examples/llm_compression/onnx/tiny_llama_scale_estimation/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_metrics": { + "word_count": 81 + } + }, + "llm_tune_params": { + "backend": "openvino", + "requirements": "examples/llm_compression/openvino/tiny_llama_find_hyperparams/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_metrics": { + "awq": true, + "ratio": 1.0, + "group_size": 128 + } + }, + "llm_compression_synthetic": { + "xfail": "Differ output on windows: ticket-187175", + "backend": "openvino", + "requirements": "examples/llm_compression/openvino/tiny_llama_synthetic_data/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_metrics": { + "word_count": 84 + } + }, + "quantization_aware_training_torch_anomalib": { + "backend": "torch", + "requirements": "examples/quantization_aware_training/torch/anomalib/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_tolerance_after_training": 0.02, + "accuracy_tolerance": 0.08, + "accuracy_metrics": { + "fp32_f1score": 0.9045643210411072, + "int8_init_f1score": 0.9045643210411072 + }, + "accuracy_metrics_after_training": { + "int8_f1score": 0.9152542352676392 + }, + "performance_metrics": { + "fp32_fps": 316.28, + "int8_fps": 879.84, + "performance_speed_up": 2.7818388769444797 + }, + "model_size_metrics": { + "fp32_model_size": 21.37990665435791, + "int8_model_size": 5.677968978881836, + "model_compression_rate": 3.7654144877995197 + } + }, + "fp8_llm_quantization": { + "backend": "openvino", + "requirements": "examples/llm_compression/openvino/smollm2_360m_fp8/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_metrics": { + "answers": [ + "Paris.", + "Mont Blanc.", + "Toronto.", + "Tokyo." + ] + } + }, + "fp8_llm_quantization_onnx": { + "backend": "onnx", + "requirements": "examples/llm_compression/onnx/smollm2_360m_fp8/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_metrics": { + "answers": [ + "Paris.", + "Mont Blanc.", + "Toronto.", + "Tokyo." + ] + } + }, + "codebook_llm_compression": { + "backend": "openvino", + "requirements": "examples/llm_compression/openvino/smollm2_360m_codebook/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_metrics": { + "answers": [ + "Paris.", + "Mont Blanc.", + "Toronto.", + "Tokyo.", + "Paris.", + "Mont Blanc.", + "Montauin.", + "Fukuoka." + ] + } + }, + "adaptive_codebook_llm_compression": { + "backend": "openvino", + "requirements": "examples/llm_compression/openvino/smollm2_360m_adaptive_codebook/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_metrics": { + "answers": [ + "Paris.", + "Mount Blanc.", + "Toronto.", + "Fukuoka." + ] + } + }, + "llm_compression_distillation_qat_with_lora": { + "backend": "torch", + "device": "cuda", + "CUDA_VISIBLE_DEVICES": "0", + "requirements": "examples/llm_compression/torch/distillation_qat_with_lora/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_tolerance": 0.1, + "accuracy_metrics": { + "best_ov_perplexity": 32.37 + } + }, + "llm_compression_qat_with_nls": { + "backend": "torch", + "device": "cuda", + "requirements": "examples/llm_compression/torch/downstream_qat_with_nls/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_tolerance": 0.02, + "accuracy_metrics": { + "accuracy_diff": 0.03 + } + }, + "magnitude_pruning_torch_resnet18": { + "backend": "torch", + "requirements": "examples/pruning/torch/resnet18/requirements.txt", + "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz", + "accuracy_tolerance": 0.2, + "accuracy_metrics": { + "acc1": 54.96 + } + }, + "llm_compression_torch_gptqmodel_convertor": { + "backend": "torch", + "device": "cuda", + "requirements": "examples/llm_compression/torch/gptq_model_convertor/requirements.txt", + "extra_requirements": "examples/llm_compression/torch/gptq_model_convertor/requirements_extra.txt", + "accuracy_metrics": { + "word_count": 28 + } } } From a9d0777af2bbd95f2e873648c211307422e12b31 Mon Sep 17 00:00:00 2001 From: Alexander Dokuchaev Date: Thu, 9 Jul 2026 12:01:45 +0300 Subject: [PATCH 4/4] Add xfail annotation for llm_compression_fx due to output differences on Windows --- tests/cross_fw/examples/example_scope.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cross_fw/examples/example_scope.json b/tests/cross_fw/examples/example_scope.json index dbc9501189c..45146e5210e 100644 --- a/tests/cross_fw/examples/example_scope.json +++ b/tests/cross_fw/examples/example_scope.json @@ -204,6 +204,7 @@ } }, "llm_compression_fx": { + "xfail": "Differ output on windows: ticket-187175", "backend": "torchfx", "requirements": "examples/llm_compression/torch_fx/tiny_llama/requirements.txt", "cpu": "Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz",