Skip to content

Commit 7c67b62

Browse files
abhinaykukkadapufacebook-github-bot
abhinaykukkadapu
authored and
facebook-github-bot
committed
Enable test_qnn_delegate tests to run on PR (#19050)
Summary: Enable `test_qnn_delegate` tests to run on PR Differential Revision: D102010420
1 parent c48ea12 commit 7c67b62

4 files changed

Lines changed: 183 additions & 2 deletions

File tree

‎.ci/scripts/test_qnn_imports.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/usr/bin/env python3
2+
# Copyright (c) Qualcomm Innovation Center, Inc.
3+
# All rights reserved
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
"""Validate that all QNN backend Python modules can be imported.
9+
10+
Walks executorch.backends.qualcomm.* via pkgutil and attempts to import
11+
each module. Reports failures with module name and error, then exits
12+
non-zero if any imports failed. Run after editable install + QNN SDK
13+
setup to catch missing dependencies early in CI.
14+
"""
15+
16+
import importlib
17+
import pkgutil
18+
import sys
19+
20+
21+
def main():
22+
failures = []
23+
count = 0
24+
for _importer, name, _ispkg in pkgutil.walk_packages(
25+
["backends/qualcomm"], prefix="executorch.backends.qualcomm."
26+
):
27+
# Skip fb-internal and test modules
28+
if ".fb." in name or ".tests." in name:
29+
continue
30+
count += 1
31+
try:
32+
importlib.import_module(name)
33+
except Exception as e:
34+
failures.append((name, type(e).__name__, str(e)))
35+
36+
if failures:
37+
print(f"{len(failures)}/{count} import failure(s):")
38+
for name, etype, msg in failures:
39+
print(f" FAIL: {name} -- {etype}: {msg}")
40+
sys.exit(1)
41+
42+
print(f"All {count} QNN modules imported successfully")
43+
44+
45+
if __name__ == "__main__":
46+
main()

‎.github/workflows/pull.yml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -923,6 +923,74 @@ jobs:
923923
# Run QNN pass unit tests
924924
pytest -xvs backends/qualcomm/tests/test_passes.py
925925
926+
test-qnn-python-imports-linux:
927+
name: test-qnn-python-imports-linux
928+
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
929+
permissions:
930+
id-token: write
931+
contents: read
932+
strategy:
933+
fail-fast: false
934+
with:
935+
runner: linux.2xlarge
936+
docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
937+
submodules: 'recursive'
938+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
939+
timeout: 15
940+
script: |
941+
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
942+
conda activate "${CONDA_ENV}"
943+
944+
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh
945+
PYTHON_EXECUTABLE=python source .ci/scripts/build-qnn-sdk.sh
946+
947+
CMAKE_ARGS="-DEXECUTORCH_BUILD_QNN=ON -DQNN_SDK_ROOT=$QNN_SDK_ROOT -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON" \
948+
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake --editable true
949+
950+
pip install -r requirements-examples.txt
951+
952+
# Validate all QNN Python imports resolve.
953+
# Catches missing deps (lm_eval, openpyxl, etc.) early with a clear error.
954+
python .ci/scripts/test_qnn_imports.py
955+
956+
test-qnn-delegate-linux:
957+
name: test-qnn-delegate-linux
958+
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
959+
permissions:
960+
id-token: write
961+
contents: read
962+
strategy:
963+
fail-fast: false
964+
with:
965+
runner: linux.2xlarge
966+
docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
967+
submodules: 'recursive'
968+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
969+
timeout: 90
970+
script: |
971+
# The generic Linux job chooses to use base env, not the one setup by the image
972+
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
973+
conda activate "${CONDA_ENV}"
974+
975+
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh
976+
# Source (not bash) so QNN_SDK_ROOT stays in the environment
977+
PYTHON_EXECUTABLE=python source .ci/scripts/build-qnn-sdk.sh
978+
979+
# Editable install so the PyQnnManagerAdaptor .so built by build-qnn-sdk.sh
980+
# is visible in the source tree
981+
CMAKE_ARGS="-DEXECUTORCH_BUILD_QNN=ON -DQNN_SDK_ROOT=$QNN_SDK_ROOT -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON" \
982+
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake --editable true
983+
984+
pip install -r requirements-examples.txt
985+
986+
# Enable conftest.py to call setup_environment() for pytest runs
987+
export QNN_DELEGATE_TEST=1
988+
export LD_LIBRARY_PATH="$QNN_SDK_ROOT/lib/x86_64-linux-clang/:$(realpath build-x86/lib/):${LD_LIBRARY_PATH:-}"
989+
990+
# Run operator test classes only.
991+
pytest -vs backends/qualcomm/tests/test_qnn_delegate.py \
992+
-k "TestQNNFloatingPointOperator or TestQNNQuantizedOperator"
993+
926994
test-phi-3-mini-runner-linux:
927995
name: test-phi-3-mini-runner-linux
928996
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main

‎backends/qualcomm/tests/conftest.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Copyright (c) Qualcomm Innovation Center, Inc.
2+
# All rights reserved
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
"""Pytest conftest for test_qnn_delegate.py.
8+
9+
When test_qnn_delegate.py is invoked via pytest (instead of as __main__),
10+
setup_environment() is never called, so TestQNN class attributes remain at
11+
their defaults (None / empty string / False). This conftest bridges the gap
12+
by calling setup_environment() with a synthetic argv built from environment
13+
variables, keeping a single source of truth for attribute assignment.
14+
15+
Required env vars for x86 simulator execution:
16+
QNN_SOC_MODEL - e.g. "SM8650"
17+
QNN_SDK_ROOT - path to QNN SDK (also used by verify_output)
18+
19+
Optional env vars (with defaults shown):
20+
QNN_BACKEND - "htp"
21+
QNN_BUILD_FOLDER - "build-x86"
22+
QNN_ENABLE_X86_64 - "1" to enable x86 simulator mode
23+
EXECUTORCH_ROOT - cwd
24+
QNN_ARTIFACT_DIR - "/tmp/qnn_test_artifacts"
25+
"""
26+
27+
import os
28+
import sys
29+
30+
31+
def pytest_configure(config):
32+
if os.environ.get("QNN_DELEGATE_TEST") != "1":
33+
return
34+
35+
argv = [
36+
"test_qnn_delegate.py",
37+
"--soc_model",
38+
os.environ.get("QNN_SOC_MODEL", "SM8650"),
39+
"--backend",
40+
os.environ.get("QNN_BACKEND", "htp"),
41+
"--build_folder",
42+
os.environ.get("QNN_BUILD_FOLDER", "build-x86"),
43+
"--executorch_root",
44+
os.environ.get("EXECUTORCH_ROOT", os.getcwd()),
45+
"--artifact_dir",
46+
os.environ.get("QNN_ARTIFACT_DIR", "/tmp/qnn_test_artifacts"),
47+
]
48+
49+
if os.environ.get("QNN_ENABLE_X86_64", "1") == "1":
50+
argv.append("--enable_x86_64")
51+
52+
original_argv = sys.argv
53+
try:
54+
sys.argv = argv
55+
from executorch.backends.qualcomm.tests.test_qnn_delegate import (
56+
setup_environment,
57+
)
58+
59+
setup_environment()
60+
finally:
61+
sys.argv = original_argv

‎backends/qualcomm/tests/test_qnn_delegate.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1308,6 +1308,8 @@ def test_qnn_backend_is_inf(self):
13081308
self.lower_module_and_test_output(module, sample_input)
13091309

13101310
def test_qnn_backend_is_nan(self):
1311+
if self.enable_x86_64:
1312+
self.skipTest("isnan produces incorrect results on x86 simulator")
13111313
module = IsNan() # noqa: F405
13121314
sample_inputs = [
13131315
(
@@ -2182,7 +2184,9 @@ def test_qnn_backend_causal_mask(self):
21822184
module = CausalMask() # noqa: F405
21832185
torch.manual_seed(8)
21842186
sample_input = (torch.rand((1, 1, 1, 128)) < 0.5,)
2185-
self.lower_module_and_test_output(module, sample_input)
2187+
self.lower_module_and_test_output(
2188+
module, sample_input, skip_mutable_buffer=self.enable_x86_64
2189+
)
21862190

21872191
def test_qnn_backend_chunk_add(self):
21882192
module = ChunkAdd() # noqa: F405
@@ -4766,7 +4770,9 @@ def test_qnn_backend_causal_mask(self):
47664770
module = CausalMask() # noqa: F405
47674771
sample_input = (torch.rand((1, 1, 1, 128)) < 0.5,)
47684772
module = self.get_qdq_module(module, sample_input)
4769-
self.lower_module_and_test_output(module, sample_input)
4773+
self.lower_module_and_test_output(
4774+
module, sample_input, skip_mutable_buffer=self.enable_x86_64
4775+
)
47704776

47714777
def test_qnn_backend_chunk_add(self):
47724778
module = ChunkAdd() # noqa: F405

0 commit comments

Comments
 (0)