Skip to content

Commit edf504d

Browse files
Add OSS Buck build support and CI for Qualcomm backend (#18666)
Summary: Keep OSS CI and Internal builds in sync to avoid breaking changes when OSS prs land for ET-QNN delegate. This diff adds two things to prevent future CMake/Buck drift: 1. **Third-party dep shim** (`third_party_libs.bzl` + OSS `BUCK`): - `qnn_third_party_dep()` function routes QNN SDK deps to internal targets (`fbsource//third-party/qualcomm/qnn/...`) or OSS prebuilt targets (`//backends/qualcomm/third-party:...`) based on `runtime.is_oss`. Follows the XNNPACK third_party_libs.bzl pattern. - OSS BUCK file wraps the externally-downloaded QNN SDK (via `QNN_SDK_ROOT` in `.buckconfig`) with prebuilt Buck targets. - All `targets.bzl` files updated to use the shim instead of hardcoded `fbsource//third-party/...` paths. 2. **Buck build CI job** in `pull.yml`: - New `test-qnn-buck-build-linux` job using existing QNN SDK Docker image. Downloads SDK via `download_qnn_sdk.py`, writes path to `.buckconfig`, runs `buck2 build //backends/qualcomm/...`. - If someone adds a file to CMake but not Buck, this job fails. Differential Revision: D99222617
1 parent 042151d commit edf504d

File tree

11 files changed

+235
-29
lines changed

11 files changed

+235
-29
lines changed

.github/workflows/pull.yml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -674,6 +674,40 @@ jobs:
674674
build-tool: buck2
675675
docker-image: ci-image:executorch-ubuntu-22.04-clang12
676676

677+
test-qnn-buck-build-linux:
678+
name: test-qnn-buck-build-linux
679+
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
680+
permissions:
681+
id-token: write
682+
contents: read
683+
strategy:
684+
fail-fast: false
685+
with:
686+
runner: linux.2xlarge
687+
docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
688+
submodules: 'recursive'
689+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
690+
timeout: 90
691+
script: |
692+
set -eux
693+
694+
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
695+
conda activate "${CONDA_ENV}"
696+
697+
# Download QNN SDK and get the path
698+
QNN_SDK_ROOT=$(python3 backends/qualcomm/scripts/download_qnn_sdk.py --print-sdk-path)
699+
echo "QNN_SDK_ROOT=${QNN_SDK_ROOT}"
700+
701+
# Configure Buck to find the QNN SDK
702+
echo "[qualcomm]" >> .buckconfig
703+
echo " qnn_sdk_root = ${QNN_SDK_ROOT}" >> .buckconfig
704+
705+
# Setup buck2
706+
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool buck2
707+
708+
# Build QNN backend with Buck
709+
buck2 build //backends/qualcomm/...
710+
677711
unittest-arm-backend-with-no-deps:
678712
name: unittest-arm-backend-with-no-deps
679713
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main

backends/qualcomm/aot/python/targets.bzl

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ load(
33
"CXX",
44
)
55
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
6-
load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version")
6+
load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep")
77

88
PYTHON_MODULE_NAME = "PyQnnManagerAdaptor"
99

@@ -32,9 +32,9 @@ def define_common_targets():
3232
"//executorch/backends/qualcomm/runtime:logging",
3333
"//executorch/backends/qualcomm:schema",
3434
"//executorch/backends/qualcomm/runtime:runtime",
35-
"fbsource//third-party/pybind11:pybind11",
36-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
37-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
35+
qnn_third_party_dep("api"),
36+
qnn_third_party_dep("app_sources"),
37+
qnn_third_party_dep("pybind11"),
3838
],
3939
external_deps = [
4040
"libtorch_python",
@@ -61,8 +61,8 @@ def define_common_targets():
6161
"//executorch/backends/qualcomm/runtime:logging",
6262
"//executorch/backends/qualcomm:schema",
6363
"//executorch/backends/qualcomm/runtime:runtime",
64-
"fbsource//third-party/pybind11:pybind11",
65-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
66-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
64+
qnn_third_party_dep("api"),
65+
qnn_third_party_dep("app_sources"),
66+
qnn_third_party_dep("pybind11"),
6767
],
6868
)

backends/qualcomm/aot/wrappers/targets.bzl

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ load(
33
"ANDROID",
44
)
55
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
6-
load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version")
6+
load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep")
77

88
def define_common_targets():
99
"""Defines targets that should be shared between fbcode and xplat.
@@ -23,8 +23,8 @@ def define_common_targets():
2323
platforms = [ANDROID],
2424
visibility = ["PUBLIC"],
2525
deps = [
26-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
27-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
26+
qnn_third_party_dep("api"),
27+
qnn_third_party_dep("app_sources"),
2828
"//executorch/runtime/backend:interface",
2929
"//executorch/runtime/core:core",
3030
],

backends/qualcomm/recipes/BUCK

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,7 @@ fbcode_target(_kind = runtime.python_library,
2323
],
2424
visibility = ["PUBLIC"],
2525
deps = [
26-
"//caffe2:torch",
2726
"//executorch/export:lib",
28-
"//executorch/runtime:runtime", # @manual
2927
"//executorch/backends/qualcomm/partition:partition",
3028
"//executorch/backends/qualcomm/serialization:serialization",
3129
"//executorch/backends/qualcomm/utils:utils",

backends/qualcomm/runtime/targets.bzl

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ load(
33
"ANDROID",
44
)
55
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
6-
load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version")
6+
load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep")
77

88
def define_common_targets():
99
"""Defines targets that should be shared between fbcode and xplat.
@@ -24,12 +24,12 @@ def define_common_targets():
2424
platforms = [ANDROID],
2525
visibility = ["PUBLIC"],
2626
deps = [
27-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
28-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
27+
qnn_third_party_dep("api"),
28+
qnn_third_party_dep("app_sources"),
2929
"//executorch/runtime/backend:interface",
3030
],
3131
exported_deps = [
32-
"fbsource//third-party/toolchains:log",
32+
qnn_third_party_dep("log"),
3333
"//executorch/backends/qualcomm:schema",
3434
"//executorch/runtime/core:core",
3535
],
@@ -68,12 +68,12 @@ def define_common_targets():
6868
platforms = [ANDROID],
6969
visibility = ["PUBLIC"],
7070
resources = ({
71-
"qnn_lib": "fbsource//third-party/qualcomm/qnn/qnn-{0}:qnn_offline_compile_libs".format(get_qnn_library_version()),
71+
"qnn_lib": qnn_third_party_dep("qnn_offline_compile_libs"),
7272
} if include_aot_qnn_lib else {
7373
}),
7474
deps = [
75-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
76-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
75+
qnn_third_party_dep("api"),
76+
qnn_third_party_dep("app_sources"),
7777
":logging",
7878
"//executorch/backends/qualcomm:schema",
7979
"//executorch/backends/qualcomm/aot/wrappers:wrappers",

backends/qualcomm/targets.bzl

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,15 @@ load(
33
"ANDROID",
44
)
55
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
6-
load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version")
6+
load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep")
77

88
# Construct the input and output file names. All input and output files rely on scalar_type file.
99
SCHEMA_NAME = "qc_compiler_spec"
1010

11-
INPUT_SCHEMA = "serialization/" + SCHEMA_NAME + ".fbs"
11+
# In OSS, genrule srcs can't use cross-package relative paths, so use
12+
# an export_file target. In fbcode/xplat, relative path works and avoids
13+
# the srcs patching gap in _patch_executorch_references.
14+
INPUT_SCHEMA = "//backends/qualcomm/serialization:qc_compiler_spec.fbs" if runtime.is_oss else "serialization/" + SCHEMA_NAME + ".fbs"
1215

1316
OUTPUT_SCHEMA_HEADER = SCHEMA_NAME + "_generated.h"
1417

@@ -84,7 +87,7 @@ def define_common_targets():
8487
define_static_target = True,
8588
visibility = ["PUBLIC"],
8689
deps = [
87-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
90+
qnn_third_party_dep("api"),
8891
"//executorch/runtime/backend:interface",
8992
"//executorch/runtime/core:core",
9093
"//executorch/backends/qualcomm/runtime:runtime_android_build",

backends/qualcomm/tests/BUCK

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ fbcode_target(_kind = runtime.python_library,
1313
]
1414
)
1515

16+
# Test targets have transitive deps (caffe2, torchvision, torchaudio, etc.)
17+
# that are not available in OSS Buck builds.
1618
fbcode_target(_kind = runtime.python_library,
1719
name = "test_qnn_delegate",
1820
srcs = [
@@ -28,18 +30,22 @@ fbcode_target(_kind = runtime.python_library,
2830
"//caffe2/functorch:functorch_src",
2931
"//executorch/exir/backend:partitioner",
3032
"//executorch/exir/dialects:lib",
31-
"//executorch/extension/pybindings:portable_lib", # @manual
3233
"//executorch/extension/pytree:pylib",
3334
"//executorch/backends/qualcomm/partition:partition",
3435
"//executorch/backends/qualcomm/quantizer:quantizer",
3536
"//executorch/backends/qualcomm/serialization:serialization",
3637
"//executorch/backends/qualcomm/utils:utils",
3738
"//executorch/devtools:lib",
3839
"//executorch/examples/qualcomm:utils",
39-
"//executorch/examples/models:models",
4040
"//executorch/backends/qualcomm/debugger:utils",
4141
"//executorch/backends/qualcomm/debugger:qnn_intermediate_debugger",
42-
],
42+
] + ([] if runtime.is_oss else [
43+
# These deps have pre-existing issues in OSS Buck:
44+
# portable_lib: shim//build_defs/cpp_python_extension.bzl missing
45+
# models: examples/models/BUCK uses raw python_library (no dep patching)
46+
"//executorch/extension/pybindings:portable_lib", # @manual
47+
"//executorch/examples/models:models",
48+
]),
4349
)
4450

4551
fbcode_target(_kind = runtime.python_library,
@@ -67,9 +73,11 @@ fbcode_target(_kind = runtime.python_test,
6773
"//executorch/backends/qualcomm/partition:partition",
6874
"//executorch/backends/qualcomm/serialization:serialization",
6975
"//executorch/backends/qualcomm/utils:utils",
76+
"//executorch/backends/qualcomm/builders:builders",
77+
] + ([] if runtime.is_oss else [
78+
# These deps fail in OSS: keep_gpu_sections kwarg breaks TARGETS evaluation
7079
"//executorch/examples/models/llama:transformer_modules",
7180
"//executorch/examples/qualcomm/oss_scripts/llama:masking_utils",
7281
"//executorch/examples/qualcomm/oss_scripts/llama:static_llama",
73-
"//executorch/backends/qualcomm/builders:builders",
74-
],
82+
]),
7583
)

backends/qualcomm/third-party/BUCK

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
2+
3+
# OSS prebuilt targets for Qualcomm QNN SDK.
4+
# These wrap the externally-downloaded SDK (via QNN_SDK_ROOT env var or
5+
# backends/qualcomm/scripts/download_qnn_sdk.py) so that Buck builds
6+
# in OSS CI can compile the QNN backend without fbsource third-party targets.
7+
#
8+
# Set QNN_SDK_ROOT before running buck2 build, or use .buckconfig:
9+
# [qualcomm]
10+
# qnn_sdk_root = /path/to/qnn/sdk
11+
12+
oncall("executorch")
13+
14+
_QNN_SDK_ROOT = native.read_config("qualcomm", "qnn_sdk_root", "")
15+
_QNN_APP_SDK_FILES = [
16+
("QnnModel.cpp", "share/QNN/converter/jni/QnnModel.cpp"),
17+
("QnnWrapperUtils.cpp", "share/QNN/converter/jni/QnnWrapperUtils.cpp"),
18+
("linux_QnnModelPal.cpp", "share/QNN/converter/jni/linux/QnnModelPal.cpp"),
19+
("windows_QnnModelPal.cpp", "share/QNN/converter/jni/windows/QnnModelPal.cpp"),
20+
("QnnModel.hpp", "share/QNN/converter/jni/QnnModel.hpp"),
21+
("QnnModelPal.hpp", "share/QNN/converter/jni/QnnModelPal.hpp"),
22+
("QnnTypeMacros.hpp", "share/QNN/converter/jni/QnnTypeMacros.hpp"),
23+
("QnnWrapperUtils.hpp", "share/QNN/converter/jni/QnnWrapperUtils.hpp"),
24+
]
25+
26+
_QNN_APP_SRCS = [
27+
":qnn_app_sources_files[QnnModel.cpp]",
28+
":qnn_app_sources_files[QnnWrapperUtils.cpp]",
29+
] + select({
30+
"DEFAULT": [":qnn_app_sources_files[linux_QnnModelPal.cpp]"],
31+
"ovr_config//os:linux": [":qnn_app_sources_files[linux_QnnModelPal.cpp]"],
32+
"ovr_config//os:windows": [":qnn_app_sources_files[windows_QnnModelPal.cpp]"],
33+
})
34+
35+
_QNN_APP_HEADERS = [
36+
":qnn_app_sources_files[QnnModel.hpp]",
37+
":qnn_app_sources_files[QnnModelPal.hpp]",
38+
":qnn_app_sources_files[QnnTypeMacros.hpp]",
39+
":qnn_app_sources_files[QnnWrapperUtils.hpp]",
40+
]
41+
42+
_QNN_APP_EXPORTED_HEADERS = {
43+
name: ":qnn_app_sources_files[{}]".format(name)
44+
for name in [
45+
"QnnModel.hpp",
46+
"QnnModelPal.hpp",
47+
"QnnTypeMacros.hpp",
48+
"QnnWrapperUtils.hpp",
49+
]
50+
}
51+
52+
# Mirror QNN SDK API headers into declared outputs so that Buck2's sandbox
53+
# can access them (raw -I to an absolute SDK path is not sandbox-safe).
54+
runtime.genrule(
55+
name = "qnn_api_headers",
56+
out = "include",
57+
cmd = (
58+
"cp -rLf \"{root}/include/QNN/.\" \"$OUT/\"".format(root = _QNN_SDK_ROOT)
59+
if _QNN_SDK_ROOT else "mkdir -p \"$OUT\""
60+
),
61+
visibility = [],
62+
)
63+
64+
# QNN API headers (include/QNN/)
65+
runtime.cxx_library(
66+
name = "qnn_api",
67+
exported_preprocessor_flags = [
68+
"-I$(location :qnn_api_headers)",
69+
],
70+
exported_headers = {},
71+
visibility = ["PUBLIC"],
72+
)
73+
74+
# Mirror the fixed JNI SDK files into declared outputs instead of globbing an
75+
# absolute SDK path, which Buck rejects during package evaluation.
76+
runtime.genrule(
77+
name = "qnn_app_sources_files",
78+
outs = {name: [path] for name, path in _QNN_APP_SDK_FILES},
79+
default_outs = ["."],
80+
cmd = " && ".join(
81+
[
82+
"mkdir -p $OUT/share/QNN/converter/jni/linux $OUT/share/QNN/converter/jni/windows",
83+
] + (
84+
[
85+
"cp -f \"{root}/{path}\" \"$OUT/{path}\"".format(root = _QNN_SDK_ROOT, path = path)
86+
for _, path in _QNN_APP_SDK_FILES
87+
] if _QNN_SDK_ROOT else [
88+
"touch \"$OUT/{path}\"".format(path = path)
89+
for _, path in _QNN_APP_SDK_FILES
90+
]
91+
),
92+
),
93+
)
94+
95+
# QNN app utility sources (converter/jni/)
96+
runtime.cxx_library(
97+
name = "qnn_app_sources",
98+
srcs = _QNN_APP_SRCS if _QNN_SDK_ROOT else [],
99+
headers = _QNN_APP_HEADERS if _QNN_SDK_ROOT else [],
100+
header_namespace = "",
101+
exported_headers = _QNN_APP_EXPORTED_HEADERS if _QNN_SDK_ROOT else {},
102+
visibility = ["PUBLIC"],
103+
deps = [":qnn_api"],
104+
)
105+
106+
# QNN offline compile libs (placeholder — only needed for AOT resources)
107+
runtime.filegroup(
108+
name = "qnn_offline_compile_libs",
109+
srcs = [],
110+
visibility = ["PUBLIC"],
111+
)
112+
113+
# Android log stub (noop on non-Android)
114+
runtime.cxx_library(
115+
name = "log",
116+
exported_headers = {},
117+
visibility = ["PUBLIC"],
118+
)
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
2+
load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version")
3+
4+
# Dictionary mapping third-party library name to [internal_dep, oss_dep].
5+
# Internal deps use fbsource//third-party/qualcomm/qnn/qnn-{version}:target.
6+
# OSS deps use //backends/qualcomm/third-party:target (prebuilt from QNN_SDK_ROOT).
7+
_QNN_THIRD_PARTY_LIBS = {
8+
"api": [
9+
"fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
10+
"//backends/qualcomm/third-party:qnn_api",
11+
],
12+
"app_sources": [
13+
"fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
14+
"//backends/qualcomm/third-party:qnn_app_sources",
15+
],
16+
"qnn_offline_compile_libs": [
17+
"fbsource//third-party/qualcomm/qnn/qnn-{0}:qnn_offline_compile_libs".format(get_qnn_library_version()),
18+
"//backends/qualcomm/third-party:qnn_offline_compile_libs",
19+
],
20+
"log": [
21+
"fbsource//third-party/toolchains:log",
22+
"//backends/qualcomm/third-party:log",
23+
],
24+
"pybind11": [
25+
"fbsource//third-party/pybind11:pybind11",
26+
"//third-party:pybind11",
27+
],
28+
}
29+
30+
def qnn_third_party_dep(name):
31+
if name not in _QNN_THIRD_PARTY_LIBS:
32+
fail("Cannot find QNN third party library " + name + ", please register it in _QNN_THIRD_PARTY_LIBS first!")
33+
return _QNN_THIRD_PARTY_LIBS[name][1] if runtime.is_oss else _QNN_THIRD_PARTY_LIBS[name][0]

devtools/inspector/TARGETS

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary")
21
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
32

43
oncall("executorch")
@@ -23,7 +22,7 @@ runtime.python_library(
2322
],
2423
)
2524

26-
python_binary(
25+
runtime.python_binary(
2726
name = "inspector_cli",
2827
main_function = ".inspector_cli.main",
2928
main_src = "inspector_cli.py",

0 commit comments

Comments
 (0)