Skip to content

Commit 9dd342c

Browse files
Add OSS Buck build support and CI for Qualcomm backend (#18666)
Differential Revision: D99222617 Pull Request resolved: #18666
1 parent eafc6e2 commit 9dd342c

15 files changed

Lines changed: 254 additions & 49 deletions

File tree

.github/workflows/pull.yml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -674,6 +674,40 @@ jobs:
674674
build-tool: buck2
675675
docker-image: ci-image:executorch-ubuntu-22.04-clang12
676676

677+
test-qnn-buck-build-linux:
678+
name: test-qnn-buck-build-linux
679+
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
680+
permissions:
681+
id-token: write
682+
contents: read
683+
strategy:
684+
fail-fast: false
685+
with:
686+
runner: linux.2xlarge
687+
docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
688+
submodules: 'recursive'
689+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
690+
timeout: 90
691+
script: |
692+
set -eux
693+
694+
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
695+
conda activate "${CONDA_ENV}"
696+
697+
# Download QNN SDK and get the path
698+
QNN_SDK_ROOT=$(python3 backends/qualcomm/scripts/download_qnn_sdk.py --print-sdk-path)
699+
echo "QNN_SDK_ROOT=${QNN_SDK_ROOT}"
700+
701+
# Configure Buck to find the QNN SDK
702+
echo "[qualcomm]" >> .buckconfig
703+
echo " qnn_sdk_root = ${QNN_SDK_ROOT}" >> .buckconfig
704+
705+
# Setup buck2
706+
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool buck2
707+
708+
# Build QNN backend with Buck
709+
buck2 build //backends/qualcomm/...
710+
677711
unittest-arm-backend-with-no-deps:
678712
name: unittest-arm-backend-with-no-deps
679713
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main

backends/qualcomm/aot/python/targets.bzl

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ load(
33
"CXX",
44
)
55
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
6-
load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version")
6+
load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep")
77

88
PYTHON_MODULE_NAME = "PyQnnManagerAdaptor"
99

@@ -32,17 +32,15 @@ def define_common_targets():
3232
"//executorch/backends/qualcomm/runtime:logging",
3333
"//executorch/backends/qualcomm:schema",
3434
"//executorch/backends/qualcomm/runtime:runtime",
35-
"fbsource//third-party/pybind11:pybind11",
36-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
37-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
35+
qnn_third_party_dep("api"),
36+
qnn_third_party_dep("app_sources"),
37+
qnn_third_party_dep("pybind11"),
3838
],
3939
external_deps = [
4040
"libtorch_python",
4141
],
4242
use_static_deps = True,
43-
visibility = [
44-
"//executorch/backends/qualcomm/...",
45-
],
43+
visibility = ["PUBLIC"],
4644
)
4745

4846

@@ -61,8 +59,8 @@ def define_common_targets():
6159
"//executorch/backends/qualcomm/runtime:logging",
6260
"//executorch/backends/qualcomm:schema",
6361
"//executorch/backends/qualcomm/runtime:runtime",
64-
"fbsource//third-party/pybind11:pybind11",
65-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
66-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
62+
qnn_third_party_dep("api"),
63+
qnn_third_party_dep("app_sources"),
64+
qnn_third_party_dep("pybind11"),
6765
],
6866
)

backends/qualcomm/aot/wrappers/targets.bzl

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ load(
33
"ANDROID",
44
)
55
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
6-
load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version")
6+
load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep")
77

88
def define_common_targets():
99
"""Defines targets that should be shared between fbcode and xplat.
@@ -23,8 +23,8 @@ def define_common_targets():
2323
platforms = [ANDROID],
2424
visibility = ["PUBLIC"],
2525
deps = [
26-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
27-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
26+
qnn_third_party_dep("api"),
27+
qnn_third_party_dep("app_sources"),
2828
"//executorch/runtime/backend:interface",
2929
"//executorch/runtime/core:core",
3030
],

backends/qualcomm/recipes/BUCK

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,15 @@ fbcode_target(_kind = runtime.python_library,
2323
],
2424
visibility = ["PUBLIC"],
2525
deps = [
26-
"//caffe2:torch",
2726
"//executorch/export:lib",
28-
"//executorch/runtime:runtime", # @manual
2927
"//executorch/backends/qualcomm/partition:partition",
3028
"//executorch/backends/qualcomm/serialization:serialization",
3129
"//executorch/backends/qualcomm/utils:utils",
3230
"//executorch/backends/qualcomm/_passes:passes",
3331
":qnn_recipe_types",
34-
],
32+
] + ([] if runtime.is_oss else [
33+
"//executorch/runtime:runtime", # @manual
34+
]),
3535
)
3636

3737
fbcode_target(_kind = runtime.python_library,

backends/qualcomm/runtime/targets.bzl

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ load(
33
"ANDROID",
44
)
55
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
6-
load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version")
6+
load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep")
77

88
def define_common_targets():
99
"""Defines targets that should be shared between fbcode and xplat.
@@ -24,12 +24,12 @@ def define_common_targets():
2424
platforms = [ANDROID],
2525
visibility = ["PUBLIC"],
2626
deps = [
27-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
28-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
27+
qnn_third_party_dep("api"),
28+
qnn_third_party_dep("app_sources"),
2929
"//executorch/runtime/backend:interface",
3030
],
3131
exported_deps = [
32-
"fbsource//third-party/toolchains:log",
32+
qnn_third_party_dep("log"),
3333
"//executorch/backends/qualcomm:schema",
3434
"//executorch/runtime/core:core",
3535
],
@@ -71,12 +71,12 @@ def define_common_targets():
7171
platforms = [ANDROID],
7272
visibility = ["PUBLIC"],
7373
resources = ({
74-
"qnn_lib": "fbsource//third-party/qualcomm/qnn/qnn-{0}:qnn_offline_compile_libs".format(get_qnn_library_version()),
74+
"qnn_lib": qnn_third_party_dep("qnn_offline_compile_libs"),
7575
} if include_aot_qnn_lib else {
7676
}),
7777
deps = [
78-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
79-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()),
78+
qnn_third_party_dep("api"),
79+
qnn_third_party_dep("app_sources"),
8080
":logging",
8181
"//executorch/backends/qualcomm:schema",
8282
"//executorch/backends/qualcomm/aot/wrappers:wrappers",

backends/qualcomm/serialization/targets.bzl

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,17 @@ def define_common_targets():
77
TARGETS and BUCK files that call this function.
88
"""
99

10+
# Used by the INPUT_SCHEMA genrule in targets.bzl for OSS builds,
11+
# where cross-package relative paths don't work in genrule srcs.
12+
export_file(
13+
name = "qc_compiler_spec.fbs",
14+
visibility = ["PUBLIC"],
15+
)
16+
1017
export_file(
1118
name = "qc_compiler_spec_schema",
1219
src = "qc_compiler_spec.fbs",
13-
visibility = ["//executorch/backends/qualcomm/serialization/..."],
20+
visibility = ["PUBLIC"],
1421
)
1522

1623
runtime.python_library(

backends/qualcomm/targets.bzl

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,15 @@ load(
33
"ANDROID",
44
)
55
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
6-
load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version")
6+
load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep")
77

88
# Construct the input and output file names. All input and output files rely on scalar_type file.
99
SCHEMA_NAME = "qc_compiler_spec"
1010

11-
INPUT_SCHEMA = "serialization/" + SCHEMA_NAME + ".fbs"
11+
# In OSS, genrule srcs can't use cross-package relative paths, so use
12+
# an export_file target. In fbcode/xplat, relative path works and avoids
13+
# the srcs patching gap in _patch_executorch_references.
14+
INPUT_SCHEMA = "//backends/qualcomm/serialization:qc_compiler_spec.fbs" if runtime.is_oss else "serialization/" + SCHEMA_NAME + ".fbs"
1215

1316
OUTPUT_SCHEMA_HEADER = SCHEMA_NAME + "_generated.h"
1417

@@ -60,15 +63,7 @@ def define_common_targets():
6063
runtime.cxx_library(
6164
name = "schema",
6265
srcs = [],
63-
visibility = [
64-
# Lock this down as tightly as possible to ensure that flatbuffers
65-
# are an implementation detail. Ideally this list would only include
66-
# //executorch/runtime/executor/...
67-
"//executorch/codegen/tools/...",
68-
"//executorch/runtime/executor/...",
69-
"//executorch/backends/qualcomm/...",
70-
"//executorch/backends/qualcomm/runtime/...",
71-
],
66+
visibility = ["PUBLIC"],
7267
exported_headers = {
7368
OUTPUT_SCHEMA_HEADER: ":{}[{}]".format(SCHEMA_GEN_RULE_NAME, OUTPUT_SCHEMA_HEADER),
7469
},
@@ -84,7 +79,7 @@ def define_common_targets():
8479
define_static_target = True,
8580
visibility = ["PUBLIC"],
8681
deps = [
87-
"fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()),
82+
qnn_third_party_dep("api"),
8883
"//executorch/runtime/backend:interface",
8984
"//executorch/runtime/core:core",
9085
"//executorch/backends/qualcomm/runtime:runtime_android_build",

backends/qualcomm/tests/BUCK

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ fbcode_target(_kind = runtime.python_library,
1313
]
1414
)
1515

16+
# Test targets have transitive deps (caffe2, torchvision, torchaudio, etc.)
17+
# that are not available in OSS Buck builds.
1618
fbcode_target(_kind = runtime.python_library,
1719
name = "test_qnn_delegate",
1820
srcs = [
@@ -28,19 +30,23 @@ fbcode_target(_kind = runtime.python_library,
2830
"//caffe2/functorch:functorch_src",
2931
"//executorch/exir/backend:partitioner",
3032
"//executorch/exir/dialects:lib",
31-
"//executorch/extension/pybindings:portable_lib", # @manual
3233
"//executorch/extension/pytree:pylib",
3334
"//executorch/backends/qualcomm/partition:partition",
3435
"//executorch/backends/qualcomm/quantizer:quantizer",
3536
"//executorch/backends/qualcomm/serialization:serialization",
3637
"//executorch/backends/qualcomm/utils:utils",
3738
"//executorch/devtools:lib",
3839
"//executorch/examples/qualcomm:utils",
39-
"//executorch/examples/models:models",
4040
"//executorch/backends/qualcomm:export_utils",
4141
"//executorch/backends/qualcomm/debugger:utils",
4242
"//executorch/backends/qualcomm/debugger:qnn_intermediate_debugger",
43-
],
43+
] + ([] if runtime.is_oss else [
44+
# These deps have pre-existing issues in OSS Buck:
45+
# portable_lib: shim//build_defs/cpp_python_extension.bzl missing
46+
# models: examples/models/BUCK uses raw python_library (no dep patching)
47+
"//executorch/extension/pybindings:portable_lib", # @manual
48+
"//executorch/examples/models:models",
49+
]),
4450
)
4551

4652
fbcode_target(_kind = runtime.python_library,
@@ -68,9 +74,11 @@ fbcode_target(_kind = runtime.python_test,
6874
"//executorch/backends/qualcomm/partition:partition",
6975
"//executorch/backends/qualcomm/serialization:serialization",
7076
"//executorch/backends/qualcomm/utils:utils",
77+
"//executorch/backends/qualcomm/builders:builders",
78+
] + ([] if runtime.is_oss else [
79+
# These deps fail in OSS: keep_gpu_sections kwarg breaks TARGETS evaluation
7180
"//executorch/examples/models/llama:transformer_modules",
7281
"//executorch/examples/qualcomm/oss_scripts/llama:masking_utils",
7382
"//executorch/examples/qualcomm/oss_scripts/llama:static_llama",
74-
"//executorch/backends/qualcomm/builders:builders",
75-
],
83+
]),
7684
)

backends/qualcomm/third-party/BUCK

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
2+
3+
# OSS prebuilt targets for Qualcomm QNN SDK.
4+
# These wrap the externally-downloaded SDK (via QNN_SDK_ROOT env var or
5+
# backends/qualcomm/scripts/download_qnn_sdk.py) so that Buck builds
6+
# in OSS CI can compile the QNN backend without fbsource third-party targets.
7+
#
8+
# Set QNN_SDK_ROOT before running buck2 build, or use .buckconfig:
9+
# [qualcomm]
10+
# qnn_sdk_root = /path/to/qnn/sdk
11+
12+
oncall("executorch")
13+
14+
_QNN_SDK_ROOT = native.read_config("qualcomm", "qnn_sdk_root", "")
15+
_QNN_APP_SDK_FILES = [
16+
("QnnModel.cpp", "share/QNN/converter/jni/QnnModel.cpp"),
17+
("QnnWrapperUtils.cpp", "share/QNN/converter/jni/QnnWrapperUtils.cpp"),
18+
("linux_QnnModelPal.cpp", "share/QNN/converter/jni/linux/QnnModelPal.cpp"),
19+
("windows_QnnModelPal.cpp", "share/QNN/converter/jni/windows/QnnModelPal.cpp"),
20+
("QnnModel.hpp", "share/QNN/converter/jni/QnnModel.hpp"),
21+
("QnnModelPal.hpp", "share/QNN/converter/jni/QnnModelPal.hpp"),
22+
("QnnTypeMacros.hpp", "share/QNN/converter/jni/QnnTypeMacros.hpp"),
23+
("QnnWrapperUtils.hpp", "share/QNN/converter/jni/QnnWrapperUtils.hpp"),
24+
]
25+
26+
_QNN_APP_SRCS = [
27+
":qnn_app_sources_files[QnnModel.cpp]",
28+
":qnn_app_sources_files[QnnWrapperUtils.cpp]",
29+
] + select({
30+
"DEFAULT": [":qnn_app_sources_files[linux_QnnModelPal.cpp]"],
31+
"ovr_config//os:linux": [":qnn_app_sources_files[linux_QnnModelPal.cpp]"],
32+
"ovr_config//os:windows": [":qnn_app_sources_files[windows_QnnModelPal.cpp]"],
33+
})
34+
35+
_QNN_APP_HEADERS = [
36+
":qnn_app_sources_files[QnnModel.hpp]",
37+
":qnn_app_sources_files[QnnModelPal.hpp]",
38+
":qnn_app_sources_files[QnnTypeMacros.hpp]",
39+
":qnn_app_sources_files[QnnWrapperUtils.hpp]",
40+
]
41+
42+
_QNN_APP_EXPORTED_HEADERS = {
43+
name: ":qnn_app_sources_files[{}]".format(name)
44+
for name in [
45+
"QnnModel.hpp",
46+
"QnnModelPal.hpp",
47+
"QnnTypeMacros.hpp",
48+
"QnnWrapperUtils.hpp",
49+
]
50+
}
51+
52+
# Mirror QNN SDK API headers into declared outputs so that Buck2's sandbox
53+
# can access them (raw -I to an absolute SDK path is not sandbox-safe).
54+
runtime.genrule(
55+
name = "qnn_api_headers",
56+
out = "include",
57+
cmd = (
58+
"cp -rLf \"{root}/include/QNN/.\" \"$OUT/\"".format(root = _QNN_SDK_ROOT)
59+
if _QNN_SDK_ROOT else "mkdir -p \"$OUT\""
60+
),
61+
visibility = [],
62+
)
63+
64+
# QNN API headers (include/QNN/)
65+
runtime.cxx_library(
66+
name = "qnn_api",
67+
exported_preprocessor_flags = [
68+
"-I$(location :qnn_api_headers)",
69+
],
70+
exported_headers = {},
71+
visibility = ["PUBLIC"],
72+
)
73+
74+
# Mirror the fixed JNI SDK files into declared outputs instead of globbing an
75+
# absolute SDK path, which Buck rejects during package evaluation.
76+
runtime.genrule(
77+
name = "qnn_app_sources_files",
78+
outs = {name: [path] for name, path in _QNN_APP_SDK_FILES},
79+
default_outs = ["."],
80+
cmd = " && ".join(
81+
[
82+
"mkdir -p $OUT/share/QNN/converter/jni/linux $OUT/share/QNN/converter/jni/windows",
83+
] + (
84+
[
85+
"cp -f \"{root}/{path}\" \"$OUT/{path}\"".format(root = _QNN_SDK_ROOT, path = path)
86+
for _, path in _QNN_APP_SDK_FILES
87+
] if _QNN_SDK_ROOT else [
88+
"touch \"$OUT/{path}\"".format(path = path)
89+
for _, path in _QNN_APP_SDK_FILES
90+
]
91+
),
92+
),
93+
)
94+
95+
# QNN app utility sources (converter/jni/)
96+
runtime.cxx_library(
97+
name = "qnn_app_sources",
98+
srcs = _QNN_APP_SRCS if _QNN_SDK_ROOT else [],
99+
headers = _QNN_APP_HEADERS if _QNN_SDK_ROOT else [],
100+
header_namespace = "",
101+
exported_headers = _QNN_APP_EXPORTED_HEADERS if _QNN_SDK_ROOT else {},
102+
visibility = ["PUBLIC"],
103+
deps = [":qnn_api"],
104+
)
105+
106+
# QNN offline compile libs (placeholder — only needed for AOT resources)
107+
runtime.filegroup(
108+
name = "qnn_offline_compile_libs",
109+
srcs = [],
110+
visibility = ["PUBLIC"],
111+
)
112+
113+
# Android log stub (noop on non-Android)
114+
runtime.cxx_library(
115+
name = "log",
116+
exported_headers = {},
117+
visibility = ["PUBLIC"],
118+
)

0 commit comments

Comments
 (0)