diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 8a5b2f4805a..060b5ae3da5 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -674,6 +674,40 @@ jobs: build-tool: buck2 docker-image: ci-image:executorch-ubuntu-22.04-clang12 + test-qnn-buck-build-linux: + name: test-qnn-buck-build-linux + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + with: + runner: linux.2xlarge + docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 + script: | + set -eux + + CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + conda activate "${CONDA_ENV}" + + # Download QNN SDK and get the path + QNN_SDK_ROOT=$(python3 backends/qualcomm/scripts/download_qnn_sdk.py --print-sdk-path) + echo "QNN_SDK_ROOT=${QNN_SDK_ROOT}" + + # Configure Buck to find the QNN SDK + echo "[qualcomm]" >> .buckconfig + echo " qnn_sdk_root = ${QNN_SDK_ROOT}" >> .buckconfig + + # Setup buck2 + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool buck2 + + # Build QNN backend with Buck + buck2 build //backends/qualcomm/... + unittest-arm-backend-with-no-deps: name: unittest-arm-backend-with-no-deps uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main diff --git a/backends/qualcomm/aot/python/targets.bzl b/backends/qualcomm/aot/python/targets.bzl index f9dffafed9f..8dc8bcba3d4 100644 --- a/backends/qualcomm/aot/python/targets.bzl +++ b/backends/qualcomm/aot/python/targets.bzl @@ -3,7 +3,7 @@ load( "CXX", ) load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") -load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version") +load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep") PYTHON_MODULE_NAME = "PyQnnManagerAdaptor" @@ -32,9 +32,9 @@ def define_common_targets(): "//executorch/backends/qualcomm/runtime:logging", "//executorch/backends/qualcomm:schema", "//executorch/backends/qualcomm/runtime:runtime", - "fbsource//third-party/pybind11:pybind11", - "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), - "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()), + qnn_third_party_dep("api"), + qnn_third_party_dep("app_sources"), + qnn_third_party_dep("pybind11"), ], external_deps = [ "libtorch_python", @@ -61,8 +61,8 @@ def define_common_targets(): "//executorch/backends/qualcomm/runtime:logging", "//executorch/backends/qualcomm:schema", "//executorch/backends/qualcomm/runtime:runtime", - "fbsource//third-party/pybind11:pybind11", - "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), - "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()), + qnn_third_party_dep("api"), + qnn_third_party_dep("app_sources"), + qnn_third_party_dep("pybind11"), ], ) diff --git a/backends/qualcomm/aot/wrappers/targets.bzl b/backends/qualcomm/aot/wrappers/targets.bzl index cbd7ce32427..89f8efdea3e 100644 --- a/backends/qualcomm/aot/wrappers/targets.bzl +++ b/backends/qualcomm/aot/wrappers/targets.bzl @@ -3,7 +3,7 @@ load( "ANDROID", ) load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") -load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version") +load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep") def define_common_targets(): """Defines targets that should be shared between fbcode and xplat. @@ -23,8 +23,8 @@ def define_common_targets(): platforms = [ANDROID], visibility = ["PUBLIC"], deps = [ - "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), - "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()), + qnn_third_party_dep("api"), + qnn_third_party_dep("app_sources"), "//executorch/runtime/backend:interface", "//executorch/runtime/core:core", ], diff --git a/backends/qualcomm/recipes/BUCK b/backends/qualcomm/recipes/BUCK index e76c3760fb3..9b3e43a4641 100644 --- a/backends/qualcomm/recipes/BUCK +++ b/backends/qualcomm/recipes/BUCK @@ -23,9 +23,7 @@ fbcode_target(_kind = runtime.python_library, ], visibility = ["PUBLIC"], deps = [ - "//caffe2:torch", "//executorch/export:lib", - "//executorch/runtime:runtime", # @manual "//executorch/backends/qualcomm/partition:partition", "//executorch/backends/qualcomm/serialization:serialization", "//executorch/backends/qualcomm/utils:utils", diff --git a/backends/qualcomm/runtime/targets.bzl b/backends/qualcomm/runtime/targets.bzl index 78ec448ba84..c6be1ef23b7 100644 --- a/backends/qualcomm/runtime/targets.bzl +++ b/backends/qualcomm/runtime/targets.bzl @@ -3,7 +3,7 @@ load( "ANDROID", ) load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") -load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version") +load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep") def define_common_targets(): """Defines targets that should be shared between fbcode and xplat. @@ -24,12 +24,12 @@ def define_common_targets(): platforms = [ANDROID], visibility = ["PUBLIC"], deps = [ - "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), - "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()), + qnn_third_party_dep("api"), + qnn_third_party_dep("app_sources"), "//executorch/runtime/backend:interface", ], exported_deps = [ - "fbsource//third-party/toolchains:log", + qnn_third_party_dep("log"), "//executorch/backends/qualcomm:schema", "//executorch/runtime/core:core", ], @@ -68,12 +68,12 @@ def define_common_targets(): platforms = [ANDROID], visibility = ["PUBLIC"], resources = ({ - "qnn_lib": "fbsource//third-party/qualcomm/qnn/qnn-{0}:qnn_offline_compile_libs".format(get_qnn_library_version()), + "qnn_lib": qnn_third_party_dep("qnn_offline_compile_libs"), } if include_aot_qnn_lib else { }), deps = [ - "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), - "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()), + qnn_third_party_dep("api"), + qnn_third_party_dep("app_sources"), ":logging", "//executorch/backends/qualcomm:schema", "//executorch/backends/qualcomm/aot/wrappers:wrappers", diff --git a/backends/qualcomm/targets.bzl b/backends/qualcomm/targets.bzl index 5c604f9837b..e3b9b7ccfbb 100644 --- a/backends/qualcomm/targets.bzl +++ b/backends/qualcomm/targets.bzl @@ -3,12 +3,15 @@ load( "ANDROID", ) load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") -load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version") +load("@fbsource//xplat/executorch/backends/qualcomm/third-party:third_party_libs.bzl", "qnn_third_party_dep") # Construct the input and output file names. All input and output files rely on scalar_type file. SCHEMA_NAME = "qc_compiler_spec" -INPUT_SCHEMA = "serialization/" + SCHEMA_NAME + ".fbs" +# In OSS, genrule srcs can't use cross-package relative paths, so use +# an export_file target. In fbcode/xplat, relative path works and avoids +# the srcs patching gap in _patch_executorch_references. +INPUT_SCHEMA = "//backends/qualcomm/serialization:qc_compiler_spec.fbs" if runtime.is_oss else "serialization/" + SCHEMA_NAME + ".fbs" OUTPUT_SCHEMA_HEADER = SCHEMA_NAME + "_generated.h" @@ -84,7 +87,7 @@ def define_common_targets(): define_static_target = True, visibility = ["PUBLIC"], deps = [ - "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), + qnn_third_party_dep("api"), "//executorch/runtime/backend:interface", "//executorch/runtime/core:core", "//executorch/backends/qualcomm/runtime:runtime_android_build", diff --git a/backends/qualcomm/tests/BUCK b/backends/qualcomm/tests/BUCK index 502ad59df24..1328cdc48cf 100644 --- a/backends/qualcomm/tests/BUCK +++ b/backends/qualcomm/tests/BUCK @@ -13,6 +13,8 @@ fbcode_target(_kind = runtime.python_library, ] ) +# Test targets have transitive deps (caffe2, torchvision, torchaudio, etc.) +# that are not available in OSS Buck builds. fbcode_target(_kind = runtime.python_library, name = "test_qnn_delegate", srcs = [ @@ -28,7 +30,6 @@ fbcode_target(_kind = runtime.python_library, "//caffe2/functorch:functorch_src", "//executorch/exir/backend:partitioner", "//executorch/exir/dialects:lib", - "//executorch/extension/pybindings:portable_lib", # @manual "//executorch/extension/pytree:pylib", "//executorch/backends/qualcomm/partition:partition", "//executorch/backends/qualcomm/quantizer:quantizer", @@ -36,10 +37,15 @@ fbcode_target(_kind = runtime.python_library, "//executorch/backends/qualcomm/utils:utils", "//executorch/devtools:lib", "//executorch/examples/qualcomm:utils", - "//executorch/examples/models:models", "//executorch/backends/qualcomm/debugger:utils", "//executorch/backends/qualcomm/debugger:qnn_intermediate_debugger", - ], + ] + ([] if runtime.is_oss else [ + # These deps have pre-existing issues in OSS Buck: + # portable_lib: shim//build_defs/cpp_python_extension.bzl missing + # models: examples/models/BUCK uses raw python_library (no dep patching) + "//executorch/extension/pybindings:portable_lib", # @manual + "//executorch/examples/models:models", + ]), ) fbcode_target(_kind = runtime.python_library, @@ -67,9 +73,11 @@ fbcode_target(_kind = runtime.python_test, "//executorch/backends/qualcomm/partition:partition", "//executorch/backends/qualcomm/serialization:serialization", "//executorch/backends/qualcomm/utils:utils", + "//executorch/backends/qualcomm/builders:builders", + ] + ([] if runtime.is_oss else [ + # These deps fail in OSS: keep_gpu_sections kwarg breaks TARGETS evaluation "//executorch/examples/models/llama:transformer_modules", "//executorch/examples/qualcomm/oss_scripts/llama:masking_utils", "//executorch/examples/qualcomm/oss_scripts/llama:static_llama", - "//executorch/backends/qualcomm/builders:builders", - ], + ]), ) diff --git a/backends/qualcomm/third-party/BUCK b/backends/qualcomm/third-party/BUCK new file mode 100644 index 00000000000..2084fd5568c --- /dev/null +++ b/backends/qualcomm/third-party/BUCK @@ -0,0 +1,118 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") + +# OSS prebuilt targets for Qualcomm QNN SDK. +# These wrap the externally-downloaded SDK (via QNN_SDK_ROOT env var or +# backends/qualcomm/scripts/download_qnn_sdk.py) so that Buck builds +# in OSS CI can compile the QNN backend without fbsource third-party targets. +# +# Set QNN_SDK_ROOT before running buck2 build, or use .buckconfig: +# [qualcomm] +# qnn_sdk_root = /path/to/qnn/sdk + +oncall("executorch") + +_QNN_SDK_ROOT = native.read_config("qualcomm", "qnn_sdk_root", "") +_QNN_APP_SDK_FILES = [ + ("QnnModel.cpp", "share/QNN/converter/jni/QnnModel.cpp"), + ("QnnWrapperUtils.cpp", "share/QNN/converter/jni/QnnWrapperUtils.cpp"), + ("linux_QnnModelPal.cpp", "share/QNN/converter/jni/linux/QnnModelPal.cpp"), + ("windows_QnnModelPal.cpp", "share/QNN/converter/jni/windows/QnnModelPal.cpp"), + ("QnnModel.hpp", "share/QNN/converter/jni/QnnModel.hpp"), + ("QnnModelPal.hpp", "share/QNN/converter/jni/QnnModelPal.hpp"), + ("QnnTypeMacros.hpp", "share/QNN/converter/jni/QnnTypeMacros.hpp"), + ("QnnWrapperUtils.hpp", "share/QNN/converter/jni/QnnWrapperUtils.hpp"), +] + +_QNN_APP_SRCS = [ + ":qnn_app_sources_files[QnnModel.cpp]", + ":qnn_app_sources_files[QnnWrapperUtils.cpp]", +] + select({ + "DEFAULT": [":qnn_app_sources_files[linux_QnnModelPal.cpp]"], + "ovr_config//os:linux": [":qnn_app_sources_files[linux_QnnModelPal.cpp]"], + "ovr_config//os:windows": [":qnn_app_sources_files[windows_QnnModelPal.cpp]"], +}) + +_QNN_APP_HEADERS = [ + ":qnn_app_sources_files[QnnModel.hpp]", + ":qnn_app_sources_files[QnnModelPal.hpp]", + ":qnn_app_sources_files[QnnTypeMacros.hpp]", + ":qnn_app_sources_files[QnnWrapperUtils.hpp]", +] + +_QNN_APP_EXPORTED_HEADERS = { + name: ":qnn_app_sources_files[{}]".format(name) + for name in [ + "QnnModel.hpp", + "QnnModelPal.hpp", + "QnnTypeMacros.hpp", + "QnnWrapperUtils.hpp", + ] +} + +# Mirror QNN SDK API headers into declared outputs so that Buck2's sandbox +# can access them (raw -I to an absolute SDK path is not sandbox-safe). +runtime.genrule( + name = "qnn_api_headers", + out = "include", + cmd = ( + "cp -rLf \"{root}/include/QNN/.\" \"$OUT/\"".format(root = _QNN_SDK_ROOT) + if _QNN_SDK_ROOT else "mkdir -p \"$OUT\"" + ), + visibility = [], +) + +# QNN API headers (include/QNN/) +runtime.cxx_library( + name = "qnn_api", + exported_preprocessor_flags = [ + "-I$(location :qnn_api_headers)", + ], + exported_headers = {}, + visibility = ["PUBLIC"], +) + +# Mirror the fixed JNI SDK files into declared outputs instead of globbing an +# absolute SDK path, which Buck rejects during package evaluation. +runtime.genrule( + name = "qnn_app_sources_files", + outs = {name: [path] for name, path in _QNN_APP_SDK_FILES}, + default_outs = ["."], + cmd = " && ".join( + [ + "mkdir -p $OUT/share/QNN/converter/jni/linux $OUT/share/QNN/converter/jni/windows", + ] + ( + [ + "cp -f \"{root}/{path}\" \"$OUT/{path}\"".format(root = _QNN_SDK_ROOT, path = path) + for _, path in _QNN_APP_SDK_FILES + ] if _QNN_SDK_ROOT else [ + "touch \"$OUT/{path}\"".format(path = path) + for _, path in _QNN_APP_SDK_FILES + ] + ), + ), +) + +# QNN app utility sources (converter/jni/) +runtime.cxx_library( + name = "qnn_app_sources", + srcs = _QNN_APP_SRCS if _QNN_SDK_ROOT else [], + headers = _QNN_APP_HEADERS if _QNN_SDK_ROOT else [], + header_namespace = "", + exported_headers = _QNN_APP_EXPORTED_HEADERS if _QNN_SDK_ROOT else {}, + visibility = ["PUBLIC"], + deps = [":qnn_api"], +) + +# QNN offline compile libs (placeholder — only needed for AOT resources) +runtime.filegroup( + name = "qnn_offline_compile_libs", + srcs = [], + visibility = ["PUBLIC"], +) + +# Android log stub (noop on non-Android) +runtime.cxx_library( + name = "log", + exported_headers = {}, + visibility = ["PUBLIC"], +) diff --git a/backends/qualcomm/third-party/third_party_libs.bzl b/backends/qualcomm/third-party/third_party_libs.bzl new file mode 100644 index 00000000000..0c8ad53c483 --- /dev/null +++ b/backends/qualcomm/third-party/third_party_libs.bzl @@ -0,0 +1,33 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") +load("@fbsource//xplat/executorch/backends/qualcomm/qnn_version.bzl", "get_qnn_library_version") + +# Dictionary mapping third-party library name to [internal_dep, oss_dep]. +# Internal deps use fbsource//third-party/qualcomm/qnn/qnn-{version}:target. +# OSS deps use //backends/qualcomm/third-party:target (prebuilt from QNN_SDK_ROOT). +_QNN_THIRD_PARTY_LIBS = { + "api": [ + "fbsource//third-party/qualcomm/qnn/qnn-{0}:api".format(get_qnn_library_version()), + "//backends/qualcomm/third-party:qnn_api", + ], + "app_sources": [ + "fbsource//third-party/qualcomm/qnn/qnn-{0}:app_sources".format(get_qnn_library_version()), + "//backends/qualcomm/third-party:qnn_app_sources", + ], + "qnn_offline_compile_libs": [ + "fbsource//third-party/qualcomm/qnn/qnn-{0}:qnn_offline_compile_libs".format(get_qnn_library_version()), + "//backends/qualcomm/third-party:qnn_offline_compile_libs", + ], + "log": [ + "fbsource//third-party/toolchains:log", + "//backends/qualcomm/third-party:log", + ], + "pybind11": [ + "fbsource//third-party/pybind11:pybind11", + "//third-party:pybind11", + ], +} + +def qnn_third_party_dep(name): + if name not in _QNN_THIRD_PARTY_LIBS: + fail("Cannot find QNN third party library " + name + ", please register it in _QNN_THIRD_PARTY_LIBS first!") + return _QNN_THIRD_PARTY_LIBS[name][1] if runtime.is_oss else _QNN_THIRD_PARTY_LIBS[name][0] diff --git a/devtools/inspector/TARGETS b/devtools/inspector/TARGETS index 8834dfbb6ba..db71623d208 100644 --- a/devtools/inspector/TARGETS +++ b/devtools/inspector/TARGETS @@ -1,4 +1,3 @@ -load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") oncall("executorch") @@ -23,7 +22,7 @@ runtime.python_library( ], ) -python_binary( +runtime.python_binary( name = "inspector_cli", main_function = ".inspector_cli.main", main_src = "inspector_cli.py", diff --git a/shim_et/xplat/executorch/build/runtime_wrapper.bzl b/shim_et/xplat/executorch/build/runtime_wrapper.bzl index 01004595ff1..9761bc7d0e9 100644 --- a/shim_et/xplat/executorch/build/runtime_wrapper.bzl +++ b/shim_et/xplat/executorch/build/runtime_wrapper.bzl @@ -377,6 +377,19 @@ def _python_library(*args, **kwargs): def _python_binary(*args, **kwargs): _patch_kwargs_common(kwargs) + + # In OSS, native.python_binary doesn't support fbcode-specific params. + # Convert main_src -> main, and move srcs entries into main if needed. + if env.is_oss: + main_src = kwargs.pop("main_src", None) + srcs = kwargs.pop("srcs", None) + if main_src: + kwargs.setdefault("main", main_src) + elif srcs: + # If srcs provided but no main/main_src, use first src as main + if "main" not in kwargs: + kwargs["main"] = srcs[0] + env.python_binary(*args, **kwargs) def _python_test(*args, **kwargs):