From 3f406f094d40c02b24f138d41745db21e27677e1 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Mon, 28 Apr 2025 12:13:47 -0400 Subject: [PATCH] build(python): Use meson-python for adbc_driver_manager --- .gitignore | 4 +- c/CMakeLists.txt | 8 +- ci/scripts/python_sdist_build.sh | 4 +- ci/scripts/python_wheel_unix_relocate.sh | 4 +- dev/release/rat_exclude_files.txt | 1 + .../adbc_driver_manager/meson.build | 44 ++++++ python/adbc_driver_manager/meson.build | 58 ++++++++ python/adbc_driver_manager/pyproject.toml | 4 +- .../scripts/remove_third_party_wraps.py | 26 ++++ python/adbc_driver_manager/setup.py | 130 ------------------ .../subprojects/arrow-adbc | 1 + .../subprojects/backward-cpp.wrap | 30 ++++ .../adbc_driver_manager/subprojects/fmt.wrap | 19 +++ .../subprojects/nanoarrow.wrap | 19 +++ 14 files changed, 208 insertions(+), 144 deletions(-) create mode 100644 python/adbc_driver_manager/adbc_driver_manager/meson.build create mode 100644 python/adbc_driver_manager/meson.build create mode 100644 python/adbc_driver_manager/scripts/remove_third_party_wraps.py delete mode 100644 python/adbc_driver_manager/setup.py create mode 120000 python/adbc_driver_manager/subprojects/arrow-adbc create mode 100644 python/adbc_driver_manager/subprojects/backward-cpp.wrap create mode 100644 python/adbc_driver_manager/subprojects/fmt.wrap create mode 100644 python/adbc_driver_manager/subprojects/nanoarrow.wrap diff --git a/.gitignore b/.gitignore index 8716f5d7b3..1fbb3a9157 100644 --- a/.gitignore +++ b/.gitignore @@ -124,5 +124,5 @@ target/ /ci/linux-packages/yum/tmp/ # Meson subproject support -/c/subprojects/* -!/c/subprojects/*.wrap +**/subprojects/* +!**/subprojects/*.wrap diff --git a/c/CMakeLists.txt b/c/CMakeLists.txt index 746547636c..cafb79ebac 100644 --- a/c/CMakeLists.txt +++ b/c/CMakeLists.txt @@ -99,12 +99,8 @@ if(ADBC_BUILD_PYTHON) # but you don't technically need -DADBC_DRIVER_MANAGER=ON when installing # other Python packages. To be safe then, we always install the driver # manager package, regardless of the value of -DABC_DRIVER_MANAGER - # --config-settings eidtable_mode=compat required due to - # https://github.com/python/mypy/issues/13392 - add_custom_target(python - COMMAND ${Python3_EXECUTABLE} -m pip install --no-deps -e - "${REPOSITORY_ROOT}/python/adbc_driver_manager" - --config-settings editable_mode=compat) + add_custom_target(python COMMAND ${Python3_EXECUTABLE} -m pip install --no-deps -e + "${REPOSITORY_ROOT}/python/adbc_driver_manager") macro(adbc_install_python_package TARGET) string(TOUPPER ${TARGET} ${TARGET}_LIB_upper) diff --git a/ci/scripts/python_sdist_build.sh b/ci/scripts/python_sdist_build.sh index a153503f7c..f9af87a758 100755 --- a/ci/scripts/python_sdist_build.sh +++ b/ci/scripts/python_sdist_build.sh @@ -30,7 +30,7 @@ echo "=== (${PYTHON_VERSION}) Building ADBC sdists ===" # https://github.com/pypa/pip/issues/7555 # Get the latest pip so we have in-tree-build by default -pip install --upgrade pip setuptools +pip install --upgrade pip build # For drivers, which bundle shared libraries, defer that to install time export _ADBC_IS_SDIST=1 @@ -41,7 +41,7 @@ for component in ${COMPONENTS}; do echo "=== Building $component sdist ===" # python -m build copies to a tempdir, so we can't reference other files in the repo # https://github.com/pypa/pip/issues/5519 - python setup.py sdist + python -m build --sdist . popd done diff --git a/ci/scripts/python_wheel_unix_relocate.sh b/ci/scripts/python_wheel_unix_relocate.sh index dde802aeec..e96b421036 100755 --- a/ci/scripts/python_wheel_unix_relocate.sh +++ b/ci/scripts/python_wheel_unix_relocate.sh @@ -56,7 +56,7 @@ fi echo "=== Relocating wheels ===" # https://github.com/pypa/pip/issues/7555 # Get the latest pip so we have in-tree-build by default -python -m pip install --upgrade pip auditwheel 'cibuildwheel>=2.21.2' delocate setuptools wheel +python -m pip install --upgrade pip auditwheel 'cibuildwheel>=2.21.2' delocate build wheel # Build with Cython debug info export ADBC_BUILD_TYPE="debug" @@ -76,7 +76,7 @@ for component in $COMPONENTS; do # container during build, but it only copies the package # directory, which omits the C++ sources and .git directory, # causing the build to fail. - python setup.py sdist + python -m build --sdist . if [[ "$component" = "adbc_driver_manager" ]]; then python -m cibuildwheel --output-dir repaired_wheels/ dist/$component-*.tar.gz else diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 54ec748354..0cd1a52c5d 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -29,6 +29,7 @@ go/adbc/status_string.go go/adbc/infocode_string.go go/adbc/go.sum java/.mvn/jvm.config +python/*/subprojects/arrow-adbc/* python/*/*/py.typed rat.txt r/*/DESCRIPTION diff --git a/python/adbc_driver_manager/adbc_driver_manager/meson.build b/python/adbc_driver_manager/adbc_driver_manager/meson.build new file mode 100644 index 0000000000..a09f4bf8df --- /dev/null +++ b/python/adbc_driver_manager/adbc_driver_manager/meson.build @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +py.extension_module( + '_backward', + sources: ['_backward.pyx'], + dependencies: [backward_dep], + include_directories: backward_incdir, + override_options: ['cython_language=cpp'], + install: true, + subdir: 'adbc_driver_manager', +) + +py.extension_module( + '_lib', + sources: ['_blocking_impl.cc', '_lib.pyx'], + dependencies: [adbc_driver_manager_dep], + override_options: ['cython_language=cpp'], + install: true, + subdir: 'adbc_driver_manager', +) + +py.extension_module( + '_reader', + sources: ['_reader.pyx'], + dependencies: [adbc_driver_manager_dep], + override_options: ['cython_language=cpp'], + install: true, + subdir: 'adbc_driver_manager', +) diff --git a/python/adbc_driver_manager/meson.build b/python/adbc_driver_manager/meson.build new file mode 100644 index 0000000000..b8e1414faa --- /dev/null +++ b/python/adbc_driver_manager/meson.build @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +project( + 'adbc_driver_manager', + 'cython', + version: run_command( + ['python', 'adbc_driver_manager/_version.py'], + check: true, + ).stdout().replace( + 'Version:', + '', + ).strip(), + license: 'Apache-2.0', + meson_version: '>=1.2.1', + default_options: [ + 'buildtype=release', + 'warning_level=2', + 'cpp_std=c++17', + # conda environments may only provide a static library for + # libbfd, which backward-cpp requires. + 'default_library=static', + ], +) + +backward_dep = dependency('backward-cpp') +# hack until https://github.com/mesonbuild/wrapdb/pull/2072 +backward_incdir = include_directories('subprojects/backward-cpp-1.6') + +adbc_driver_manager_dep = dependency( + 'adbc-driver-manager', + fallback: ['arrow-adbc', 'adbc_driver_manager_dep'], + default_options: ['driver_manager=enabled'], +) +py = import('python').find_installation(pure: false) + +cython_args = [] +if get_option('buildtype') in ['debug', 'debugoptimized'] + cython_args += ['--gdb'] +endif + +subdir('adbc_driver_manager') + +meson.add_dist_script(py, files('scripts/remove_third_party_wraps.py')) diff --git a/python/adbc_driver_manager/pyproject.toml b/python/adbc_driver_manager/pyproject.toml index a99be7c9b2..53e9db06c3 100644 --- a/python/adbc_driver_manager/pyproject.toml +++ b/python/adbc_driver_manager/pyproject.toml @@ -34,8 +34,8 @@ homepage = "https://arrow.apache.org/adbc/" repository = "https://github.com/apache/arrow-adbc" [build-system] -requires = ["Cython", "setuptools >= 61.0.0"] -build-backend = "setuptools.build_meta" +requires = ["Cython", "meson-python"] +build-backend = "mesonpy" [tool.pytest.ini_options] markers = [ diff --git a/python/adbc_driver_manager/scripts/remove_third_party_wraps.py b/python/adbc_driver_manager/scripts/remove_third_party_wraps.py new file mode 100644 index 0000000000..29b2a0ad2d --- /dev/null +++ b/python/adbc_driver_manager/scripts/remove_third_party_wraps.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script is a workaround for the issue described in +# https://github.com/mesonbuild/meson/issues/14520 + +import os +import pathlib + +dist_root = pathlib.Path(os.environ["MESON_DIST_ROOT"]) +os.unlink(dist_root / "subprojects" / "nanoarrow.wrap") +os.unlink(dist_root / "subprojects" / "fmt.wrap") diff --git a/python/adbc_driver_manager/setup.py b/python/adbc_driver_manager/setup.py deleted file mode 100644 index 793ebda15a..0000000000 --- a/python/adbc_driver_manager/setup.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import os -import shutil -import sys -from collections import namedtuple -from pathlib import Path - -from setuptools import Extension, setup - -source_root = Path(__file__).parent -repo_root = source_root.joinpath("../../") - -# ------------------------------------------------------------ -# Resolve C++ Sources - -FileToCopy = namedtuple("FileToCopy", ["source", "dest_dir"]) -files_to_copy = [ - FileToCopy("c/include/arrow-adbc/adbc.h", "arrow-adbc"), - FileToCopy("c/driver_manager/adbc_driver_manager.cc", ""), - FileToCopy("c/include/arrow-adbc/adbc_driver_manager.h", "arrow-adbc"), - FileToCopy("c/vendor/backward/backward.hpp", ""), -] - -for file_to_copy in files_to_copy: - target_filename = file_to_copy.source.split("/")[-1] - source = repo_root.joinpath(file_to_copy.source).resolve() - target_dir = source_root.joinpath( - "adbc_driver_manager", file_to_copy.dest_dir - ).resolve() - target = target_dir.joinpath(target_filename).resolve() - if source.is_file(): - # In-tree build/creating an sdist: copy from project root to local file - # so that setuptools isn't confused - target_dir.mkdir(parents=True, exist_ok=True) - shutil.copy(source, target) - elif not target.is_file(): - # Out-of-tree build missing the C++ source files - raise FileNotFoundError(str(target)) - # Else, when building from sdist, the target will exist but not the source - -# ------------------------------------------------------------ -# Resolve Version (miniver) - - -def get_version(pkg_path): - """ - Load version.py module without importing the whole package. - - Template code from miniver. - """ - from importlib.util import module_from_spec, spec_from_file_location - - spec = spec_from_file_location("version", os.path.join(pkg_path, "_version.py")) - module = module_from_spec(spec) - spec.loader.exec_module(module) - return module.__version__ - - -version = get_version("adbc_driver_manager") - -# ------------------------------------------------------------ -# Resolve compiler flags - -build_type = os.environ.get("ADBC_BUILD_TYPE", "release") - -if sys.platform == "win32": - extra_compile_args = ["/std:c++17", "/DADBC_EXPORTING", "/D_CRT_SECURE_NO_WARNINGS"] - if build_type == "debug": - extra_compile_args.extend(["/DEBUG:FULL"]) -else: - extra_compile_args = ["-std=c++17"] - if build_type == "debug": - # Useful to step through driver manager code in GDB - extra_compile_args.extend(["-ggdb", "-Og"]) - -# ------------------------------------------------------------ -# Setup - -setup( - ext_modules=[ - Extension( - name="adbc_driver_manager._backward", - extra_compile_args=extra_compile_args, - include_dirs=[str(source_root.joinpath("adbc_driver_manager").resolve())], - language="c++", - sources=[ - "adbc_driver_manager/_backward.pyx", - ], - ), - Extension( - name="adbc_driver_manager._lib", - extra_compile_args=extra_compile_args, - include_dirs=[str(source_root.joinpath("adbc_driver_manager").resolve())], - language="c++", - sources=[ - "adbc_driver_manager/_blocking_impl.cc", - "adbc_driver_manager/_lib.pyx", - "adbc_driver_manager/adbc_driver_manager.cc", - ], - ), - Extension( - name="adbc_driver_manager._reader", - extra_compile_args=extra_compile_args, - include_dirs=[str(source_root.joinpath("adbc_driver_manager").resolve())], - language="c++", - sources=[ - "adbc_driver_manager/_reader.pyx", - ], - ), - ], - version=version, -) diff --git a/python/adbc_driver_manager/subprojects/arrow-adbc b/python/adbc_driver_manager/subprojects/arrow-adbc new file mode 120000 index 0000000000..67bbc7af1c --- /dev/null +++ b/python/adbc_driver_manager/subprojects/arrow-adbc @@ -0,0 +1 @@ +../../../c/ \ No newline at end of file diff --git a/python/adbc_driver_manager/subprojects/backward-cpp.wrap b/python/adbc_driver_manager/subprojects/backward-cpp.wrap new file mode 100644 index 0000000000..81cc563564 --- /dev/null +++ b/python/adbc_driver_manager/subprojects/backward-cpp.wrap @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-file] +directory = backward-cpp-1.6 +source_url = https://github.com/bombela/backward-cpp/archive/refs/tags/v1.6.tar.gz +source_filename = backward-cpp-1.6.tar.gz +source_hash = c654d0923d43f1cea23d086729673498e4741fb2457e806cfaeaea7b20c97c10 +patch_filename = backward-cpp_1.6-3_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/backward-cpp_1.6-3/get_patch +patch_hash = 219b7a62f8f05037586ec90ba02b6d552d356f6f1d1171e5803bbe483daf69f0 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/backward-cpp_1.6-3/backward-cpp-1.6.tar.gz +wrapdb_version = 1.6-3 + +[provide] +backward-cpp = backward_dep diff --git a/python/adbc_driver_manager/subprojects/fmt.wrap b/python/adbc_driver_manager/subprojects/fmt.wrap new file mode 100644 index 0000000000..706755cad3 --- /dev/null +++ b/python/adbc_driver_manager/subprojects/fmt.wrap @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-redirect] +filename = arrow-adbc/subprojects/fmt.wrap diff --git a/python/adbc_driver_manager/subprojects/nanoarrow.wrap b/python/adbc_driver_manager/subprojects/nanoarrow.wrap new file mode 100644 index 0000000000..9a89b07d93 --- /dev/null +++ b/python/adbc_driver_manager/subprojects/nanoarrow.wrap @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-redirect] +filename = arrow-adbc/subprojects/nanoarrow.wrap