Move to nanobind (duckdb#522)

Author: evertlammerts

.github/workflows/code_quality.yml

@@ -42,4 +42,4 @@ jobs:
 
       - name: pre-commit (--all-files)
         run: |
-          uvx pre-commit run --show-diff-on-failure --color=always --all-files
+          uvx --python 3.12 pre-commit run --show-diff-on-failure --color=always --all-files

.github/workflows/packaging_wheels.yml

@@ -30,7 +30,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: [ cp314 ]
+        python: [ cp311, cp314 ]
         platform:
           - { os: windows-2022,     arch: amd64,      cibw_system: win }
           - { os: windows-11-arm,   arch: ARM64,      cibw_system: win }
@@ -127,7 +127,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: [ cp310, cp311, cp312, cp313 ]
+        python: [ cp311, cp312, cp313 ]
         platform:
           - { os: windows-2025,     arch: amd64,      cibw_system: win }
           - { os: windows-11-arm,   arch: ARM64,      cibw_system: win }
@@ -143,7 +143,6 @@ jobs:
           - { minimal: true, python: cp312 }
           - { minimal: true, python: cp313 }
           - { minimal: true, platform: { arch: universal2 } }
-          - { python: cp310,  platform: { os: windows-11-arm, arch: ARM64 } }
     runs-on: ${{ matrix.platform.os }}
     env:
       CCACHE_DIR: ${{ github.workspace }}/.ccache

CLAUDE.md

@@ -10,7 +10,7 @@ This is the **production** duckdb-python client — the `duckdb` package on PyPI
 - **Package name**: `duckdb`
 - **Bindings**: pybind11
 - **Build backend**: `duckdb_packaging.build_backend` (custom wrapper around scikit-build-core)
-- **Supported Python**: 3.10, 3.11, 3.12, 3.13, 3.14
+- **Supported Python**: 3.11, 3.12, 3.13, 3.14
 - **Free-threaded Python**: not supported in this client. A separate prototype client based on DuckDB's C API targets free-threading, Stable ABI, and multi-interpreter support.
 
 ## IMPORTANT: build before running anything
@@ -115,7 +115,7 @@ uv sync --no-build-isolation -v --reinstall -p 3.11
 uv sync --no-build-isolation -v --reinstall -p 3.14
 ```
 
-Supported: `3.10`, `3.11`, `3.12`, `3.13`, `3.14`. Do **not** use free-threaded variants (`3.13t`, `3.14t`) — the production client does not support them.
+Supported: `3.11`, `3.12`, `3.13`, `3.14`. Do **not** use free-threaded variants (`3.13t`, `3.14t`) — the production client does not support them.
 
 ### Build configuration reference
 
@@ -188,8 +188,11 @@ uv run ruff format src/ tests/
 # Type checking (mypy — strict mode, see [tool.mypy] in pyproject.toml)
 uv run mypy
 
-# Pre-commit hooks (configured in .pre-commit-config.yaml)
-uvx pre-commit run --all-files
+# Pre-commit hooks (configured in .pre-commit-config.yaml). Install pinned to 3.12
+# (cmakelang crashes on 3.14; keeps hooks off the build interpreter):
+uv tool install --python 3.12 pre-commit
+pre-commit install          # git hook, runs on commit
+pre-commit run --all-files  # run across the tree
 ```
 
 ## Debugging

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.29)
 
-project(duckdb_py LANGUAGES CXX)
+project(duckdb_python LANGUAGES CXX)
 
 # Always use C++17
 set(CMAKE_CXX_STANDARD 17)
@@ -35,8 +35,26 @@ endif()
 # ────────────────────────────────────────────
 # Dependencies
 # ────────────────────────────────────────────
-# PyBind11
-find_package(pybind11 REQUIRED CONFIG)
+# nanobind (requires Python to be located first; pybind11 used to do this
+# internally)
+find_package(
+  Python
+  COMPONENTS Interpreter Development.Module NumPy
+  REQUIRED)
+# Nanobind ships its CMake config inside site-packages/nanobind/cmake, so
+# find_package() can't discover it unless we set it. (scikit-build-core does
+# this as well)
+if(NOT nanobind_ROOT)
+  execute_process(
+    COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    OUTPUT_VARIABLE nanobind_ROOT)
+endif()
+find_package(nanobind CONFIG REQUIRED)
+# Build nanobind's core support library up front so the object libraries below
+# (which include nanobind headers via the umbrella) compile against its include
+# dirs + Python headers + flags.
+nanobind_build_library(nanobind-static)
 
 # DuckDB
 include(cmake/duckdb_loader.cmake)
@@ -49,26 +67,42 @@ duckdb_add_library(duckdb_target)
 
 # Bundle in INTERFACE library
 add_library(_duckdb_dependencies INTERFACE)
-target_link_libraries(_duckdb_dependencies INTERFACE pybind11::pybind11
+target_link_libraries(_duckdb_dependencies INTERFACE nanobind-static
                                                      duckdb_target)
 # Also add include directory
 target_include_directories(
   _duckdb_dependencies
-  INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/duckdb_py/include>
-)
+  INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/include>)
 
 # We link duckdb_static. Without this define, duckdb.h marks C API symbols
 # __declspec(dllimport) on Windows, producing unresolvable __imp_* references at
 # link time. No-op on non-Windows.
 target_compile_definitions(_duckdb_dependencies INTERFACE DUCKDB_STATIC_BUILD)
 
+# Optional AddressSanitizer instrumentation of the Python binding objects ONLY.
+# Every binding object library consumes _duckdb_dependencies (for headers) and
+# _duckdb links it, so adding the flag here instruments the bindings and links
+# the ASAN runtime, while the engine target (duckdb_target, which does NOT
+# consume this) stays uninstrumented and keeps hitting the sccache cache. ASAN's
+# allocator is process-global, so heap errors involving the instrumented binding
+# code are still caught. OFF by default; enable with -DDUCKDB_PY_ASAN=ON.
+option(DUCKDB_PY_ASAN
+       "Instrument the Python binding objects with AddressSanitizer" OFF)
+if(DUCKDB_PY_ASAN)
+  target_compile_options(
+    _duckdb_dependencies INTERFACE -fsanitize=address -fno-omit-frame-pointer
+                                   -g)
+  target_link_options(_duckdb_dependencies INTERFACE -fsanitize=address)
+endif()
+
 # ────────────────────────────────────────────
 # Descend into the real DuckDB‑Python sources
 # ────────────────────────────────────────────
-add_subdirectory(src/duckdb_py)
+add_subdirectory(src)
 
-pybind11_add_module(
+nanobind_add_module(
   _duckdb
+  NB_STATIC
   $<TARGET_OBJECTS:python_src>
   $<TARGET_OBJECTS:python_arrow>
   $<TARGET_OBJECTS:python_common>
@@ -77,7 +111,6 @@ pybind11_add_module(
   $<TARGET_OBJECTS:python_native>
   $<TARGET_OBJECTS:python_numpy>
   $<TARGET_OBJECTS:python_pandas>
-  $<TARGET_OBJECTS:python_pybind11>
   $<TARGET_OBJECTS:python_connection>
   $<TARGET_OBJECTS:python_expression>
   $<TARGET_OBJECTS:python_relation>

CONTRIBUTING.md

@@ -4,6 +4,18 @@
 
 See the [instructions on duckdb.org](https://duckdb.org/docs/stable/dev/building/python).
 
+### Pre-commit hooks
+
+Formatting and linting run through [pre-commit](https://pre-commit.com). Install it pinned to Python 3.12 (the `cmake-format` hook's `cmakelang` dependency crashes on 3.14) so the hooks stay independent of your build interpreter, which may be 3.13 or 3.14t:
+
+```bash
+uv tool install --python 3.12 pre-commit
+pre-commit install          # git hook, runs on `git commit`
+pre-commit run --all-files  # run across the tree
+```
+
+The same checks run in CI.
+
 ## General Guidelines
 
 ### **Did you find a bug?**

duckdb/experimental/spark/_typing.py

@@ -19,9 +19,10 @@
 from collections.abc import Callable, Iterable, Sized
 from typing import Literal, TypeVar
 
-from numpy import float32, float64, int32, int64, ndarray
 from typing_extensions import Protocol, Self
 
+from numpy import float32, float64, int32, int64, ndarray
+
 F = TypeVar("F", bound=Callable)
 T_co = TypeVar("T_co", covariant=True)
 

external/duckdb

@@ -10,7 +10,7 @@ dynamic = ["version"]
 description = "DuckDB in-process database"
 readme = "README.md"
 keywords = ["DuckDB", "Database", "SQL", "OLAP"]
-requires-python = ">=3.10.0"
+requires-python = ">=3.11.0"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "License :: OSI Approved :: MIT License",
@@ -25,7 +25,6 @@ classifiers = [
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
@@ -63,8 +62,12 @@ build-backend = "duckdb_packaging.build_backend"
 backend-path = ["./"]
 requires = [
     "scikit-build-core>=0.11.4",
-	"pybind11[global]>=2.6.0",
+	"nanobind>=2.0",
     "setuptools_scm>=8.0",
+    # numpy C API headers (PyArray_Empty in the result path). Building against numpy 2.x yields a
+    # binary compatible with numpy >=1.19 AND 2.x at runtime (numpy 2.0 backward-compat), so the
+    # unpinned runtime numpy range is preserved. Build-time only; the runtime numpy dep is unchanged.
+    "numpy>=2.0",
 ]
 
 [tool.scikit-build]
@@ -246,6 +249,7 @@ test = [ # dependencies used for running tests
     "pytest-reraise",
     "pytest-timeout",
     "pytest-timestamper",
+    "pytest-xdist", # parallel test execution (-n auto); without this `uv sync --reinstall` prunes a manual install
     "coverage",
     "gcovr; sys_platform != 'win32' or platform_machine != 'ARM64'",
     "gcsfs; sys_platform != 'win32' or platform_machine != 'ARM64'",
@@ -294,7 +298,7 @@ pypi = [ # dependencies used by the pypi cleanup script
 build = [
     "cmake>=3.29.0",
     "ninja>=1.10",
-    "pybind11[global]>=2.6.0",
+    "nanobind>=2.0",
     "scikit_build_core>=0.11.4",
     "setuptools_scm>=8.0",
 ]
@@ -474,6 +478,11 @@ before-build = ["yum install -y ccache"]
 
 [tool.cibuildwheel.macos]
 before-build = ["brew install ccache"]
+# nanobind uses C++17 aligned new/delete (std::align_val_t), which the runtime only provides on macOS 10.13+.
+# cp311's framework defaults to a 10.9 deployment target (used for the x86_64 slice of x86_64/universal2
+# wheels), so nanobind fails to compile there; cp312+ frameworks already target 10.13+. Pin 10.14 so every CPython
+# version builds (arm64 slices are 11.0 regardless).
+environment = { MACOSX_DEPLOYMENT_TARGET = "10.14" }
 
 [tool.cibuildwheel.windows]
 before-build = ["choco install ccache"]

pyproject.toml

@@ -1,7 +1,6 @@
 # this is used for clang-tidy checks
 add_subdirectory(pyrelation)
 add_subdirectory(pyexpression)
-add_subdirectory(pybind11)
 add_subdirectory(numpy)
 add_subdirectory(native)
 add_subdirectory(jupyter)
@@ -25,6 +24,7 @@ add_library(
   pyrelation.cpp
   pyresult.cpp
   pystatement.cpp
+  pyutil.cpp
   python_dependency.cpp
   python_import_cache.cpp
   python_replacement_scan.cpp