Merge with v1.5-andium (#351)

Author: evertlammerts

.github/actions/ccache-action/action.yml

@@ -0,0 +1,23 @@
+name: 'Setup Ccache'
+inputs:
+  key:
+    description: 'Cache key (defaults to github.job)'
+    required: false
+    default: ''
+runs:
+  using: "composite"
+  steps:
+     - name: Setup Ccache
+       uses: hendrikmuhs/ccache-action@main
+       with:
+         key: ${{ inputs.key || github.job }}
+         save: ${{ github.repository != 'duckdb/duckdb-python' || contains('["refs/heads/main", "refs/heads/v1.4-andium", "refs/heads/v1.5-variegata"]', github.ref) }}
+         # Dump verbose ccache statistics report at end of CI job.
+         verbose: 1
+         # Increase per-directory limit: 5*1024 MB / 16 = 320 MB.
+         # Note: `layout=subdirs` computes the size limit divided by 16 dirs.
+         # See also: https://ccache.dev/manual/4.9.html#_cache_size_management
+         max-size: 1500MB
+         # Evicts all cache files that were not touched during the job run.
+         # Removing cache files from previous runs avoids creating huge caches.
+         evict-old-files: 'job'

.github/workflows/code_quality.yml

@@ -32,7 +32,7 @@ jobs:
         uses: astral-sh/setup-uv@v7
         with:
           version: "0.9.0"
-          python-version: 3.9
+          python-version: "3.12"
 
       - name: pre-commit (cache)
         uses: actions/cache@v4

.github/workflows/coverage.yml

@@ -71,7 +71,7 @@ jobs:
         uses: astral-sh/setup-uv@v7
         with:
           version: "0.9.0"
-          python-version: 3.9
+          python-version: 3.12
           enable-cache: true
           cache-suffix: -${{ github.workflow }}
 
@@ -111,6 +111,7 @@ jobs:
           mkdir coverage-cpp
           uv run gcovr \
             --gcov-ignore-errors all \
+            --gcov-ignore-parse-errors=negative_hits.warn_once_per_file \
             --root "$PWD" \
             --filter "${PWD}/src/duckdb_py" \
             --exclude '.*/\.cache/.*' \

.github/workflows/packaging_sdist.yml

@@ -59,7 +59,7 @@ jobs:
         uses: astral-sh/setup-uv@v7
         with:
           version: "0.9.0"
-          python-version: 3.11
+          python-version: 3.12
 
       - name: Build sdist
         run: uv build --sdist

.github/workflows/packaging_wheels.yml

@@ -25,15 +25,112 @@ on:
         type: string
 
 jobs:
+  seed_wheels:
+    name: 'Seed: cp314-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
+    strategy:
+      fail-fast: false
+      matrix:
+        python: [ cp314 ]
+        platform:
+          - { os: windows-2025,     arch: amd64,      cibw_system: win }
+          - { os: windows-11-arm,   arch: ARM64,      cibw_system: win }
+          - { os: ubuntu-24.04,     arch: x86_64,     cibw_system: manylinux }
+          - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }
+          - { os: macos-15,         arch: arm64,      cibw_system: macosx }
+          - { os: macos-15,         arch: universal2, cibw_system: macosx }
+          - { os: macos-15-intel,   arch: x86_64,     cibw_system: macosx }
+        minimal:
+          - ${{ inputs.minimal }}
+        exclude:
+          - { minimal: true, platform: { arch: universal2 } }
+    runs-on: ${{ matrix.platform.os }}
+    env:
+      CCACHE_DIR: ${{ github.workspace }}/.ccache
+      ### cibuildwheel configuration
+      #
+      # This is somewhat brittle, so be careful with changes. Some notes for our future selves (and others):
+      # - cibw will change its cwd to a temp dir and create a separate venv for testing. It then installs the wheel it
+      #   built into that venv, and run the CIBW_TEST_COMMAND. We have to install all dependencies ourselves, and make
+      #   sure that the pytest config in pyproject.toml is available.
+      # - CIBW_BEFORE_TEST installs the test dependencies by exporting them into a pylock.toml. At the time of writing,
+      #   `uv sync --no-install-project` had problems correctly resolving dependencies using resolution environments
+      #   across all platforms we build for. This might be solved in newer uv versions.
+      # - CIBW_TEST_COMMAND specifies pytest conf from pyproject.toml. --confcutdir is needed to prevent pytest from
+      #   traversing the full filesystem, which produces an error on Windows.
+      # - CIBW_TEST_SKIP we always skip tests for *-macosx_universal2 builds, because we run tests for arm64 and x86_64.
+      CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }}
+      CIBW_TEST_SOURCES: tests
+      CIBW_BEFORE_TEST: >
+        uv export --only-group test --no-emit-project  --quiet --output-file pylock.toml --directory {project} &&
+        uv pip install -r pylock.toml
+      CIBW_TEST_COMMAND: >
+        uv run -v pytest --confcutdir=. --rootdir . -c {project}/pyproject.toml ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }}
+
+    steps:
+      - name: Checkout DuckDB Python
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.duckdb-python-sha }}
+          fetch-depth: 0
+          submodules: true
+
+      - name: Checkout DuckDB
+        shell: bash
+        if: ${{ inputs.duckdb-sha }}
+        run: |
+          cd external/duckdb
+          git fetch origin
+          git checkout ${{ inputs.duckdb-sha }}
+
+      - name: Set CIBW_ENVIRONMENT
+        shell: bash
+        run: |
+          cibw_env=""
+          if [[ "${{ matrix.platform.cibw_system }}" == "manylinux" ]]; then
+            cibw_env="CCACHE_DIR=/host${{ github.workspace }}/.ccache"
+          fi
+          if [[ -n "${{ inputs.set-version }}" ]]; then
+            cibw_env="${cibw_env:+$cibw_env }OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}"
+          fi
+          if [[ -n "$cibw_env" ]]; then
+            echo "CIBW_ENVIRONMENT=${cibw_env}" >> $GITHUB_ENV
+          fi
+
+      - name: Setup Ccache
+        uses: ./.github/actions/ccache-action
+        with:
+          key: ${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
+
+      # Install Astral UV, which will be used as build-frontend for cibuildwheel
+      - uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.9.0"
+          enable-cache: false
+          cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
+
+      - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels
+        uses: pypa/cibuildwheel@v3.2
+        env:
+          CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }}
+          CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
+
+      - name: Upload wheel
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
+          path: wheelhouse/*.whl
+          compression-level: 0
+
   build_wheels:
     name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
+    needs: seed_wheels
     strategy:
       fail-fast: false
       matrix:
-        python: [ cp39, cp310, cp311, cp312, cp313, cp314 ]
+        python: [ cp310, cp311, cp312, cp313 ]
         platform:
           - { os: windows-2025,     arch: amd64,      cibw_system: win }
-          - { os: windows-11-arm,   arch: ARM64,      cibw_system: win } # cibw requires ARM64 to be uppercase
+          - { os: windows-11-arm,   arch: ARM64,      cibw_system: win }
           - { os: ubuntu-24.04,     arch: x86_64,     cibw_system: manylinux }
           - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }
           - { os: macos-15,         arch: arm64,      cibw_system: macosx }
@@ -42,15 +139,14 @@ jobs:
         minimal:
           - ${{ inputs.minimal }}
         exclude:
-          - { minimal: true, python: cp310 }
           - { minimal: true, python: cp311 }
           - { minimal: true, python: cp312 }
           - { minimal: true, python: cp313 }
           - { minimal: true, platform: { arch: universal2 } }
-          - { python: cp39,  platform: { os: windows-11-arm, arch: ARM64 } }  # too many dependency problems for win arm64
-          - { python: cp310,  platform: { os: windows-11-arm, arch: ARM64 } }  # too many dependency problems for win arm64
+          - { python: cp310,  platform: { os: windows-11-arm, arch: ARM64 } }
     runs-on: ${{ matrix.platform.os }}
     env:
+      CCACHE_DIR: ${{ github.workspace }}/.ccache
       ### cibuildwheel configuration
       #
       # This is somewhat brittle, so be careful with changes. Some notes for our future selves (and others):
@@ -87,11 +183,24 @@ jobs:
           git fetch origin
           git checkout ${{ inputs.duckdb-sha }}
 
-      # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds
-      - name: Set OVERRIDE_GIT_DESCRIBE
+      - name: Set CIBW_ENVIRONMENT
         shell: bash
-        if: ${{ inputs.set-version != '' }}
-        run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
+        run: |
+          cibw_env=""
+          if [[ "${{ matrix.platform.cibw_system }}" == "manylinux" ]]; then
+            cibw_env="CCACHE_DIR=/host${{ github.workspace }}/.ccache"
+          fi
+          if [[ -n "${{ inputs.set-version }}" ]]; then
+            cibw_env="${cibw_env:+$cibw_env }OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}"
+          fi
+          if [[ -n "$cibw_env" ]]; then
+            echo "CIBW_ENVIRONMENT=${cibw_env}" >> $GITHUB_ENV
+          fi
+
+      - name: Setup Ccache
+        uses: ./.github/actions/ccache-action
+        with:
+          key: ${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
 
       # Install Astral UV, which will be used as build-frontend for cibuildwheel
       - uses: astral-sh/setup-uv@v7

.github/workflows/release.yml

@@ -39,7 +39,6 @@ defaults:
 jobs:
   build_sdist:
     name: Build an sdist and determine versions
-    if: ${{ github.ref != 'refs/heads/main' }}
     uses: ./.github/workflows/packaging_sdist.yml
     with:
       testsuite: all

.github/workflows/targeted_test.yml

@@ -19,7 +19,6 @@ on:
         required: true
         type: choice
         options:
-          - '3.9'
           - '3.10'
           - '3.11'
           - '3.12'

.gitignore

@@ -14,6 +14,8 @@
 .sw?
 #OS X specific files.
 .DS_store
+#VSCode specifics
+.vscode/
 
 #==============================================================================#
 # Build artifacts
@@ -45,6 +47,7 @@ cmake-build-release
 cmake-build-relwithdebinfo
 duckdb_packaging/duckdb_version.txt
 test.db
+tmp/
 
 #==============================================================================#
 # Python

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2018-2025 Stichting DuckDB Foundation
+Copyright 2018-2026 Stichting DuckDB Foundation
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: