diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index a2e4f857..8e656abd 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -37,7 +37,7 @@ jobs: - { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } - { os: macos-15, arch: arm64, cibw_system: macosx } - { os: macos-15, arch: universal2, cibw_system: macosx } - - { os: macos-13, arch: x86_64, cibw_system: macosx } + - { os: macos-15-intel, arch: x86_64, cibw_system: macosx } minimal: - ${{ inputs.minimal }} exclude: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f54b0f76..727f8027 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,6 +22,11 @@ on: options: - test - prod + nightly-stale-after-days: + type: string + description: After how many days should nightlies be considered stale + required: true + default: 3 store-s3: type: boolean description: Also store test packages in S3 (always true for prod) @@ -41,6 +46,17 @@ jobs: duckdb-sha: ${{ inputs.duckdb-sha }} set-version: ${{ inputs.stable-version }} + submodule_pr: + name: Create or update PR to bump submodule to given SHA + needs: build_sdist + uses: ./.github/workflows/submodule_auto_pr.yml + with: + duckdb-python-sha: ${{ inputs.duckdb-python-sha }} + duckdb-sha: ${{ inputs.duckdb-sha }} + secrets: + # reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206 + DUCKDBLABS_BOT_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }} + workflow_state: name: Set state for the release workflow needs: build_sdist @@ -51,23 +67,36 @@ jobs: runs-on: ubuntu-latest steps: - id: index_check - name: Check ${{ needs.build_sdist.outputs.package-version }} on PyPI + name: Check version on PyPI run: | - set -eu - # Check PyPI whether the release we're building is already present + set -ex pypi_hostname=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org - pkg_version=${{ needs.build_sdist.outputs.package-version }} - url=https://${pypi_hostname}/pypi/duckdb/${pkg_version}/json - http_status=$( curl -s -o /dev/null -w "%{http_code}" $url || echo $? ) - if [[ $http_status == "200" ]]; then - echo "::warning::Package version ${pkg_version} is already present on ${pypi_hostname}" - pypi_state=VERSION_FOUND - elif [[ $http_status == 000* ]]; then - echo "::error::Error checking PyPI at ${url}: curl exit code ${http_status#'000'}" - pypi_state=UNKNOWN - else - echo "::notice::Package version ${pkg_version} not found on ${pypi_hostname} (http status: ${http_status})" + # install duckdb + curl https://install.duckdb.org | sh + # query pypi + result=$(cat <>'upload_time_iso_8601')::DATE AS age, + FROM read_json('https://${pypi_hostname}/pypi/duckdb/json') AS jd + CROSS JOIN json_each(jd.releases) AS rel(key, value) + CROSS JOIN unnest(FROM_JSON(rel.value, '["JSON"]')) AS file(value) + WHERE rel.key='${{ needs.build_sdist.outputs.package-version }}' + LIMIT 1; + EOF + ) + if [ -z "$result" ]; then pypi_state=VERSION_NOT_FOUND + else + pypi_state=VERSION_FOUND + fi + if [[ -z "${{ inputs.stable-version }}" ]]; then + age=${result#age = } + if [ "${age}" -ge "${{ inputs.nightly-stale-after-days }}" ]; then + echo "::warning title=Stale nightly for ${{ github.ref_name }}::Nightly is ${age} days old (max=${{ inputs.nightly-stale-after-days }})" + fi fi echo "pypi_state=${pypi_state}" >> $GITHUB_OUTPUT @@ -96,7 +125,7 @@ jobs: echo "::notice::S3 upload disabled in inputs, not generating S3 URL" exit 0 fi - if [[ VERSION_FOUND == "${{ steps.index_check.outputs.pypi_state }}" ]]; then + if [[ VERSION_NOT_FOUND != "${{ steps.index_check.outputs.pypi_state }}" ]]; then echo "::warning::S3 upload disabled because package version already uploaded to PyPI" exit 0 fi @@ -110,7 +139,7 @@ jobs: build_wheels: name: Build and test releases needs: workflow_state - if: ${{ needs.workflow_state.outputs.pypi_state != 'VERSION_FOUND' }} + if: ${{ needs.workflow_state.outputs.pypi_state == 'VERSION_NOT_FOUND' }} uses: ./.github/workflows/packaging_wheels.yml with: minimal: false @@ -132,14 +161,12 @@ jobs: path: artifacts/ merge-multiple: true - - name: Authenticate with AWS - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: 'us-east-2' - aws-access-key-id: ${{ secrets.S3_DUCKDB_STAGING_ID }} - aws-secret-access-key: ${{ secrets.S3_DUCKDB_STAGING_KEY }} - - name: Upload Artifacts + env: + AWS_ENDPOINT_URL: ${{ secrets.S3_DUCKDB_STAGING_ENDPOINT }} + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} + run: | aws s3 cp artifacts ${{ needs.workflow_state.outputs.s3_url }} --recursive diff --git a/.github/workflows/submodule_auto_pr.yml b/.github/workflows/submodule_auto_pr.yml new file mode 100644 index 00000000..43a9860a --- /dev/null +++ b/.github/workflows/submodule_auto_pr.yml @@ -0,0 +1,130 @@ +name: Submodule Auto PR +on: + workflow_call: + inputs: + duckdb-python-sha: + type: string + description: The commit to build against (defaults to latest commit of current ref) + required: false + duckdb-sha: + type: string + description: The DuckDB submodule commit or ref to build against + required: true + auto-land: + type: boolean + description: Immediately merge the PR (placeholder - doesn't work) + default: false + secrets: + DUCKDBLABS_BOT_TOKEN: + description: Github token of the DuckDBLabs bot + required: true + +defaults: + run: + shell: bash + +jobs: + create_pr: + name: Create PR to bump duckdb submodule to given SHA + runs-on: ubuntu-latest + steps: + - name: Checkout DuckDB Python + uses: actions/checkout@v4 + with: + ref: ${{ inputs.duckdb-python-sha }} + fetch-depth: 0 + submodules: true + + - name: Checkout or Create Needed Branch + run: | + git fetch --all + head_sha=${{ inputs.duckdb-python-sha }} + branch_name="vendoring-${{ github.ref_name }}" + if [[ `git rev-parse --verify ${branch_name} 2>/dev/null` ]]; then + # branch exists + git checkout ${branch_name} + else + # new branch + git checkout -b ${branch_name} + fi + [[ ${head_sha} ]] && git reset --hard ${head_sha} || true + + - name: Checkout DuckDB at Given SHA + run: | + cd external/duckdb + git fetch origin + git checkout ${{ inputs.duckdb-sha }} + + - name: Determine GH PR Command + id: gh_pr_command + env: + GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }} + run: | + pr_url=$( gh pr list --head vendoring-${{ github.ref_name }} --state open --json url --jq '.[].url' ) + if [[ $pr_url ]]; then + echo "::notice::Found existing pr, will edit (${pr_url})" + gh_command="edit ${pr_url}" + else + echo "::notice::No existing PR, will create new" + gh_command="create --head vendoring-${{ github.ref_name }} --base ${{ github.ref_name }}" + fi + echo "subcommand=${gh_command}" >> $GITHUB_OUTPUT + + - name: Set Git User + run: | + git config --global user.email "github_bot@duckdblabs.com" + git config --global user.name "DuckDB Labs GitHub Bot" + + - name: Create PR to Bump DuckDB Submodule + env: + GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }} + run: | + # No need to do anything if the submodule is already at the given sha + [[ `git status --porcelain -- external/duckdb` == "" ]] && exit 0 + # We have changes. Commit and push + git add external/duckdb + git commit -m "Bump submodule" + git push --force origin vendoring-${{ github.ref_name }} + # create PR msg + echo "Bump duckdb submodule:" > body.txt + echo "- Target branch: ${{ github.ref_name }}" >> body.txt + echo "- Date: $( date +"%Y-%m-%d %H:%M:%S" )" >> body.txt + echo "- DuckDB SHA: ${{ inputs.duckdb-sha }}" >> body.txt + echo "- Trigger: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> body.txt + subcommand="${{ steps.gh_pr_command.outputs.subcommand }}" + gh pr ${subcommand} \ + --title "[duckdb-labs bot] Bump DuckDB submodule" \ + --body-file body.txt > output.txt 2>&1 + success=$? + # Show summary + url=$( [[ $success ]] && gh pr view vendoring-${{ github.ref_name }} --json url --jq .url || true ) + echo "## Submodule PR Summary" >> $GITHUB_STEP_SUMMARY + if [[ $success ]]; then + prefix=$( [[ $subcommand == edit* ]] && echo "Created" || echo "Updated" ) + echo "### ${prefix} PR: [${url}](${url})" >> $GITHUB_STEP_SUMMARY + else + echo "### Failed to create PR" >> $GITHUB_STEP_SUMMARY + fi + echo '```' >> $GITHUB_STEP_SUMMARY + cat output.txt >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + [[ $success ]] || exit 1 + + - name: Automerge PR + if: ${{ inputs.auto-land }} + env: + GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }} + run: | + # PLACEHOLDER: DUCKDBLABS_BOT_TOKEN DOES NOT HAVE PERMISSIONS TO MERGE PRS + set -ex + gh pr merge vendoring-${{ github.ref_name }} --rebase > output.txt + success=$? + # Show summary + if [[ $success ]]; then + echo "### PR merged" >> $GITHUB_STEP_SUMMARY + else + echo "### Failed to auto-merge PR" >> $GITHUB_STEP_SUMMARY + fi + echo '```' >> $GITHUB_STEP_SUMMARY + cat output.txt >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/targeted_test.yml b/.github/workflows/targeted_test.yml new file mode 100644 index 00000000..812bb9c5 --- /dev/null +++ b/.github/workflows/targeted_test.yml @@ -0,0 +1,86 @@ +name: Targeted Platform Testing + +on: + workflow_dispatch: + inputs: + platform: + description: 'Platform to test on' + required: true + type: choice + options: + - 'windows-2025' + - 'ubuntu-24.04' + - 'ubuntu-24.04-arm' + - 'macos-15' + - 'macos-15-intel' + python_version: + description: 'Python version to test' + required: true + type: choice + options: + - '3.9' + - '3.10' + - '3.11' + - '3.12' + - '3.13' + - '3.14' + testsuite: + description: 'Test suite to run (ignored if custom_test_path is provided)' + required: false + type: choice + options: + - 'fast' + - 'all' + default: 'fast' + custom_test_path: + description: 'Custom test path (must be in tests/ directory, overrides testsuite)' + required: false + type: string + +jobs: + test: + name: 'Test with Python ${{ inputs.python_version }} on ${{ inputs.platform }}' + runs-on: ${{ inputs.platform }} + + steps: + - name: Checkout DuckDB Python + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: true + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.9.0" + enable-cache: false + python-version: ${{ inputs.python_version }} + + - name: Set and validate test path + id: test_path + shell: bash + run: | + if [[ -n "${{ inputs.custom_test_path }}" ]]; then + # test path was passed in + tests_base="$( pwd -P )/tests" + test_path="${{ inputs.custom_test_path }}" + + # Ensure the given test path exists + [[ -e "$test_path" ]] || { echo "${test_path} does not exist"; exit 1; } + + # Resolve custom test path to absolute path + test_path_abs=$(cd "$test_path" 2>/dev/null && pwd -P || ( cd "$(dirname "$test_path")" && printf '%s/%s' "$(pwd -P)" "$(basename "$test_path")" ) ) + + # Make sure test_path_abs is inside tests_base + [[ "$test_path_abs" == "$tests_base" || "$test_path_abs" == "$tests_base"/* ]] || { echo "${test_path_abs} is not part of ${tests_base}?"; exit 1; } + + echo "test_path=$test_path_abs" >> $GITHUB_OUTPUT + else + # use a testsuite + echo "test_path=$GITHUB_WORKSPACE/${{ inputs.testsuite == 'fast' && 'tests/fast' || 'tests' }}" >> $GITHUB_OUTPUT + fi + + - name: Run tests + shell: bash + run: | + uv run pytest -vv ${{ steps.test_path.outputs.test_path }} diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index d4f4b61b..00000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,19 +0,0 @@ -# Changelog - -## v1.4.1 -**DuckDB Core**: v1.4.1 - -### Bug Fixes -- **ADBC Driver**: Fixed ADBC driver implementation (#81) -- **SQLAlchemy compatibility**: Added `__hash__` method overload (#61) -- **Error Handling**: Reset PyErr before throwing Python exceptions (#69) -- **Polars Lazyframes**: Fixed Polars expression pushdown (#102) - -### Code Quality Improvements & Developer Experience -- **MyPy Support**: MyPy is functional again and better integrated with the dev workflow -- **Stubs**: Re-created and manually curated stubs for the binary extension -- **Type Shadowing**: Deprecated `typing` and `functional` modules -- **Linting & Formatting**: Comprehensive code quality improvements with Ruff -- **Type Annotations**: Added missing overloads and improved type coverage -- **Pre-commit Integration**: Added ruff, clang-format, cmake-format and mypy configs -- **CI/CD**: Added code quality workflow diff --git a/_duckdb-stubs/__init__.pyi b/_duckdb-stubs/__init__.pyi index 6c36d7be..6b323184 100644 --- a/_duckdb-stubs/__init__.pyi +++ b/_duckdb-stubs/__init__.pyi @@ -318,7 +318,18 @@ class DuckDBPyConnection: def list_type(self, type: sqltypes.DuckDBPyType) -> sqltypes.DuckDBPyType: ... def load_extension(self, extension: str) -> None: ... def map_type(self, key: sqltypes.DuckDBPyType, value: sqltypes.DuckDBPyType) -> sqltypes.DuckDBPyType: ... - def pl(self, rows_per_batch: pytyping.SupportsInt = 1000000, *, lazy: bool = False) -> polars.DataFrame: ... + @pytyping.overload + def pl( + self, rows_per_batch: pytyping.SupportsInt = 1000000, *, lazy: pytyping.Literal[False] = ... + ) -> polars.DataFrame: ... + @pytyping.overload + def pl( + self, rows_per_batch: pytyping.SupportsInt = 1000000, *, lazy: pytyping.Literal[True] + ) -> polars.LazyFrame: ... + @pytyping.overload + def pl( + self, rows_per_batch: pytyping.SupportsInt = 1000000, *, lazy: bool = False + ) -> pytyping.Union[polars.DataFrame, polars.LazyFrame]: ... def query(self, query: str, *, alias: str = "", params: object = None) -> DuckDBPyRelation: ... def query_progress(self) -> float: ... def read_csv( @@ -596,7 +607,16 @@ class DuckDBPyRelation: ) -> DuckDBPyRelation: ... def order(self, order_expr: str) -> DuckDBPyRelation: ... def percent_rank(self, window_spec: str, projected_columns: str = "") -> DuckDBPyRelation: ... - def pl(self, batch_size: pytyping.SupportsInt = 1000000, *, lazy: bool = False) -> polars.DataFrame: ... + @pytyping.overload + def pl( + self, batch_size: pytyping.SupportsInt = 1000000, *, lazy: pytyping.Literal[False] = ... + ) -> polars.DataFrame: ... + @pytyping.overload + def pl(self, batch_size: pytyping.SupportsInt = 1000000, *, lazy: pytyping.Literal[True]) -> polars.LazyFrame: ... + @pytyping.overload + def pl( + self, batch_size: pytyping.SupportsInt = 1000000, *, lazy: bool = False + ) -> pytyping.Union[polars.DataFrame, polars.LazyFrame]: ... def product( self, column: str, groups: str = "", window_spec: str = "", projected_columns: str = "" ) -> DuckDBPyRelation: ... @@ -700,6 +720,7 @@ class DuckDBPyRelation: partition_by: pytyping.List[str] | None = None, write_partition_columns: bool | None = None, append: bool | None = None, + filename_pattern: str | None = None, ) -> None: ... def to_table(self, table_name: str) -> None: ... def to_view(self, view_name: str, replace: bool = True) -> DuckDBPyRelation: ... @@ -752,6 +773,7 @@ class DuckDBPyRelation: partition_by: pytyping.List[str] | None = None, write_partition_columns: bool | None = None, append: bool | None = None, + filename_pattern: str | None = None, ) -> None: ... @property def alias(self) -> str: ... @@ -1048,7 +1070,7 @@ def commit(*, connection: DuckDBPyConnection | None = None) -> DuckDBPyConnectio def connect( database: str | pathlib.Path = ":memory:", read_only: bool = False, - config: dict[str, str] | None = None, + config: dict[str, str | bool | int | float | list[str]] | None = None, ) -> DuckDBPyConnection: ... def create_function( name: str, @@ -1241,12 +1263,27 @@ def map_type( def order( df: pandas.DataFrame, order_expr: str, *, connection: DuckDBPyConnection | None = None ) -> DuckDBPyRelation: ... +@pytyping.overload def pl( rows_per_batch: pytyping.SupportsInt = 1000000, *, - lazy: bool = False, + lazy: pytyping.Literal[False] = ..., connection: DuckDBPyConnection | None = None, ) -> polars.DataFrame: ... +@pytyping.overload +def pl( + rows_per_batch: pytyping.SupportsInt = 1000000, + *, + lazy: pytyping.Literal[True], + connection: DuckDBPyConnection | None = None, +) -> polars.LazyFrame: ... +@pytyping.overload +def pl( + rows_per_batch: pytyping.SupportsInt = 1000000, + *, + lazy: bool = False, + connection: DuckDBPyConnection | None = None, +) -> pytyping.Union[polars.DataFrame, polars.LazyFrame]: ... def project( df: pandas.DataFrame, *args: str | Expression, groups: str = "", connection: DuckDBPyConnection | None = None ) -> DuckDBPyRelation: ... diff --git a/adbc_driver_duckdb/__init__.py b/adbc_driver_duckdb/__init__.py index e81f5090..f925ea9e 100644 --- a/adbc_driver_duckdb/__init__.py +++ b/adbc_driver_duckdb/__init__.py @@ -19,7 +19,7 @@ import enum import functools -import importlib +import importlib.util import typing import adbc_driver_manager @@ -46,5 +46,4 @@ def driver_path() -> str: if duckdb_module_spec is None: msg = "Could not find duckdb shared library. Did you pip install duckdb?" raise ImportError(msg) - print(f"Found duckdb shared library at {duckdb_module_spec.origin}") return duckdb_module_spec.origin diff --git a/duckdb/experimental/__init__.py b/duckdb/experimental/__init__.py index 1b5ee51b..51d08709 100644 --- a/duckdb/experimental/__init__.py +++ b/duckdb/experimental/__init__.py @@ -1,3 +1,5 @@ from . import spark # noqa: D104 -__all__ = spark.__all__ +__all__ = [ + "spark", +] diff --git a/external/duckdb b/external/duckdb index abd077cd..36451fcf 160000 --- a/external/duckdb +++ b/external/duckdb @@ -1 +1 @@ -Subproject commit abd077cd1ee41fcd9417f7f1e61fe9228da02416 +Subproject commit 36451fcf23b4d8ebf2cb52e26eef93f7fb54561b diff --git a/pyproject.toml b/pyproject.toml index 7df13b61..6895b12e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ readme = "README.md" keywords = ["DuckDB", "Database", "SQL", "OLAP"] requires-python = ">=3.9.0" classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Topic :: Database", @@ -40,7 +40,7 @@ maintainers = [{name = "DuckDB Foundation"}] Documentation = "https://duckdb.org/docs/stable/clients/python/overview" Source = "https://github.com/duckdb/duckdb-python" Issues = "https://github.com/duckdb/duckdb-python/issues" -Changelog = "https://github.com/duckdb/duckdb/releases" +Changelog = "https://github.com/duckdb/duckdb-python/releases" [project.optional-dependencies] all = [ # users can install duckdb with 'duckdb[all]', which will install this list @@ -48,7 +48,7 @@ all = [ # users can install duckdb with 'duckdb[all]', which will install this l "fsspec", # used in duckdb.filesystem "numpy", # used in duckdb.experimental.spark and in duckdb.fetchnumpy() "pandas", # used for pandas dataframes all over the place - "pyarrow; python_version < '3.14'", # used for pyarrow support + "pyarrow", # used for pyarrow support "adbc-driver-manager", # for the adbc driver ] @@ -226,7 +226,7 @@ stubdeps = [ # dependencies used for typehints in the stubs "fsspec", "pandas", "polars", - "pyarrow; python_version < '3.14'", + "pyarrow", ] test = [ # dependencies used for running tests "adbc-driver-manager", @@ -248,7 +248,7 @@ test = [ # dependencies used for running tests "urllib3", "fsspec>=2022.11.0", "pandas>=2.0.0", - "pyarrow>=18.0.0; python_version < '3.14'", + "pyarrow>=18.0.0", "torch>=2.2.2; python_version < '3.14' and ( sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13' )", "tensorflow==2.14.0; sys_platform == 'darwin' and python_version < '3.12'", "tensorflow-cpu>=2.14.0; sys_platform == 'linux' and platform_machine != 'aarch64' and python_version < '3.12'", @@ -265,7 +265,7 @@ scripts = [ # dependencies used for running scripts "pandas", "pcpp", "polars", - "pyarrow; python_version < '3.14'", + "pyarrow", "pytz" ] pypi = [ # dependencies used by the pypi cleanup script @@ -327,6 +327,10 @@ exclude = [ "tests", "scripts", ] +[[tool.mypy.overrides]] +module = "duckdb.experimental.*" +ignore_errors = true + [[tool.mypy.overrides]] module = [ "fsspec.*", diff --git a/src/duckdb_py/include/duckdb_python/pyrelation.hpp b/src/duckdb_py/include/duckdb_python/pyrelation.hpp index e1f78b5a..06cf9e94 100644 --- a/src/duckdb_py/include/duckdb_python/pyrelation.hpp +++ b/src/duckdb_py/include/duckdb_python/pyrelation.hpp @@ -214,7 +214,7 @@ struct DuckDBPyRelation { const py::object &row_group_size = py::none(), const py::object &overwrite = py::none(), const py::object &per_thread_output = py::none(), const py::object &use_tmp_file = py::none(), const py::object &partition_by = py::none(), const py::object &write_partition_columns = py::none(), - const py::object &append = py::none()); + const py::object &append = py::none(), const py::object &filename_pattern = py::none()); void ToCSV(const string &filename, const py::object &sep = py::none(), const py::object &na_rep = py::none(), const py::object &header = py::none(), const py::object "echar = py::none(), @@ -235,7 +235,7 @@ struct DuckDBPyRelation { void InsertInto(const string &table); - void Insert(const py::object ¶ms = py::list()); + void Insert(const py::object ¶ms = py::list()) const; void Update(const py::object &set, const py::object &where = py::none()); void Create(const string &table); diff --git a/src/duckdb_py/include/duckdb_python/pyresult.hpp b/src/duckdb_py/include/duckdb_python/pyresult.hpp index fc3641c4..941a203b 100644 --- a/src/duckdb_py/include/duckdb_python/pyresult.hpp +++ b/src/duckdb_py/include/duckdb_python/pyresult.hpp @@ -66,7 +66,7 @@ struct DuckDBPyResult { PandasDataFrame FrameFromNumpy(bool date_as_object, const py::handle &o); - void ChangeToTZType(PandasDataFrame &df); + void ConvertDateTimeTypes(PandasDataFrame &df, bool date_as_object) const; unique_ptr FetchNext(QueryResult &result); unique_ptr FetchNextRaw(QueryResult &result); unique_ptr InitializeNumpyConversion(bool pandas = false); diff --git a/src/duckdb_py/pyrelation.cpp b/src/duckdb_py/pyrelation.cpp index 3553bff0..bbc7a2ec 100644 --- a/src/duckdb_py/pyrelation.cpp +++ b/src/duckdb_py/pyrelation.cpp @@ -1213,7 +1213,8 @@ void DuckDBPyRelation::ToParquet(const string &filename, const py::object &compr const py::object &row_group_size_bytes, const py::object &row_group_size, const py::object &overwrite, const py::object &per_thread_output, const py::object &use_tmp_file, const py::object &partition_by, - const py::object &write_partition_columns, const py::object &append) { + const py::object &write_partition_columns, const py::object &append, + const py::object &filename_pattern) { case_insensitive_map_t> options; if (!py::none().is(compression)) { @@ -1304,6 +1305,13 @@ void DuckDBPyRelation::ToParquet(const string &filename, const py::object &compr options["use_tmp_file"] = {Value::BOOLEAN(py::bool_(use_tmp_file))}; } + if (!py::none().is(filename_pattern)) { + if (!py::isinstance(filename_pattern)) { + throw InvalidInputException("to_parquet only accepts 'filename_pattern' as a string"); + } + options["filename_pattern"] = {Value(py::str(filename_pattern))}; + } + auto write_parquet = rel->WriteParquetRel(filename, std::move(options)); PyExecuteRelation(write_parquet); } @@ -1511,14 +1519,10 @@ DuckDBPyRelation &DuckDBPyRelation::Execute() { void DuckDBPyRelation::InsertInto(const string &table) { AssertRelation(); auto parsed_info = QualifiedName::Parse(table); - auto insert = rel->InsertRel(parsed_info.schema, parsed_info.name); + auto insert = rel->InsertRel(parsed_info.catalog, parsed_info.schema, parsed_info.name); PyExecuteRelation(insert); } -static bool IsAcceptedInsertRelationType(const Relation &relation) { - return relation.type == RelationType::TABLE_RELATION; -} - void DuckDBPyRelation::Update(const py::object &set_p, const py::object &where) { AssertRelation(); unique_ptr condition; @@ -1563,9 +1567,9 @@ void DuckDBPyRelation::Update(const py::object &set_p, const py::object &where) return rel->Update(std::move(names), std::move(expressions), std::move(condition)); } -void DuckDBPyRelation::Insert(const py::object ¶ms) { +void DuckDBPyRelation::Insert(const py::object ¶ms) const { AssertRelation(); - if (!IsAcceptedInsertRelationType(*this->rel)) { + if (this->rel->type != RelationType::TABLE_RELATION) { throw InvalidInputException("'DuckDBPyRelation.insert' can only be used on a table relation"); } vector> values {DuckDBPyConnection::TransformPythonParamList(params)}; diff --git a/src/duckdb_py/pyrelation/initialize.cpp b/src/duckdb_py/pyrelation/initialize.cpp index cd1f042c..7bfea441 100644 --- a/src/duckdb_py/pyrelation/initialize.cpp +++ b/src/duckdb_py/pyrelation/initialize.cpp @@ -36,7 +36,8 @@ static void InitializeConsumers(py::class_ &m) { py::arg("row_group_size_bytes") = py::none(), py::arg("row_group_size") = py::none(), py::arg("overwrite") = py::none(), py::arg("per_thread_output") = py::none(), py::arg("use_tmp_file") = py::none(), py::arg("partition_by") = py::none(), - py::arg("write_partition_columns") = py::none(), py::arg("append") = py::none()); + py::arg("write_partition_columns") = py::none(), py::arg("append") = py::none(), + py::arg("filename_pattern") = py::none()); DefineMethod( {"to_csv", "write_csv"}, m, &DuckDBPyRelation::ToCSV, "Write the relation object to a CSV file in 'file_name'", diff --git a/src/duckdb_py/pyresult.cpp b/src/duckdb_py/pyresult.cpp index 43edf0e1..e92f6abe 100644 --- a/src/duckdb_py/pyresult.cpp +++ b/src/duckdb_py/pyresult.cpp @@ -287,8 +287,13 @@ py::dict DuckDBPyResult::FetchNumpyInternal(bool stream, idx_t vectors_per_chunk return res; } +static void ReplaceDFColumn(PandasDataFrame &df, const char *col_name, idx_t idx, const py::handle &new_value) { + df.attr("drop")("columns"_a = col_name, "inplace"_a = true); + df.attr("insert")(idx, col_name, new_value, "allow_duplicates"_a = false); +} + // TODO: unify these with an enum/flag to indicate which conversions to do -void DuckDBPyResult::ChangeToTZType(PandasDataFrame &df) { +void DuckDBPyResult::ConvertDateTimeTypes(PandasDataFrame &df, bool date_as_object) const { auto names = df.attr("columns").cast>(); for (idx_t i = 0; i < result->ColumnCount(); i++) { @@ -297,8 +302,10 @@ void DuckDBPyResult::ChangeToTZType(PandasDataFrame &df) { auto utc_local = df[names[i].c_str()].attr("dt").attr("tz_localize")("UTC"); auto new_value = utc_local.attr("dt").attr("tz_convert")(result->client_properties.time_zone); // We need to create the column anew because the exact dt changed to a new timezone - df.attr("drop")("columns"_a = names[i].c_str(), "inplace"_a = true); - df.attr("__setitem__")(names[i].c_str(), new_value); + ReplaceDFColumn(df, names[i].c_str(), i, new_value); + } else if (date_as_object && result->types[i] == LogicalType::DATE) { + auto new_value = df[names[i].c_str()].attr("dt").attr("date"); + ReplaceDFColumn(df, names[i].c_str(), i, new_value); } } } @@ -374,20 +381,11 @@ PandasDataFrame DuckDBPyResult::FrameFromNumpy(bool date_as_object, const py::ha } PandasDataFrame df = py::cast(pandas.attr("DataFrame").attr("from_dict")(o)); - // Unfortunately we have to do a type change here for timezones since these types are not supported by numpy - ChangeToTZType(df); + // Convert TZ and (optionally) Date types + ConvertDateTimeTypes(df, date_as_object); auto names = df.attr("columns").cast>(); D_ASSERT(result->ColumnCount() == names.size()); - if (date_as_object) { - for (idx_t i = 0; i < result->ColumnCount(); i++) { - if (result->types[i] == LogicalType::DATE) { - auto new_value = df[names[i].c_str()].attr("dt").attr("date"); - df.attr("drop")("columns"_a = names[i].c_str(), "inplace"_a = true); - df.attr("__setitem__")(names[i].c_str(), new_value); - } - } - } return df; } diff --git a/tests/conftest.py b/tests/conftest.py index f2ad7cec..bfb458a5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,4 @@ import os -import sys import warnings from importlib import import_module from pathlib import Path @@ -36,47 +35,6 @@ def import_pandas(): pytest.skip("Couldn't import pandas") -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_call(item): - """Convert missing pyarrow imports to skips. - - TODO(evertlammerts): Remove skip when pyarrow releases for 3.14. - https://github.com/duckdblabs/duckdb-internal/issues/6182 - """ - outcome = yield - if sys.version_info[:2] == (3, 14): - try: - outcome.get_result() - except ImportError as e: - if e.name == "pyarrow": - pytest.skip(f"pyarrow not available - {item.name} requires pyarrow") - else: - raise - - -@pytest.hookimpl(hookwrapper=True) -def pytest_make_collect_report(collector): - """Wrap module collection to catch pyarrow import errors on Python 3.14. - - If we're on Python 3.14 and a test module raises ModuleNotFoundError - for 'pyarrow', mark the entire module as xfailed rather than failing collection. - - TODO(evertlammerts): Remove skip when pyarrow releases for 3.14. - https://github.com/duckdblabs/duckdb-internal/issues/6182 - """ - outcome = yield - report: pytest.CollectReport = outcome.get_result() - - if sys.version_info[:2] == (3, 14): - # Only handle failures from module collectors - if report.failed and collector.__class__.__name__ == "Module": - longreprtext = report.longreprtext - if "ModuleNotFoundError: No module named 'pyarrow'" in longreprtext: - report.outcome = "skipped" - reason = f"XFAIL: [pyarrow not available] {longreprtext}" - report.longrepr = (report.fspath, None, reason) - - # https://docs.pytest.org/en/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option # https://stackoverflow.com/a/47700320 def pytest_addoption(parser): diff --git a/tests/fast/adbc/test_adbc.py b/tests/fast/adbc/test_adbc.py index 80920a99..c20d2a0e 100644 --- a/tests/fast/adbc/test_adbc.py +++ b/tests/fast/adbc/test_adbc.py @@ -1,5 +1,4 @@ import datetime -import sys from pathlib import Path import adbc_driver_manager.dbapi @@ -29,7 +28,6 @@ def example_table(): ) -@xfail(sys.platform == "win32", reason="adbc-driver-manager.adbc_get_info() returns an empty dict on windows") def test_connection_get_info(duck_conn): assert duck_conn.adbc_get_info() != {} @@ -42,9 +40,6 @@ def test_connection_get_table_types(duck_conn): assert duck_conn.adbc_get_table_types() == ["BASE TABLE"] -@xfail( - sys.platform == "win32", reason="adbc-driver-manager.adbc_get_objects() returns an invalid schema dict on windows" -) def test_connection_get_objects(duck_conn): with duck_conn.cursor() as cursor: cursor.execute("CREATE TABLE getobjects (ints BIGINT PRIMARY KEY)") @@ -66,9 +61,6 @@ def test_connection_get_objects(duck_conn): assert depth_all.schema == depth_catalogs.schema -@xfail( - sys.platform == "win32", reason="adbc-driver-manager.adbc_get_objects() returns an invalid schema dict on windows" -) def test_connection_get_objects_filters(duck_conn): with duck_conn.cursor() as cursor: cursor.execute("CREATE TABLE getobjects (ints BIGINT PRIMARY KEY)") @@ -207,7 +199,6 @@ def test_statement_query(duck_conn): assert cursor.fetch_arrow_table().to_pylist() == [{"foo": 1}] -@xfail(sys.platform == "win32", reason="adbc-driver-manager returns an invalid table schema on windows") def test_insertion(duck_conn): table = example_table() reader = table.to_reader() @@ -234,7 +225,6 @@ def test_insertion(duck_conn): assert cursor.fetch_arrow_table().to_pydict() == {"count_star()": [8]} -@xfail(sys.platform == "win32", reason="adbc-driver-manager returns an invalid table schema on windows") def test_read(duck_conn): with duck_conn.cursor() as cursor: filename = Path(__file__).parent / ".." / "data" / "category.csv" diff --git a/tests/fast/api/test_to_parquet.py b/tests/fast/api/test_to_parquet.py index 8d8162b0..f0952e68 100644 --- a/tests/fast/api/test_to_parquet.py +++ b/tests/fast/api/test_to_parquet.py @@ -1,4 +1,6 @@ import os +import pathlib +import re import tempfile import pytest @@ -170,3 +172,56 @@ def test_append(self, pd): ("shinji", 123.0, "a"), ] assert result.execute().fetchall() == expected + + @pytest.mark.parametrize("pd", [NumpyPandas(), ArrowPandas()]) + def test_filename_pattern_with_index(self, pd): + temp_file_name = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names())) # noqa: PTH118 + df = pd.DataFrame( + { + "name": ["rei", "shinji", "asuka", "kaworu"], + "float": [321.0, 123.0, 23.0, 340.0], + "category": ["a", "a", "b", "c"], + } + ) + rel = duckdb.from_df(df) + rel.to_parquet(temp_file_name, partition_by=["category"], filename_pattern="orders_{i}") + # Check that files follow the pattern with {i} + files_a = list(pathlib.Path(f"{temp_file_name}/category=a").iterdir()) + files_b = list(pathlib.Path(f"{temp_file_name}/category=b").iterdir()) + files_c = list(pathlib.Path(f"{temp_file_name}/category=c").iterdir()) + filename_pattern = re.compile(r"^orders_[09]+\.parquet$") + assert all(filename_pattern.search(str(f.name)) for f in files_a) + assert all(filename_pattern.search(str(f.name)) for f in files_b) + assert all(filename_pattern.search(str(f.name)) for f in files_c) + + # Verify data integrity + result = duckdb.sql(f"FROM read_parquet('{temp_file_name}/*/*.parquet', hive_partitioning=TRUE)") + expected = [("rei", 321.0, "a"), ("shinji", 123.0, "a"), ("asuka", 23.0, "b"), ("kaworu", 340.0, "c")] + assert result.execute().fetchall() == expected + + @pytest.mark.parametrize("pd", [NumpyPandas(), ArrowPandas()]) + def test_filename_pattern_with_uuid(self, pd): + temp_file_name = os.path.join(tempfile.mkdtemp(), next(tempfile._get_candidate_names())) # noqa: PTH118 + df = pd.DataFrame( + { + "name": ["rei", "shinji", "asuka", "kaworu"], + "float": [321.0, 123.0, 23.0, 340.0], + "category": ["a", "a", "b", "c"], + } + ) + rel = duckdb.from_df(df) + rel.to_parquet(temp_file_name, partition_by=["category"], filename_pattern="file_{uuid}") + # Check that files follow the pattern with {uuid} + files_a = list(pathlib.Path(f"{temp_file_name}/category=a").iterdir()) + files_b = list(pathlib.Path(f"{temp_file_name}/category=b").iterdir()) + files_c = list(pathlib.Path(f"{temp_file_name}/category=c").iterdir()) + filename_pattern = re.compile(r"^file_[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}\.parquet$") + print(files_a) + assert all(filename_pattern.search(str(f.name)) for f in files_a) + assert all(filename_pattern.search(str(f.name)) for f in files_b) + assert all(filename_pattern.search(str(f.name)) for f in files_c) + + # Verify data integrity + result = duckdb.sql(f"FROM read_parquet('{temp_file_name}/*/*.parquet', hive_partitioning=TRUE)") + expected = [("rei", 321.0, "a"), ("shinji", 123.0, "a"), ("asuka", 23.0, "b"), ("kaworu", 340.0, "c")] + assert result.execute().fetchall() == expected diff --git a/tests/fast/pandas/test_column_order.py b/tests/fast/pandas/test_column_order.py new file mode 100644 index 00000000..0600bc4c --- /dev/null +++ b/tests/fast/pandas/test_column_order.py @@ -0,0 +1,16 @@ +import duckdb + + +class TestColumnOrder: + def test_column_order(self, duckdb_cursor): + to_execute = """ + CREATE OR REPLACE TABLE t1 AS ( + SELECT NULL AS col1, + NULL::TIMESTAMPTZ AS timepoint, + NULL::DATE AS date, + ); + SELECT timepoint, date, col1 FROM t1; + """ + df = duckdb.execute(to_execute).fetchdf() + cols = list(df.columns) + assert cols == ["timepoint", "date", "col1"] diff --git a/tests/fast/test_insert.py b/tests/fast/test_insert.py index a61efd2e..455e6e48 100644 --- a/tests/fast/test_insert.py +++ b/tests/fast/test_insert.py @@ -27,6 +27,4 @@ def test_insert_with_schema(self, duckdb_cursor): res = duckdb_cursor.table("not_main.tbl").fetchall() assert len(res) == 10 - # TODO: This is not currently supported # noqa: TD002, TD003 - with pytest.raises(duckdb.CatalogException, match="Table with name tbl does not exist"): - duckdb_cursor.table("not_main.tbl").insert([42, 21, 1337]) + duckdb_cursor.table("not_main.tbl").insert((42,)) diff --git a/tests/fast/test_relation.py b/tests/fast/test_relation.py index 7b60a105..4d6f6591 100644 --- a/tests/fast/test_relation.py +++ b/tests/fast/test_relation.py @@ -280,7 +280,9 @@ def test_value_relation(self, duckdb_cursor): rel = duckdb_cursor.values((const(1), const(2), const(3)), const(4)) # Using Expressions that can't be resolved: - with pytest.raises(duckdb.BinderException, match='Referenced column "a" not found in FROM clause!'): + with pytest.raises( + duckdb.BinderException, match='Referenced column "a" was not found because the FROM clause is missing' + ): duckdb_cursor.values(duckdb.ColumnExpression("a")) def test_insert_into_operator(self):