diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 68089beb54..8a233bccd7 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -103,7 +103,6 @@ import pyarrow as pa import ray from duckdb import DuckDBPyConnection - from pyiceberg_core.datafusion import IcebergDataFusionTable from pyiceberg.catalog import Catalog from pyiceberg.catalog.rest.scan_planning import RESTContentFile, RESTDeleteFile, RESTFileScanTask @@ -1559,7 +1558,7 @@ def to_polars(self) -> pl.LazyFrame: return pl.scan_iceberg(self) - def __datafusion_table_provider__(self) -> IcebergDataFusionTable: + def __datafusion_table_provider__(self, session: Any | None = None) -> Any: """Return the DataFusion table provider PyCapsule interface. To support DataFusion features such as push down filtering, this function will return a PyCapsule @@ -1602,7 +1601,7 @@ def __datafusion_table_provider__(self) -> IcebergDataFusionTable: identifier=self.name(), metadata_location=self.metadata_location, file_io_properties=self.io.properties, - ).__datafusion_table_provider__() + ).__datafusion_table_provider__(session) class StaticTable(Table): diff --git a/pyproject.toml b/pyproject.toml index c06fad1270..ed6bfbb590 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ pyiceberg = "pyiceberg.cli.console:run" [project.optional-dependencies] pyarrow = [ "pyarrow>=17.0.0", - "pyiceberg-core>=0.5.1,<0.9.0", + "pyiceberg-core==0.9.0rc1", ] pandas = [ "pandas>=1.0.0", @@ -93,8 +93,8 @@ sql-sqlite = ["sqlalchemy>=2.0.18,<3"] gcsfs = ["gcsfs>=2023.1.0"] rest-sigv4 = ["boto3>=1.24.59"] hf = ["huggingface-hub>=0.24.0"] -pyiceberg-core = ["pyiceberg-core>=0.5.1,<0.9.0"] -datafusion = ["datafusion>=51,<52"] +pyiceberg-core = ["pyiceberg-core==0.9.0rc1"] +datafusion = ["datafusion==52"] gcp-auth = ["google-auth>=2.4.0"] entra-auth = ["azure-identity>=1.25.1"] geoarrow = ["geoarrow-pyarrow>=0.2.0"] diff --git a/uv.lock b/uv.lock index 6770675728..02c79ae9ec 100644 --- a/uv.lock +++ b/uv.lock @@ -1192,19 +1192,19 @@ wheels = [ [[package]] name = "datafusion" -version = "51.0.0" +version = "52.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyarrow" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2c/6d/d0e2632c93bbcca0687eeda672af3f92042ecd349df7be55da86253594a9/datafusion-51.0.0.tar.gz", hash = "sha256:1887c7d5ed3ae5d9f389e62ba869864afad4006a3f7c99ef0ca4707782a7838f", size = 193751, upload-time = "2026-01-09T13:23:41.562Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/04/4dabd255e04801b942221bf7eeea661f540d8c116e6b4a783fe2479410f0/datafusion-52.0.0.tar.gz", hash = "sha256:842cf9cdb523d04a053c5408da24645e3b2adce5d6c42ddc80a8c5edf9013ff3", size = 204988, upload-time = "2026-02-23T12:22:50.919Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/a9/7717cec053a3309be3020fe3147e3f76e5bf21295fa8adf9b52dd44ea3ff/datafusion-51.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:0c0d265fe3ee0dcbfa7cc3c64c7cd94fc493f38418bd79debb7ec29f29b7176e", size = 30389413, upload-time = "2026-01-09T13:23:23.266Z" }, - { url = "https://files.pythonhosted.org/packages/55/45/72c9874fd3740a4cb9d55049fdbae0df512dc5433e9f1176f3cfd970f1a1/datafusion-51.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:43e6011db86e950bf9a21ed73cc089c2346b340a41a4f1044268af6c3a357acc", size = 26982206, upload-time = "2026-01-09T13:23:27.437Z" }, - { url = "https://files.pythonhosted.org/packages/21/ac/b32ba1f25d38fc16e7623cc4bfb7bd68db61be2ef27b2d9969ea5c865765/datafusion-51.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e76803907150159aa059d5cc9291645bbaac1b6a46d07e56035118d327b741ae", size = 33246117, upload-time = "2026-01-09T13:23:30.981Z" }, - { url = "https://files.pythonhosted.org/packages/0b/4e/437121422ef010690fc3cdd7f080203e986ba00e0e3c3b577e03f5b54ca2/datafusion-51.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9d0cfabfe1853994adc2e6e9da5f36c1eb061102e34a2f1101fa935c6991c9e1", size = 31421867, upload-time = "2026-01-09T13:23:34.436Z" }, - { url = "https://files.pythonhosted.org/packages/db/fc/58cf27fcb85b2fd2a698253ae46213b1cbda784407e205c148f4006c1429/datafusion-51.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:fd5f9abfd6669062debf0658d13e4583234c89d4df95faf381927b11cea411f5", size = 32517679, upload-time = "2026-01-09T13:23:39.615Z" }, + { url = "https://files.pythonhosted.org/packages/77/38/66b2f2fd77d3fb66ff48a8922130379dece3ba6d2e29fc86fbb4298a874b/datafusion-52.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:999881df12ab78b6c8f04dd2056b24389374e93775a649ed20c5e35db2f42f65", size = 31473623, upload-time = "2026-02-23T12:22:30.437Z" }, + { url = "https://files.pythonhosted.org/packages/d0/b5/ce6c6030fa8e4fc38d10d5c4aa9cc6fe1cda625e409a18eb08ea09a87c8d/datafusion-52.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd58e64158152f5c4a5836a3ce3bcca2a109d600c9ce7efdcf82e61c1ab0fbc8", size = 28108736, upload-time = "2026-02-23T12:22:33.5Z" }, + { url = "https://files.pythonhosted.org/packages/d8/c1/d7ac9ddc9f54a8f178900f529a723d6121361111f0d0d2527bb47f86f6ce/datafusion-52.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ab3591904f32ce290ff7161fb804e1c7bf323de16e3ddc8cf1f76310e994208e", size = 30699663, upload-time = "2026-02-23T12:22:37.193Z" }, + { url = "https://files.pythonhosted.org/packages/b0/2f/14cffc5305abe05d56f3e99e8054c96bd94411185de059a98fc1ca0e5ec0/datafusion-52.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ac4b364937c277bbfcac032dbc49d08c078b13ba3f8bfda117da5fda4ea328bc", size = 33050161, upload-time = "2026-02-23T12:22:40.858Z" }, + { url = "https://files.pythonhosted.org/packages/24/ae/3fdea50fa88f304db96728a67deb6e07bb0d9a02f665ca09db4237a9a199/datafusion-52.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:67e252ef20b918537c8fdb47e6c825c0bd639795e19715a85fedde331a83d2e1", size = 33717685, upload-time = "2026-02-23T12:22:44.463Z" }, ] [[package]] @@ -4693,7 +4693,7 @@ requires-dist = [ { name = "cachetools", specifier = ">=5.5,<8.0" }, { name = "click", specifier = ">=7.1.1,<9.0.0" }, { name = "daft", marker = "extra == 'daft'", specifier = ">=0.5.0" }, - { name = "datafusion", marker = "extra == 'datafusion'", specifier = ">=51,<52" }, + { name = "datafusion", marker = "extra == 'datafusion'", specifier = "==52" }, { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=0.5.0,<2.0.0" }, { name = "fsspec", specifier = ">=2023.1.0" }, { name = "gcsfs", marker = "extra == 'gcsfs'", specifier = ">=2023.1.0" }, @@ -4712,8 +4712,8 @@ requires-dist = [ { name = "pyarrow", marker = "extra == 'pyarrow'", specifier = ">=17.0.0" }, { name = "pyarrow", marker = "extra == 'ray'", specifier = ">=17.0.0" }, { name = "pydantic", specifier = ">=2.0,!=2.4.0,!=2.4.1,!=2.12.0,!=2.12.1,<3.0" }, - { name = "pyiceberg-core", marker = "extra == 'pyarrow'", specifier = ">=0.5.1,<0.9.0" }, - { name = "pyiceberg-core", marker = "extra == 'pyiceberg-core'", specifier = ">=0.5.1,<0.9.0" }, + { name = "pyiceberg-core", marker = "extra == 'pyarrow'", specifier = "==0.9.0rc1" }, + { name = "pyiceberg-core", marker = "extra == 'pyiceberg-core'", specifier = "==0.9.0rc1" }, { name = "pyparsing", specifier = ">=3.1.0,<4.0.0" }, { name = "pyroaring", specifier = ">=1.0.0,<2.0.0" }, { name = "python-snappy", marker = "extra == 'snappy'", specifier = ">=0.6.0,<1.0.0" }, @@ -4772,17 +4772,16 @@ notebook = [{ name = "jupyterlab", specifier = ">=4.0.0" }] [[package]] name = "pyiceberg-core" -version = "0.8.0" +version = "0.9.0rc1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9c/a0/0bcedbbe901484aacb6c605505f8574fd65954826e592fdb163e1cfb09f2/pyiceberg_core-0.8.0.tar.gz", hash = "sha256:59021ca5bc7ca95f2b06fb0730280fb3f60ed898060bcd874c156d093853b5f3", size = 618882, upload-time = "2026-01-20T00:50:40.076Z" } +sdist = { url = "https://files.pythonhosted.org/packages/de/5b/ef356c051920c11b3adf9849963503b49b0aa9902d78ec7762a7a06076cb/pyiceberg_core-0.9.0rc1.tar.gz", hash = "sha256:a0bb7224132d71ef60041b2987e10a5203e69096b574e23b2347e160d39cfd73", size = 687620, upload-time = "2026-03-10T22:07:05.203Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/77/e0/9a8fa537d29d34e3265682056d6517b926975107b5b1af6057d1713557d6/pyiceberg_core-0.8.0-cp310-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:d60c75a741a1d9199277a9e50fc3adbc84ab286a881f9b1f721fa120e7197912", size = 24733948, upload-time = "2026-01-20T00:50:20.566Z" }, - { url = "https://files.pythonhosted.org/packages/6d/3e/f5522c1e9c20c3e89bfd76b2f54ba38e57389e5a2872233e49e60a131e04/pyiceberg_core-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1d71e566b2d56141760ff8734667eede5a5d60963dfbcdce80c2dd3cf2edb39d", size = 11682041, upload-time = "2026-01-20T00:50:22.843Z" }, - { url = "https://files.pythonhosted.org/packages/95/4b/f799e5c7a2b2ede75514e64901503358a7a134ca1ea217fd86535af533b6/pyiceberg_core-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82782d1b974200c5526d069391ba2bc235a868b5d0d6ac17ca406df735ab89a3", size = 13835428, upload-time = "2026-01-20T00:50:25.021Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ff/2dbd6f7c99a2f782f908be2cc997371de45cc1df61abeeff1fc0165c05b6/pyiceberg_core-0.8.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e14b2aea26293ba5878c398adc880fff0f1ce5d989e00d4b1a930c143541114", size = 14580807, upload-time = "2026-01-20T00:50:27.158Z" }, - { url = "https://files.pythonhosted.org/packages/bc/13/176c2b00a9b804af79d8b697ba1a2525f4390e959be777076972071ca069/pyiceberg_core-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:a5726cc62f9ac2582a0d5dde92e4140b711b5e29ec0c6c636d6d2782d984031b", size = 13354110, upload-time = "2026-01-20T00:50:29.785Z" }, - { url = "https://files.pythonhosted.org/packages/43/33/2c93b4d40f38e173dcfb2f555a2d992e3345d38eba8f8fbb2731c2e18462/pyiceberg_core-0.8.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0c51f1e6a4112864c8fface904429814b26baa45efab7efbae8013796e815915", size = 11682427, upload-time = "2026-01-20T00:50:31.714Z" }, - { url = "https://files.pythonhosted.org/packages/7c/97/25e835c2bf9b090bf97174638bc26a034952208e2767730035e285766bb6/pyiceberg_core-0.8.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:6d2e71fb83ae578d93f24699ee131b77e5ee630b8ec6310acf570d04062ef2fb", size = 14581679, upload-time = "2026-01-20T00:50:33.869Z" }, + { url = "https://files.pythonhosted.org/packages/c4/65/b6ca4593cd2c6027edce8f6efd2414b0d339f13a84f7b8eaf4ed8987a759/pyiceberg_core-0.9.0rc1-cp310-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:944e440fc562c3b6952d2d08d592e2e595a718c115c92b659c69219786e358ec", size = 18447422, upload-time = "2026-03-10T22:06:28.915Z" }, + { url = "https://files.pythonhosted.org/packages/4c/90/7ed768b2e262150a4e6ea5d3ce50158ed97103e4b93198a85a1c644ef18a/pyiceberg_core-0.9.0rc1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:040431b8e6928092c3da125dcf3b5dd4a3b35a4b42c77ca610d07c4dfc0f06be", size = 8796488, upload-time = "2026-03-10T22:06:32.853Z" }, + { url = "https://files.pythonhosted.org/packages/9d/c9/0d89f050df7f04080d50e4553000f8e53eeabf69b786c410dd6eba49b8c1/pyiceberg_core-0.9.0rc1-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:b4a1cf17727aceed6fcb34c263dd3b58b1ad3451fbf78b7d4b11556ac9682c4f", size = 10869149, upload-time = "2026-03-10T22:06:37.031Z" }, + { url = "https://files.pythonhosted.org/packages/0c/4a/c5fbe512680feaacf8823e8013598148d0a2eb190212f87f836c4befda0d/pyiceberg_core-0.9.0rc1-cp310-abi3-win_amd64.whl", hash = "sha256:d5077afd2b00e3bb616de0ee87f13fd2445f24584d7c6e2087756855d388a9e3", size = 9924327, upload-time = "2026-03-10T22:06:41.112Z" }, + { url = "https://files.pythonhosted.org/packages/4c/e4/88ef8dbd12c07e5f32c359dbe4e62ec2e858520aef4cac003995e9cef375/pyiceberg_core-0.9.0rc1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:05e62e7ef6e5f27eed952ca409622b8364769120c6d124c021f73292a3989508", size = 8796084, upload-time = "2026-03-10T22:06:48.69Z" }, + { url = "https://files.pythonhosted.org/packages/70/66/22357818d467cfd1dc8f015dc4d6b7eff6d3fab68055e60f65af6f028066/pyiceberg_core-0.9.0rc1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:1bf9a5b438829bf724c2dfc5dc00c9f10f01b7aa97a2d9a2b917f62a5b23aeed", size = 10869289, upload-time = "2026-03-10T22:06:52.969Z" }, ] [[package]]