Build: Bump datafusion to 52.3.0 and pyiceberg-core to 0.9.0 (#3171)

<!--
Thanks for opening a pull request!
-->

<!-- In the case this PR will resolve an issue, please replace
${GITHUB_ISSUE_ID} below with the actual Github issue id. -->
<!-- Closes #${GITHUB_ISSUE_ID} -->

# Rationale for this change
Upgrade deps
* datafusion 51 -> 52.3
* pyiceberg-core 0.8.0 -> 0.9.0

Update `__datafusion_table_provider__` to take in `session` param as per
[df 52 migration
guide](https://github.com/apache/datafusion-python/blob/85a3595444e7946dc4eaa166cb4843bee2bf2f07/docs/source/user-guide/upgrade-guides.rst#datafusion-5200)

Prevent `dependabot` from auto update `datafusion`, we do it manually

## Are these changes tested?

## Are there any user-facing changes?

<!-- In the case of user-facing changes, please add the changelog label.
-->

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Author: kevinjqliu

.github/dependabot.yml

@@ -24,6 +24,8 @@ updates:
     schedule:
       interval: "weekly"
     open-pull-requests-limit: 50
+    ignore:
+      - dependency-name: "datafusion"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:

pyiceberg/table/__init__.py

@@ -1559,7 +1559,7 @@ def to_polars(self) -> pl.LazyFrame:
 
         return pl.scan_iceberg(self)
 
-    def __datafusion_table_provider__(self) -> IcebergDataFusionTable:
+    def __datafusion_table_provider__(self, session: Any | None = None) -> IcebergDataFusionTable:
         """Return the DataFusion table provider PyCapsule interface.
 
         To support DataFusion features such as push down filtering, this function will return a PyCapsule
@@ -1598,11 +1598,12 @@ def __datafusion_table_provider__(self) -> IcebergDataFusionTable:
         """
         from pyiceberg_core.datafusion import IcebergDataFusionTable
 
-        return IcebergDataFusionTable(
+        provider = IcebergDataFusionTable(
             identifier=self.name(),
             metadata_location=self.metadata_location,
             file_io_properties=self.io.properties,
-        ).__datafusion_table_provider__()
+        ).__datafusion_table_provider__
+        return provider(session)
 
 
 class StaticTable(Table):

pyproject.toml

@@ -55,7 +55,7 @@ pyiceberg = "pyiceberg.cli.console:run"
 [project.optional-dependencies]
 pyarrow = [
     "pyarrow>=17.0.0",
-    "pyiceberg-core>=0.5.1,<0.9.0",
+    "pyiceberg-core>=0.5.1,<0.10.0",
 ]
 pandas = [
     "pandas>=1.0.0",
@@ -93,8 +93,8 @@ sql-sqlite = ["sqlalchemy>=2.0.18,<3"]
 gcsfs = ["gcsfs>=2023.1.0"]
 rest-sigv4 = ["boto3>=1.24.59"]
 hf = ["huggingface-hub>=0.24.0"]
-pyiceberg-core = ["pyiceberg-core>=0.5.1,<0.9.0"]
-datafusion = ["datafusion>=51,<52"]
+pyiceberg-core = ["pyiceberg-core>=0.5.1,<0.10.0"]
+datafusion = ["datafusion>=52,<53"]
 gcp-auth = ["google-auth>=2.4.0"]
 entra-auth = ["azure-identity>=1.25.1"]
 geoarrow = ["geoarrow-pyarrow>=0.2.0"]