dep: upgrade pyiceberg-core to 0.8.0 and datafusion to 51 (#2928)

<!--
Thanks for opening a pull request!
-->

<!-- In the case this PR will resolve an issue, please replace
${GITHUB_ISSUE_ID} below with the actual Github issue id. -->
<!-- Closes #${GITHUB_ISSUE_ID} -->

# Rationale for this change
Update pyiceberg-core to the latest release, 0.8.0
Also update datafusion to match the version iceberg-rust uses, v51
https://github.com/apache/iceberg-rust/blob/v0.8.0/Cargo.toml#L65

Pulled in #2749 as well. Otherwise CI will fail.
`register_table_provider` has been deprecated:
https://datafusion.apache.org/python/autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_table_provider

## Are these changes tested?

## Are there any user-facing changes?

<!-- In the case of user-facing changes, please add the changelog label.
-->

---------

Co-authored-by: Fokko Driesprong <fokko@apache.org>

Author: kevinjqliu

mkdocs/docs/api.md

@@ -1941,7 +1941,7 @@ PyIceberg integrates with [Apache DataFusion](https://datafusion.apache.org/) th
 
     The integration has a few caveats:
 
-    - Only works with `datafusion >= 45, < 49`
+    - Only works with `datafusion == 51`, aligns with the version used in `pyiceberg-core`
     - Depends directly on `iceberg-rust` instead of PyIceberg's implementation
     - Has limited features compared to the full PyIceberg API
 
@@ -1967,7 +1967,7 @@ iceberg_table.append(data)
 
 # Register the table with DataFusion
 ctx = SessionContext()
-ctx.register_table_provider("test", iceberg_table)
+ctx.register_table("test", iceberg_table)
 
 # Query the table using DataFusion SQL
 ctx.table("test").show()

pyiceberg/table/__init__.py

@@ -1601,7 +1601,7 @@ def __datafusion_table_provider__(self) -> IcebergDataFusionTable:
 
         To support DataFusion features such as push down filtering, this function will return a PyCapsule
         interface that conforms to the FFI Table Provider required by DataFusion. From an end user perspective
-        you should not need to call this function directly. Instead you can use ``register_table_provider`` in
+        you should not need to call this function directly. Instead you can use ``register_table`` in
         the DataFusion SessionContext.
 
         Returns:
@@ -1618,7 +1618,7 @@ def __datafusion_table_provider__(self) -> IcebergDataFusionTable:
             iceberg_table = catalog.create_table("default.test", schema=data.schema)
             iceberg_table.append(data)
             ctx = SessionContext()
-            ctx.register_table_provider("test", iceberg_table)
+            ctx.register_table("test", iceberg_table)
             ctx.table("test").show()
             ```
             Results in

pyproject.toml

@@ -56,7 +56,7 @@ pyiceberg = "pyiceberg.cli.console:run"
 [project.optional-dependencies]
 pyarrow = [
     "pyarrow>=17.0.0",
-    "pyiceberg-core>=0.5.1,<0.8.0",
+    "pyiceberg-core>=0.5.1,<0.9.0",
 ]
 pandas = [
     "pandas>=1.0.0,<3.0.0",
@@ -94,8 +94,8 @@ sql-sqlite = ["sqlalchemy>=2.0.18,<3"]
 gcsfs = ["gcsfs>=2023.1.0"]
 rest-sigv4 = ["boto3>=1.24.59"]
 hf = ["huggingface-hub>=0.24.0"]
-pyiceberg-core = ["pyiceberg-core>=0.5.1,<0.8.0"]
-datafusion = ["datafusion>=45,<49"]
+pyiceberg-core = ["pyiceberg-core>=0.5.1,<0.9.0"]
+datafusion = ["datafusion>=51,<52"]
 gcp-auth = ["google-auth>=2.4.0"]
 
 [dependency-groups]

tests/table/test_datafusion.py

@@ -49,7 +49,7 @@ def test_datafusion_register_pyiceberg_table(catalog: Catalog, arrow_table_with_
     iceberg_table.append(arrow_table_with_null)
 
     ctx = SessionContext()
-    ctx.register_table_provider("test", iceberg_table)
+    ctx.register_table("test", iceberg_table)
 
     datafusion_table = ctx.table("test")
     assert datafusion_table is not None