UNPICK changes to review

Author: kosiew

docs/source/contributor-guide/ffi.rst

@@ -34,7 +34,7 @@ as performant as possible and to utilize the features of DataFusion, you may dec
 your source in Rust and then expose it through `PyO3 <https://pyo3.rs>`_ as a Python library.
 
 At first glance, it may appear the best way to do this is to add the ``datafusion-python``
-crate as a dependency, produce a DataFusion table in Rust, and then register it with the
+crate as a dependency, provide a ``PyTable``, and then to register it with the 
 ``SessionContext``. Unfortunately, this will not work.
 
 When you produce your code as a Python library and it needs to interact with the DataFusion

docs/source/user-guide/data-sources.rst

@@ -152,11 +152,9 @@ as Delta Lake. This will require a recent version of
 .. code-block:: python
 
     from deltalake import DeltaTable
-    from datafusion import TableProvider
 
     delta_table = DeltaTable("path_to_table")
-    provider = TableProvider.from_capsule(delta_table)
-    ctx.register_table("my_delta_table", provider)
+    ctx.register_table_provider("my_delta_table", delta_table)
     df = ctx.table("my_delta_table")
     df.show()
 

docs/source/user-guide/io/table_provider.rst

@@ -47,33 +47,12 @@ A complete example can be found in the `examples folder <https://github.com/apac
         }
     }
 
-Once you have this library available, you can construct a
-:py:class:`~datafusion.TableProvider` in Python and register it with the
-``SessionContext``.  Table providers can be created either from the PyCapsule exposed by
-your Rust provider or from an existing :py:class:`~datafusion.dataframe.DataFrame`.
-Call the provider's ``__datafusion_table_provider__()`` method to obtain the capsule
-before constructing a ``TableProvider``. The ``TableProvider.from_view()`` helper is
-deprecated; instead use ``TableProvider.from_dataframe()`` or ``DataFrame.into_view()``.
+Once you have this library available, in python you can register your table provider
+to the ``SessionContext``.
 
 .. code-block:: python
 
-    from datafusion import SessionContext, TableProvider
-
-    ctx = SessionContext()
     provider = MyTableProvider()
+    ctx.register_table_provider("my_table", provider)
 
-    capsule = provider.__datafusion_table_provider__()
-    capsule_provider = TableProvider.from_capsule(capsule)
-
-    df = ctx.from_pydict({"a": [1]})
-    view_provider = TableProvider.from_dataframe(df)
-    # or: view_provider = df.into_view()
-
-    ctx.register_table("capsule_table", capsule_provider)
-    ctx.register_table("view_table", view_provider)
-
-    ctx.table("capsule_table").show()
-    ctx.table("view_table").show()
-
-Both ``TableProvider.from_capsule()`` and ``TableProvider.from_dataframe()`` create
-table providers that can be registered with the SessionContext using ``register_table()``.
+    ctx.table("my_table").show()

examples/datafusion-ffi-example/python/tests/_test_table_function.py

@@ -53,7 +53,7 @@ def test_ffi_table_function_call_directly():
     table_udtf = udtf(table_func, "my_table_func")
 
     my_table = table_udtf()
-    ctx.register_table("t", my_table)
+    ctx.register_table_provider("t", my_table)
     result = ctx.table("t").collect()
 
     assert len(result) == 2

examples/datafusion-ffi-example/python/tests/_test_table_provider.py

@@ -18,16 +18,14 @@
 from __future__ import annotations
 
 import pyarrow as pa
-from datafusion import SessionContext, TableProvider
+from datafusion import SessionContext
 from datafusion_ffi_example import MyTableProvider
 
 
 def test_table_loading():
     ctx = SessionContext()
     table = MyTableProvider(3, 2, 4)
-    ctx.register_table(
-        "t", TableProvider.from_capsule(table.__datafusion_table_provider__())
-    )
+    ctx.register_table_provider("t", table)
     result = ctx.table("t").collect()
 
     assert len(result) == 4

python/datafusion/__init__.py

@@ -33,7 +33,7 @@
 from . import functions, object_store, substrait, unparser
 
 # The following imports are okay to remain as opaque to the user.
-from ._internal import Config, TableProvider
+from ._internal import Config
 from .catalog import Catalog, Database, Table
 from .col import col, column
 from .common import (
@@ -66,10 +66,7 @@
     udwf,
 )
 
-try:
-    __version__ = importlib_metadata.version(__name__)
-except importlib_metadata.PackageNotFoundError:
-    __version__ = "0.0.0"
+__version__ = importlib_metadata.version(__name__)
 
 __all__ = [
     "Accumulator",
@@ -93,7 +90,6 @@
     "SessionContext",
     "Table",
     "TableFunction",
-    "TableProvider",
     "WindowFrame",
     "WindowUDF",
     "catalog",

python/datafusion/catalog.py

@@ -27,8 +27,6 @@
 if TYPE_CHECKING:
     import pyarrow as pa
 
-    from datafusion import TableProvider
-
 try:
     from warnings import deprecated  # Python 3.13+
 except ImportError:
@@ -124,8 +122,8 @@ def table(self, name: str) -> Table:
         """Return the table with the given ``name`` from this schema."""
         return Table(self._raw_schema.table(name))
 
-    def register_table(self, name, table: Table | TableProvider) -> None:
-        """Register a table or table provider in this schema."""
+    def register_table(self, name, table) -> None:
+        """Register a table provider in this schema."""
         if isinstance(table, Table):
             return self._raw_schema.register_table(name, table.table)
         return self._raw_schema.register_table(name, table)
@@ -221,8 +219,8 @@ def table(self, name: str) -> Table | None:
         """Retrieve a specific table from this schema."""
         ...
 
-    def register_table(self, name: str, table: Table | TableProvider) -> None:  # noqa: B027
-        """Add a table to this schema.
+    def register_table(self, name: str, table: Table) -> None:  # noqa: B027
+        """Add a table from this schema.
 
         This method is optional. If your schema provides a fixed list of tables, you do
         not need to implement this method.

python/datafusion/context.py

@@ -46,7 +46,6 @@
     import pandas as pd
     import polars as pl
 
-    from datafusion import TableProvider
     from datafusion.plan import ExecutionPlan, LogicalPlan
 
 
@@ -735,7 +734,7 @@ def from_polars(self, data: pl.DataFrame, name: str | None = None) -> DataFrame:
     # https://github.com/apache/datafusion-python/pull/1016#discussion_r1983239116
     # is the discussion on how we arrived at adding register_view
     def register_view(self, name: str, df: DataFrame) -> None:
-        """Register a :py:class:`~datafusion.dataframe.DataFrame` as a view.
+        """Register a :py:class: `~datafusion.detaframe.DataFrame` as a view.
 
         Args:
             name (str): The name to register the view under.
@@ -744,25 +743,16 @@ def register_view(self, name: str, df: DataFrame) -> None:
         view = df.into_view()
         self.ctx.register_table(name, view)
 
-    def register_table(self, name: str, table: Table | TableProvider) -> None:
-        """Register a :py:class:`~datafusion.catalog.Table` or ``TableProvider``.
+    def register_table(self, name: str, table: Table) -> None:
+        """Register a :py:class: `~datafusion.catalog.Table` as a table.
 
-        The registered table can be referenced from SQL statements executed against
-        this context.
-
-        Plain :py:class:`~datafusion.dataframe.DataFrame` objects are not supported;
-        convert them first with :meth:`datafusion.dataframe.DataFrame.into_view` or
-        :meth:`datafusion.catalog.TableProvider.from_dataframe`.
+        The registered table can be referenced from SQL statement executed against.
 
         Args:
             name: Name of the resultant table.
-            table: DataFusion :class:`Table` or :class:`TableProvider` to add to the
-                session context.
+            table: DataFusion table to add to the session context.
         """
-        if isinstance(table, Table):
-            self.ctx.register_table(name, table.table)
-        else:
-            self.ctx.register_table(name, table)
+        self.ctx.register_table(name, table.table)
 
     def deregister_table(self, name: str) -> None:
         """Remove a table from the session."""
@@ -782,18 +772,14 @@ def register_catalog_provider(
             self.ctx.register_catalog_provider(name, provider)
 
     def register_table_provider(
-        self, name: str, provider: TableProviderExportable | TableProvider
+        self, name: str, provider: TableProviderExportable
     ) -> None:
         """Register a table provider.
 
-        Deprecated: use :meth:`register_table` instead.
+        This table provider must have a method called ``__datafusion_table_provider__``
+        which returns a PyCapsule that exposes a ``FFI_TableProvider``.
         """
-        warnings.warn(
-            "register_table_provider is deprecated; use register_table",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        self.register_table(name, provider)
+        self.ctx.register_table_provider(name, provider)
 
     def register_udtf(self, func: TableFunction) -> None:
         """Register a user defined table function."""

python/datafusion/dataframe.py

@@ -40,7 +40,6 @@
 from datafusion._internal import DataFrame as DataFrameInternal
 from datafusion._internal import ParquetColumnOptions as ParquetColumnOptionsInternal
 from datafusion._internal import ParquetWriterOptions as ParquetWriterOptionsInternal
-from datafusion._internal import TableProvider as TableProviderInternal
 from datafusion.expr import Expr, SortExpr, sort_or_default
 from datafusion.plan import ExecutionPlan, LogicalPlan
 from datafusion.record_batch import RecordBatchStream
@@ -308,14 +307,8 @@ def __init__(self, df: DataFrameInternal) -> None:
         """
         self.df = df
 
-    def into_view(self) -> TableProviderInternal:
-        """Convert ``DataFrame`` into a ``TableProvider`` view for registration.
-
-        This is the preferred way to obtain a view for
-        :py:meth:`~datafusion.context.SessionContext.register_table`.
-        ``TableProvider.from_dataframe`` calls this method under the hood,
-        and the older ``TableProvider.from_view`` helper is deprecated.
-        """
+    def into_view(self) -> pa.Table:
+        """Convert DataFrame as a ViewTable which can be used in register_table."""
         return self.df.into_view()
 
     def __getitem__(self, key: str | list[str]) -> DataFrame:

python/tests/test_context.py

@@ -27,7 +27,6 @@
     SessionConfig,
     SessionContext,
     SQLOptions,
-    TableProvider,
     column,
     literal,
 )
@@ -331,40 +330,6 @@ def test_deregister_table(ctx, database):
     assert public.names() == {"csv1", "csv2"}
 
 
-def test_register_table_from_dataframe_into_view(ctx):
-    df = ctx.from_pydict({"a": [1, 2]})
-    provider = df.into_view()
-    ctx.register_table("view_tbl", provider)
-    result = ctx.sql("SELECT * FROM view_tbl").collect()
-    assert [b.to_pydict() for b in result] == [{"a": [1, 2]}]
-
-
-def test_table_provider_from_capsule(ctx):
-    df = ctx.from_pydict({"a": [1, 2]})
-    provider = df.into_view()
-    capsule = provider.__datafusion_table_provider__()
-    provider2 = TableProvider.from_capsule(capsule)
-    ctx.register_table("capsule_tbl", provider2)
-    result = ctx.sql("SELECT * FROM capsule_tbl").collect()
-    assert [b.to_pydict() for b in result] == [{"a": [1, 2]}]
-
-
-def test_table_provider_from_capsule_invalid():
-    with pytest.raises(Exception):  # noqa: B017
-        TableProvider.from_capsule(object())
-
-
-def test_register_table_with_dataframe_errors(ctx):
-    df = ctx.from_pydict({"a": [1]})
-    with pytest.raises(Exception) as exc_info:  # noqa: B017
-        ctx.register_table("bad", df)
-
-    assert (
-        str(exc_info.value)
-        == "Expected a Table or TableProvider. Convert DataFrames with \"DataFrame.into_view()\" or \"TableProvider.from_dataframe()\"."
-    )
-
-
 def test_register_dataset(ctx):
     # create a RecordBatch and register it as a pyarrow.dataset.Dataset
     batch = pa.RecordBatch.from_arrays(