UNPICK changes to review

Author: kosiew

docs/source/conf.py

@@ -83,9 +83,6 @@ def autoapi_skip_member_fn(app, what, name, obj, skip, options) -> bool:  # noqa
         # Duplicate modules (skip module-level docs to avoid duplication)
         ("module", "datafusion.col"),
         ("module", "datafusion.udf"),
-        # Private variables causing duplicate documentation
-        ("data", "datafusion.utils._PYARROW_DATASET_TYPES"),
-        ("variable", "datafusion.utils._PYARROW_DATASET_TYPES"),
         # Deprecated
         ("class", "datafusion.substrait.serde"),
         ("class", "datafusion.substrait.plan"),
@@ -94,28 +91,9 @@ def autoapi_skip_member_fn(app, what, name, obj, skip, options) -> bool:  # noqa
         ("method", "datafusion.context.SessionContext.tables"),
         ("method", "datafusion.dataframe.DataFrame.unnest_column"),
     ]
-    # Explicitly skip certain members listed above. These are either
-    # re-exports, duplicate module-level documentation, deprecated
-    # API surfaces, or private variables that would otherwise appear
-    # in the generated docs and cause confusing duplication.
-    # Keeping this explicit list avoids surprising entries in the
-    # AutoAPI output and gives us a single place to opt-out items
-    # when we intentionally hide them from the docs.
     if (what, name) in skip_contents:
         skip = True
 
-    # Skip private module-level names (those whose final component
-    # starts with an underscore) when AutoAPI is rendering data or
-    # variable entries. Many internal module-level constants are
-    # implementation details (for example private pyarrow dataset type
-    # mappings) that would otherwise be emitted as top-level "data"
-    # or "variable" docs. Filtering them here avoids noisy,
-    # duplicate, or implementation-specific entries in the public
-    # documentation while still allowing public members and types to
-    # be documented normally.
-    if name.split(".")[-1].startswith("_") and what in ("data", "variable"):
-        skip = True
-
     return skip
 
 

docs/source/contributor-guide/ffi.rst

@@ -34,7 +34,7 @@ as performant as possible and to utilize the features of DataFusion, you may dec
 your source in Rust and then expose it through `PyO3 <https://pyo3.rs>`_ as a Python library.
 
 At first glance, it may appear the best way to do this is to add the ``datafusion-python``
-crate as a dependency, produce a DataFusion table in Rust, and then register it with the
+crate as a dependency, provide a ``PyTable``, and then to register it with the 
 ``SessionContext``. Unfortunately, this will not work.
 
 When you produce your code as a Python library and it needs to interact with the DataFusion

docs/source/user-guide/data-sources.rst

@@ -152,26 +152,13 @@ as Delta Lake. This will require a recent version of
 .. code-block:: python
 
     from deltalake import DeltaTable
-    from datafusion import Table
 
     delta_table = DeltaTable("path_to_table")
-    table = Table.from_capsule(delta_table.__datafusion_table_provider__())
-    ctx.register_table("my_delta_table", table)
+    ctx.register_table_provider("my_delta_table", delta_table)
     df = ctx.table("my_delta_table")
     df.show()
 
-Objects that implement ``__datafusion_table_provider__`` are supported directly by
-:py:meth:`~datafusion.context.SessionContext.register_table`, making it easy to
-work with custom table providers from Python libraries such as Delta Lake.
-
-.. note::
-
-   :py:meth:`~datafusion.context.SessionContext.register_table_provider` is
-   deprecated. Use
-   :py:meth:`~datafusion.context.SessionContext.register_table` with a
-   :py:class:`~datafusion.Table` instead.
-
-On older versions of ``deltalake`` (prior to 0.22) you can use the
+On older versions of ``deltalake`` (prior to 0.22) you can use the 
 `Arrow DataSet <https://arrow.apache.org/docs/python/generated/pyarrow.dataset.Dataset.html>`_
 interface to import to DataFusion, but this does not support features such as filter push down
 which can lead to a significant performance difference.

docs/source/user-guide/io/table_provider.rst

@@ -39,47 +39,20 @@ A complete example can be found in the `examples folder <https://github.com/apac
         ) -> PyResult<Bound<'py, PyCapsule>> {
             let name = CString::new("datafusion_table_provider").unwrap();
 
-            let provider = Arc::new(self.clone());
-            let provider = FFI_TableProvider::new(provider, false, None);
+            let provider = Arc::new(self.clone())
+                .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+            let provider = FFI_TableProvider::new(Arc::new(provider), false);
 
             PyCapsule::new_bound(py, provider, Some(name.clone()))
         }
     }
 
-Once you have this library available, you can construct a
-:py:class:`~datafusion.Table` in Python and register it with the
-``SessionContext``. Tables can be created either from the PyCapsule exposed by your
-Rust provider or from an existing :py:class:`~datafusion.dataframe.DataFrame`.
-Call the provider's ``__datafusion_table_provider__()`` method to obtain the capsule
-before constructing a ``Table``. The ``Table.from_view()`` helper is
-deprecated; instead use ``Table.from_dataframe()`` or ``DataFrame.into_view()``.
-
-.. note::
-
-   :py:meth:`~datafusion.context.SessionContext.register_table_provider` is
-   deprecated. Use
-   :py:meth:`~datafusion.context.SessionContext.register_table` with the
-   resulting :py:class:`~datafusion.Table` instead.
+Once you have this library available, in python you can register your table provider
+to the ``SessionContext``.
 
 .. code-block:: python
 
-    from datafusion import SessionContext, Table
-
-    ctx = SessionContext()
     provider = MyTableProvider()
+    ctx.register_table_provider("my_table", provider)
 
-    capsule = provider.__datafusion_table_provider__()
-    capsule_table = Table.from_capsule(capsule)
-
-    df = ctx.from_pydict({"a": [1]})
-    view_table = Table.from_dataframe(df)
-    # or: view_table = df.into_view()
-
-    ctx.register_table("capsule_table", capsule_table)
-    ctx.register_table("view_table", view_table)
-
-    ctx.table("capsule_table").show()
-    ctx.table("view_table").show()
-
-Both ``Table.from_capsule()`` and ``Table.from_dataframe()`` create
-table providers that can be registered with the SessionContext using ``register_table()``.
+    ctx.table("my_table").show()

examples/datafusion-ffi-example/python/tests/_test_table_function.py

@@ -53,7 +53,7 @@ def test_ffi_table_function_call_directly():
     table_udtf = udtf(table_func, "my_table_func")
 
     my_table = table_udtf()
-    ctx.register_table("t", my_table)
+    ctx.register_table_provider("t", my_table)
     result = ctx.table("t").collect()
 
     assert len(result) == 2

examples/datafusion-ffi-example/python/tests/_test_table_provider.py

@@ -18,16 +18,14 @@
 from __future__ import annotations
 
 import pyarrow as pa
-from datafusion import SessionContext, TableProvider
+from datafusion import SessionContext
 from datafusion_ffi_example import MyTableProvider
 
 
 def test_table_loading():
     ctx = SessionContext()
     table = MyTableProvider(3, 2, 4)
-    ctx.register_table(
-        "t", TableProvider.from_capsule(table.__datafusion_table_provider__())
-    )
+    ctx.register_table_provider("t", table)
     result = ctx.table("t").collect()
 
     assert len(result) == 4

python/datafusion/__init__.py

@@ -28,16 +28,17 @@
 try:
     import importlib.metadata as importlib_metadata
 except ImportError:
-    import importlib_metadata  # type: ignore[import]
+    import importlib_metadata
 
-# Public submodules
 from . import functions, object_store, substrait, unparser
 
 # The following imports are okay to remain as opaque to the user.
-from ._internal import EXPECTED_PROVIDER_MSG, Config
+from ._internal import Config
 from .catalog import Catalog, Database, Table
 from .col import col, column
-from .common import DFSchema
+from .common import (
+    DFSchema,
+)
 from .context import (
     RuntimeEnvBuilder,
     SessionConfig,
@@ -46,7 +47,10 @@
 )
 from .dataframe import DataFrame, ParquetColumnOptions, ParquetWriterOptions
 from .dataframe_formatter import configure_formatter
-from .expr import Expr, WindowFrame
+from .expr import (
+    Expr,
+    WindowFrame,
+)
 from .io import read_avro, read_csv, read_json, read_parquet
 from .plan import ExecutionPlan, LogicalPlan
 from .record_batch import RecordBatch, RecordBatchStream
@@ -65,7 +69,6 @@
 __version__ = importlib_metadata.version(__name__)
 
 __all__ = [
-    "EXPECTED_PROVIDER_MSG",
     "Accumulator",
     "AggregateUDF",
     "Catalog",

python/datafusion/catalog.py

@@ -19,19 +19,14 @@
 
 from __future__ import annotations
 
-import warnings
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, Protocol
+from typing import TYPE_CHECKING, Protocol
 
 import datafusion._internal as df_internal
-from datafusion._internal import EXPECTED_PROVIDER_MSG
-from datafusion.utils import _normalize_table_provider
 
 if TYPE_CHECKING:
     import pyarrow as pa
 
-    from datafusion.context import TableProviderExportable
-
 try:
     from warnings import deprecated  # Python 3.13+
 except ImportError:
@@ -87,11 +82,7 @@ def database(self, name: str = "public") -> Schema:
         """Returns the database with the given ``name`` from this catalog."""
         return self.schema(name)
 
-    def register_schema(
-        self,
-        name: str,
-        schema: Schema | SchemaProvider | SchemaProviderExportable,
-    ) -> Schema | None:
+    def register_schema(self, name, schema) -> Schema | None:
         """Register a schema with this catalog."""
         if isinstance(schema, Schema):
             return self.catalog.register_schema(name, schema._raw_schema)
@@ -131,16 +122,11 @@ def table(self, name: str) -> Table:
         """Return the table with the given ``name`` from this schema."""
         return Table(self._raw_schema.table(name))
 
-    def register_table(
-        self, name: str, table: Table | TableProviderExportable | Any
-    ) -> None:
-        """Register a table or table provider in this schema.
-
-        Objects implementing ``__datafusion_table_provider__`` are also supported
-        and treated as table provider instances.
-        """
-        provider = _normalize_table_provider(table)
-        return self._raw_schema.register_table(name, provider)
+    def register_table(self, name, table) -> None:
+        """Register a table provider in this schema."""
+        if isinstance(table, Table):
+            return self._raw_schema.register_table(name, table.table)
+        return self._raw_schema.register_table(name, table)
 
     def deregister_table(self, name: str) -> None:
         """Deregister a table provider from this schema."""
@@ -152,101 +138,31 @@ class Database(Schema):
     """See `Schema`."""
 
 
-_InternalRawTable = df_internal.catalog.RawTable
-_InternalTableProvider = df_internal.TableProvider
-
-# Keep in sync with ``datafusion._internal.TableProvider.from_view``.
-_FROM_VIEW_WARN_STACKLEVEL = 2
-
-
 class Table:
-    """DataFusion table or table provider wrapper."""
-
-    __slots__ = ("_table",)
+    """DataFusion table."""
 
-    def __init__(
-        self,
-        table: _InternalRawTable | _InternalTableProvider | Table,
-    ) -> None:
-        """Wrap a low level table or table provider."""
-        if isinstance(table, Table):
-            table = table.table
-
-        if not isinstance(table, (_InternalRawTable, _InternalTableProvider)):
-            raise TypeError(EXPECTED_PROVIDER_MSG)
-
-        self._table = table
-
-    def __getattribute__(self, name: str) -> Any:
-        """Restrict provider-specific helpers to compatible tables."""
-        if name == "__datafusion_table_provider__":
-            table = object.__getattribute__(self, "_table")
-            if not hasattr(table, "__datafusion_table_provider__"):
-                raise AttributeError(name)
-        return object.__getattribute__(self, name)
+    def __init__(self, table: df_internal.catalog.RawTable) -> None:
+        """This constructor is not typically called by the end user."""
+        self.table = table
 
     def __repr__(self) -> str:
         """Print a string representation of the table."""
-        return repr(self._table)
+        return self.table.__repr__()
 
-    @property
-    def table(self) -> _InternalRawTable | _InternalTableProvider:
-        """Return the wrapped low level table object."""
-        return self._table
-
-    @classmethod
-    def from_dataset(cls, dataset: pa.dataset.Dataset) -> Table:
-        """Turn a :mod:`pyarrow.dataset` ``Dataset`` into a :class:`Table`."""
-        return cls(_InternalRawTable.from_dataset(dataset))
-
-    @classmethod
-    def from_capsule(cls, capsule: Any) -> Table:
-        """Create a :class:`Table` from a PyCapsule exported provider."""
-        provider = _InternalTableProvider.from_capsule(capsule)
-        return cls(provider)
-
-    @classmethod
-    def from_dataframe(cls, df: Any) -> Table:
-        """Create a :class:`Table` from tabular data."""
-        from datafusion.dataframe import DataFrame as DataFrameWrapper
-
-        dataframe = df if isinstance(df, DataFrameWrapper) else DataFrameWrapper(df)
-        return dataframe.into_view()
-
-    @classmethod
-    def from_view(cls, df: Any) -> Table:
-        """Deprecated helper for constructing tables from views."""
-        from datafusion.dataframe import DataFrame as DataFrameWrapper
-
-        if isinstance(df, DataFrameWrapper):
-            df = df.df
-
-        provider = _InternalTableProvider.from_view(df)
-        warnings.warn(
-            "Table.from_view is deprecated; use DataFrame.into_view or "
-            "Table.from_dataframe instead.",
-            category=DeprecationWarning,
-            stacklevel=_FROM_VIEW_WARN_STACKLEVEL,
-        )
-        return cls(provider)
+    @staticmethod
+    def from_dataset(dataset: pa.dataset.Dataset) -> Table:
+        """Turn a pyarrow Dataset into a Table."""
+        return Table(df_internal.catalog.RawTable.from_dataset(dataset))
 
     @property
     def schema(self) -> pa.Schema:
         """Returns the schema associated with this table."""
-        return self._table.schema
+        return self.table.schema
 
     @property
     def kind(self) -> str:
         """Returns the kind of table."""
-        return self._table.kind
-
-    def __datafusion_table_provider__(self) -> Any:
-        """Expose the wrapped provider for FFI integrations."""
-        exporter = getattr(self._table, "__datafusion_table_provider__", None)
-        if exporter is None:
-            msg = "Underlying object does not export __datafusion_table_provider__()"
-            raise AttributeError(msg)
-        return exporter()
+        return self.table.kind
 
 
 class CatalogProvider(ABC):
@@ -303,19 +219,14 @@ def table(self, name: str) -> Table | None:
         """Retrieve a specific table from this schema."""
         ...
 
-    def register_table(  # noqa: B027
-        self, name: str, table: Table | TableProviderExportable | Any
-    ) -> None:
-        """Add a table to this schema.
+    def register_table(self, name: str, table: Table) -> None:  # noqa: B027
+        """Add a table from this schema.
 
         This method is optional. If your schema provides a fixed list of tables, you do
         not need to implement this method.
-
-        Objects implementing ``__datafusion_table_provider__`` are also supported
-        and treated as table provider instances.
         """
 
-    def deregister_table(self, name: str, cascade: bool) -> None:  # noqa: B027
+    def deregister_table(self, name, cascade: bool) -> None:  # noqa: B027
         """Remove a table from this schema.
 
         This method is optional. If your schema provides a fixed list of tables, you do