Update pyarrow.Table docs and tests

Signed-off-by: Andrew Stein <steinlink@gmail.com>

Author: texodus

docs/md/SUMMARY.md

@@ -54,6 +54,7 @@
 - [Overview](./explanation/python.md)
 - [Installation](./how_to/python/installation.md)
 - [Loading data into a `Table`](./how_to/python/table.md)
+    - [`pandas`, `polars` and `pyarrow` integration](./how_to/python/table_data.md)
 - [Callbacks and events](./how_to/python/callbacks.md)
 - [Multithreading](./how_to/python/multithreading.md)
 - [Hosting a WebSocket server](./how_to/python/websocket.md)

docs/md/how_to/python/table_data.md

@@ -0,0 +1,81 @@
+# DataFrame and Arrow Compatibility
+
+`perspective-python` accepts a `Table` constructor argument from any of the
+common Python columnar data libraries. In all three cases, `perspective.table`
+(and `Table.update()`) consume the input directly — there is no need to
+serialize to Apache Arrow IPC bytes yourself. However, note is
+still the most efficient way to bulk load data into `Table`.
+
+## PyArrow
+
+```python
+import pyarrow as pa
+import perspective
+
+arrow_table = pa.table({
+    "int": pa.array([1, 2, 3], type=pa.int64()),
+    "float": pa.array([1.5, 2.5, 3.5], type=pa.float64()),
+    "string": pa.array(["a", "b", "c"], type=pa.string()),
+})
+
+table = perspective.table(arrow_table)
+```
+
+The same applies to `Table.update()`:
+
+```python
+table.update(arrow_table)
+```
+
+If you have Arrow data already in IPC format (e.g. read from disk, received
+over the wire, or produced by another tool), pass the raw `bytes` directly —
+both stream and file formats are auto-detected:
+
+```python
+with open("data.arrow", "rb") as f:
+    table = perspective.table(f.read())
+```
+
+## Polars
+
+```python
+import polars as pl
+import perspective
+
+df = pl.DataFrame({
+    "a": [1, 2, 3, 4, 5],
+    "b": ["x", "y", "z", "x", "y"],
+})
+
+table = perspective.table(df)
+```
+
+Internally, the `DataFrame` is converted to a `pyarrow.Table` before
+ingestion, so Polars columns inherit the Arrow type mapping above.
+
+See also Perspective [Virtual Server support for `polars.DataFrame`](./virtual_server/polars.md)
+
+## Pandas
+
+`pandas.DataFrame` is supported via `pyarrow.Table.from_pandas`, which
+dictates behavior including type support — see the
+[pyarrow pandas docs](https://arrow.apache.org/docs/python/pandas.html) for
+details on which pandas dtypes round-trip cleanly.
+
+```python
+from datetime import date, datetime
+import numpy as np
+import pandas as pd
+import perspective
+
+data = pd.DataFrame({
+    "int": np.arange(100),
+    "float": [i * 1.5 for i in range(100)],
+    "bool": [True for i in range(100)],
+    "date": [date.today() for i in range(100)],
+    "datetime": [datetime.now() for i in range(100)],
+    "string": [str(i) for i in range(100)],
+})
+
+table = perspective.table(data, index="float")
+```

rust/perspective-python/perspective/tests/conftest.py

@@ -109,26 +109,17 @@ def make_arrow(names, data, types=None, legacy=False):
 
     @staticmethod
     def make_arrow_from_pandas(df, schema=None, legacy=False):
-        """Create an arrow binary from a Pandas dataframe.
+        """Create a pyarrow Table from a Pandas dataframe.
 
         Args:
             df (:obj:`pandas.DataFrame`)
             schema (:obj:`pyarrow.Schema`)
-            legacy (bool): if True, use legacy IPC format (pre-pyarrow 0.15). Defaults to False.
+            legacy (bool): unused; retained for backwards compatibility.
 
         Returns:
-            bytes : a bytes object containing the arrow-serialized output.
+            pyarrow.Table
         """
-        stream = pa.BufferOutputStream()
-        table = pa.Table.from_pandas(df, schema=schema)
-
-        writer = pa.RecordBatchStreamWriter(
-            stream, table.schema
-        )
-
-        writer.write_table(table)
-        writer.close()
-        return stream.getvalue().to_pybytes()
+        return pa.Table.from_pandas(df, schema=schema)
 
     @staticmethod
     def make_dictionary_arrow(names, data, types=None, legacy=False):

rust/perspective-python/perspective/tests/table/test_table_arrow.py

@@ -15,10 +15,8 @@
 import pandas as pd
 from perspective.tests.conftest import Util
 import pyarrow as pa
-import pyarrow.ipc as ipc
 from datetime import date, datetime
 import perspective as psp
-import io
 
 
 client = psp.Server().new_local_client()
@@ -49,14 +47,23 @@
 
 ALL_INTEGERS_TABLE = pa.Table.from_pydict(ALL_INTEGERS_DATA)
 
-bytes_io = io.BytesIO()
-with ipc.new_stream(bytes_io, ALL_INTEGERS_TABLE.schema) as stream:
-    stream.write_table(ALL_INTEGERS_TABLE)
-ALL_INTEGERS_ARROW = bytes_io.getvalue()
 
 class TestTableArrow(object):
     def test_table_with_integer_types(self):
-        tbl = Table(ALL_INTEGERS_ARROW)
+        tbl = Table(ALL_INTEGERS_TABLE)
+        assert tbl.size() == 3
+        assert tbl.schema() == {
+            "int8": "integer",
+            "int16": "integer",
+            "int32": "integer",
+            "int64": "integer",
+            "uint8": "integer",
+            "uint16": "integer",
+            "uint32": "integer",
+            "uint64": "integer",
+            "float32": "float",
+            "float64": "float",
+        }
         for k, values in ALL_INTEGERS_DATA.items():
             v = tbl.view(filter=[[k, "==", values[0].as_py()]])
             assert len(v.to_json()) == 1
@@ -481,17 +488,7 @@ def test_table_arrow_loads_arrow_from_df_with_nan(self):
 
         assert arrow_table["a"].null_count == 4
 
-        # write arrow to stream
-        stream = pa.BufferOutputStream()
-        writer = pa.RecordBatchStreamWriter(
-            stream, arrow_table.schema
-        )
-        writer.write_table(arrow_table)
-        writer.close()
-        arrow = stream.getvalue().to_pybytes()
-
-        # load
-        tbl = Table(arrow)
+        tbl = Table(arrow_table)
         assert tbl.size() == 8
 
         # check types

rust/perspective-python/perspective/tests/table/test_update.py

@@ -32,16 +32,7 @@ def test_update_with_missing_or_null_values(self):
 
         arrow_table = pa.Table.from_pandas(data, preserve_index=False)
 
-        # write arrow to stream
-        stream = pa.BufferOutputStream()
-        writer = pa.RecordBatchStreamWriter(
-            stream, arrow_table.schema
-        )
-        writer.write_table(arrow_table)
-        writer.close()
-        arrow = stream.getvalue().to_pybytes()
-
-        tbl.update(arrow)
+        tbl.update(arrow_table)
         assert tbl.size() == 2
         assert tbl.view().to_records() == [{"a": "1", "b": ""}, {"a": "3", "b": "4"}]