Skip to content

Commit c1f4763

Browse files
committed
Fix arrow binary view canary
1 parent cd110ee commit c1f4763

1 file changed

Lines changed: 30 additions & 7 deletions

File tree

tests/fast/arrow/test_filter_pushdown.py

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -747,35 +747,58 @@ def test_2145_parquet_glob_through_arrow(self, duckdb_cursor, tmp_path):
747747
# ===========================================================================
748748

749749

750+
# pyarrow view type filter pushdown canaries.
751+
# No released pyarrow fully executes view type filters (compare plus array_filter/take); checked
752+
# against 25.0.0. These stay xfail until one does. They run .to_table() so scanner construction
753+
# alone does not flip them: pyarrow accepts a binary_view predicate at build time but still fails
754+
# inside array_filter at execution. When one XPASSes: bump its version below, enable view type
755+
# filter pushdown, and drop the matching TestUnsupportedTypes fallback. See the view type filter
756+
# pushdown tracking issue.
757+
_PA_VERSION = Version(pa.__version__)
758+
_PYARROW_STRING_VIEW_FILTER_VERSION = Version("99")
759+
_PYARROW_BINARY_VIEW_FILTER_VERSION = Version("99")
760+
761+
750762
class TestCanaries:
751763
"""Markers for behaviours we expect to change upstream eventually."""
752764

753765
# ----- pyarrow capabilities ----------------------------------------
754766

755767
@pytest.mark.xfail(
768+
_PA_VERSION < _PYARROW_STRING_VIEW_FILTER_VERSION,
756769
raises=pa_lib.ArrowNotImplementedError,
757-
reason="pyarrow does not yet implement string_view filter compare kernels",
770+
reason="pyarrow does not execute string_view filters fully (equal kernel)",
758771
strict=True,
759772
)
760773
def test_pyarrow_gains_string_view_filter_support(self):
761-
"""When pyarrow adds string_view comparison kernels this will xpass.
774+
"""XPASSes when pyarrow executes a string_view filter fully.
762775
763-
At that point we should remove the post-scan fallback in TestUnsupportedTypes.
776+
Runs .to_table() so it reflects execution, not just scanner construction. When it
777+
XPASSes: bump the version, enable view type pushdown, drop the TestUnsupportedTypes
778+
fallback.
764779
"""
765780
filter_expr = pa_ds.field("col") == pa_ds.scalar("val1")
766781
table = pa.table({"col": pa.array(["val1", "val2"], type=pa.string_view())})
767-
pa_ds.dataset(table).scanner(columns=["col"], filter=filter_expr)
782+
result = pa_ds.dataset(table).scanner(columns=["col"], filter=filter_expr).to_table()
783+
assert result.num_rows == 1
768784

769785
@pytest.mark.xfail(
786+
_PA_VERSION < _PYARROW_BINARY_VIEW_FILTER_VERSION,
770787
raises=pa_lib.ArrowNotImplementedError,
771-
reason="pyarrow does not yet implement binary_view filter compare kernels",
788+
reason="pyarrow does not execute binary_view filters fully (array_filter kernel)",
772789
strict=True,
773790
)
774791
def test_pyarrow_gains_binary_view_filter_support(self):
775-
"""When pyarrow adds binary_view comparison kernels this will xpass."""
792+
"""XPASSes when pyarrow executes a binary_view filter fully.
793+
794+
Runs .to_table() so it reflects execution, not just scanner construction. When it
795+
XPASSes: bump the version, enable view type pushdown, drop the TestUnsupportedTypes
796+
fallback.
797+
"""
776798
filter_expr = pa_ds.field("col") == pa_ds.scalar(pa.scalar(b"bin1", pa.binary_view()))
777799
table = pa.table({"col": pa.array([b"bin1", b"bin2"], type=pa.binary_view())})
778-
pa_ds.dataset(table).scanner(columns=["col"], filter=filter_expr)
800+
result = pa_ds.dataset(table).scanner(columns=["col"], filter=filter_expr).to_table()
801+
assert result.num_rows == 1
779802

780803
# ----- DuckDB optimizer decisions we expect to change --------------
781804

0 commit comments

Comments
 (0)