Fix concjunction OR (#465)

Fixes #460 

A bug in the conjunction OR pushdown for pyarrow resulted in a too
restrictive filter if one of the branches couldn't be translated.

Author: evertlammerts

src/duckdb_py/arrow/pyarrow_filter_pushdown.cpp

@@ -236,15 +236,16 @@ py::object TransformFilterRecursive(TableFilter &filter, vector<string> column_r
 		auto constant_field = field(py::tuple(py::cast(column_ref)));
 		return constant_field.attr("is_valid")();
 	}
-	//! We do not pushdown or conjunctions yet
 	case TableFilterType::CONJUNCTION_OR: {
 		auto &or_filter = filter.Cast<ConjunctionOrFilter>();
 		py::object expression = py::none();
 		for (idx_t i = 0; i < or_filter.child_filters.size(); i++) {
 			auto &child_filter = *or_filter.child_filters[i];
 			py::object child_expression = TransformFilterRecursive(child_filter, column_ref, timezone_config, type);
 			if (child_expression.is(py::none())) {
-				continue;
+				// An OR branch that can't be translated (e.g. DYNAMIC_FILTER) means the pushed-down
+				// predicate would be stricter than the engine intends — fall back to no pushdown.
+				return py::none();
 			}
 			if (expression.is(py::none())) {
 				expression = std::move(child_expression);

tests/fast/arrow/test_filter_pushdown.py

@@ -1021,6 +1021,26 @@ def test_dynamic_filter(self, duckdb_cursor):
         res = duckdb_cursor.sql("SELECT a FROM t ORDER BY a LIMIT 11").fetchall()
         assert len(res) == 11
 
+    def test_dynamic_filter_nulls_first_pyarrow(self, duckdb_cursor):
+        # Regression for #460(a): TOP_N with ASC NULLS FIRST pushes an
+        # OPTIONAL(x IS NULL OR DYNAMIC_FILTER(x)) into the arrow scan. The
+        # pyarrow translation must NOT collapse the OR by dropping the
+        # untranslatable DYNAMIC_FILTER branch — doing so produces a
+        # stricter (`field("x").is_null()`) predicate that drops every row.
+        t = pa.Table.from_pydict({"x": pa.array([3, 1, 2], type=pa.int32())})
+        duckdb_cursor.register("src", t)
+        res = duckdb_cursor.sql("SELECT * FROM src ORDER BY x ASC NULLS FIRST LIMIT 1").fetchall()
+        assert res == [(1,)]
+
+    def test_dynamic_filter_nulls_first_polars_dataframe(self, duckdb_cursor):
+        # pl.DataFrame is materialized to a pyarrow.Table before scanning,
+        # so it exercises PyarrowFilterPushdown the same way pa.Table does.
+        pl = pytest.importorskip("polars")
+        df = pl.DataFrame({"x": [3, 1, 2]})
+        duckdb_cursor.register("src", df)
+        res = duckdb_cursor.sql("SELECT * FROM src ORDER BY x ASC NULLS FIRST LIMIT 1").fetchall()
+        assert res == [(1,)]
+
     def test_binary_view_filter(self, duckdb_cursor):
         """Filters on a view column work (without pushdown because pyarrow does not support view filters yet)."""
         table = pa.table({"col": pa.array([b"abc", b"efg"], type=pa.binary_view())})

tests/fast/arrow/test_polars_filter_pushdown.py

@@ -147,6 +147,20 @@ def test_optional_filter(self):
         result = duckdb.sql("SELECT * FROM lf WHERE a = 1 OR a = 3").fetchall()
         assert sorted(result) == [(1,), (3,)]
 
+    ##### DYNAMIC_FILTER via TOP_N (issue #460(a))
+
+    def test_top_n_nulls_first_includes_min(self):
+        """ORDER BY x ASC NULLS FIRST LIMIT 1 pushes OPTIONAL(IS_NULL OR DYNAMIC_FILTER) into the scan.
+
+        The OR branch must not be partially translated: dropping the
+        untranslatable DYNAMIC_FILTER child would leave just IS_NULL and
+        silently discard every non-null row. See pyarrow_filter_pushdown
+        sibling regression test in test_filter_pushdown.py.
+        """
+        lf = pl.LazyFrame({"x": [3, 1, 2]})
+        result = duckdb.sql("SELECT * FROM lf ORDER BY x ASC NULLS FIRST LIMIT 1").fetchall()
+        assert result == [(1,)]
+
     ##### Produce path, no filters
 
     def test_unfiltered_scan(self):