Integrated with Identifier and MaxLogicalType, and using SetChildCardinality

Author: evertlammerts

src/duckdb_py/arrow/filter_pushdown_visitor.cpp

@@ -28,11 +28,11 @@ bool ValueIsNan(const Value &value) {
 // `struct_extract` chains. Anything else throws NotImplementedException —
 // that gives the OPTIONAL_FILTER catch point a chance to swallow it.
 struct ResolvedColumn {
-	vector<string> path;
+	vector<Identifier> path;
 	const ArrowType *leaf_type;
 };
 
-ResolvedColumn ResolveColumn(const Expression &expr, const vector<string> &root_path, const ArrowType *root_type) {
+ResolvedColumn ResolveColumn(const Expression &expr, const vector<Identifier> &root_path, const ArrowType *root_type) {
 	if (expr.GetExpressionClass() == ExpressionClass::BOUND_REF) {
 		return {root_path, root_type};
 	}
@@ -47,8 +47,8 @@ ResolvedColumn ResolveColumn(const Expression &expr, const vector<string> &root_
 		                              ExpressionTypeToString(expr.GetExpressionType()));
 	}
 	// Recurse innermost-first so names accumulate root → leaf.
-	auto inner = ResolveColumn(*func.children[0], root_path, root_type);
-	inner.path.push_back(StructType::GetChildName(func.children[0]->GetReturnType(), child_idx));
+	auto inner = ResolveColumn(*func.GetChildren()[0], root_path, root_type);
+	inner.path.push_back(StructType::GetChildName(func.GetChildren()[0]->GetReturnType(), child_idx));
 	if (inner.leaf_type) {
 		inner.leaf_type = &inner.leaf_type->GetTypeInfo<ArrowStructInfo>().GetChild(child_idx);
 	}
@@ -66,8 +66,8 @@ py::object EmitCompare(FilterBackend &backend, ExpressionType op, py::object col
 
 } // anonymous namespace
 
-py::object TransformExpression(const Expression &expression, const vector<string> &column_path, FilterBackend &backend,
-                               const ArrowType *arrow_type, const string &timezone_config) {
+py::object TransformExpression(const Expression &expression, const vector<Identifier> &column_path,
+                               FilterBackend &backend, const ArrowType *arrow_type, const string &timezone_config) {
 	auto expression_class = expression.GetExpressionClass();
 	auto expression_type = expression.GetExpressionType();
 
@@ -93,7 +93,7 @@ py::object TransformExpression(const Expression &expression, const vector<string
 
 			auto resolved = ResolveColumn(*column_side, column_path, arrow_type);
 			auto col = backend.MakeColumnRef(resolved.path);
-			return EmitCompare(backend, expression_type, std::move(col), constant_side->value, resolved.leaf_type,
+			return EmitCompare(backend, expression_type, std::move(col), constant_side->GetValue(), resolved.leaf_type,
 			                   timezone_config);
 		}
 
@@ -102,20 +102,22 @@ py::object TransformExpression(const Expression &expression, const vector<string
 		// filters no longer have dedicated TableFilter subtypes. They arrive as scalar
 		// function wrappers inside the ExpressionFilter expression tree (see
 		// table_filter_functions.hpp).
-		const auto &func_name = bound_function_expression.function.GetName();
+		const auto &func_name = bound_function_expression.Function().GetName();
 
 		// OPTIONAL / SELECTIVITY_OPTIONAL wrap a child predicate that lives in `bind_info`
 		// (their `children` hold only a placeholder column ref). An optional filter is never
 		// required for correctness, so if its child can't be translated we push nothing for
 		// it rather than failing the whole scan.
 		if (func_name == OptionalFilterScalarFun::NAME || func_name == SelectivityOptionalFilterScalarFun::NAME) {
 			optional_ptr<const Expression> child;
-			if (bound_function_expression.bind_info) {
+			if (bound_function_expression.BindInfo()) {
 				if (func_name == OptionalFilterScalarFun::NAME) {
-					child =
-					    bound_function_expression.bind_info->Cast<OptionalFilterFunctionData>().child_filter_expr.get();
+					child = bound_function_expression.BindInfo()
+					            ->Cast<OptionalFilterFunctionData>()
+					            .child_filter_expr.get();
 				} else {
-					child = bound_function_expression.bind_info->Cast<SelectivityOptionalFilterFunctionData>()
+					child = bound_function_expression.BindInfo()
+					            ->Cast<SelectivityOptionalFilterFunctionData>()
 					            .child_filter_expr.get();
 				}
 			}
@@ -140,24 +142,24 @@ py::object TransformExpression(const Expression &expression, const vector<string
 	if (expression_class == ExpressionClass::BOUND_OPERATOR) {
 		auto &op_expr = expression.Cast<BoundOperatorExpression>();
 		if (expression_type == ExpressionType::OPERATOR_IS_NULL) {
-			auto resolved = ResolveColumn(*op_expr.children[0], column_path, arrow_type);
+			auto resolved = ResolveColumn(*op_expr.GetChildren()[0], column_path, arrow_type);
 			auto col = backend.MakeColumnRef(resolved.path);
 			return backend.IsNull(std::move(col));
 		}
 		if (expression_type == ExpressionType::OPERATOR_IS_NOT_NULL) {
-			auto resolved = ResolveColumn(*op_expr.children[0], column_path, arrow_type);
+			auto resolved = ResolveColumn(*op_expr.GetChildren()[0], column_path, arrow_type);
 			auto col = backend.MakeColumnRef(resolved.path);
 			return backend.IsNotNull(std::move(col));
 		}
 		if (expression_type == ExpressionType::COMPARE_IN) {
-			auto resolved = ResolveColumn(*op_expr.children[0], column_path, arrow_type);
+			auto resolved = ResolveColumn(*op_expr.GetChildren()[0], column_path, arrow_type);
 			auto col = backend.MakeColumnRef(resolved.path);
 			vector<Value> values;
-			for (idx_t i = 1; i < op_expr.children.size(); i++) {
-				auto &const_expr = op_expr.children[i]->Cast<BoundConstantExpression>();
-				values.push_back(const_expr.value);
+			for (idx_t i = 1; i < op_expr.GetChildren().size(); i++) {
+				auto &const_expr = op_expr.GetChildren()[i]->Cast<BoundConstantExpression>();
+				values.push_back(const_expr.GetValue());
 			}
-			auto col_type = op_expr.children[0]->GetReturnType();
+			auto col_type = op_expr.GetChildren()[0]->GetReturnType();
 			return backend.IsIn(std::move(col), values, col_type, timezone_config);
 		}
 	}
@@ -167,9 +169,9 @@ py::object TransformExpression(const Expression &expression, const vector<string
 			const bool is_and = expression_type == ExpressionType::CONJUNCTION_AND;
 			auto &conj_expr = expression.Cast<BoundConjunctionExpression>();
 			py::object result = py::none();
-			for (idx_t i = 0; i < conj_expr.children.size(); i++) {
+			for (idx_t i = 0; i < conj_expr.GetChildren().size(); i++) {
 				py::object child_expression =
-				    TransformExpression(*conj_expr.children[i], column_path, backend, arrow_type, timezone_config);
+				    TransformExpression(*conj_expr.GetChildren()[i], column_path, backend, arrow_type, timezone_config);
 				if (child_expression.is(py::none())) {
 					if (is_and) {
 						// A conjunct we can't push can simply be dropped: the remaining AND
@@ -198,7 +200,7 @@ py::object TransformExpression(const Expression &expression, const vector<string
 	                              ExpressionClassToString(expression_class));
 }
 
-py::object TransformFilter(const TableFilter &filter, vector<string> column_path, FilterBackend &backend,
+py::object TransformFilter(const TableFilter &filter, const vector<Identifier> &column_path, FilterBackend &backend,
                            const ArrowType *arrow_type, const string &timezone_config) {
 	switch (filter.filter_type) {
 	case TableFilterType::EXPRESSION_FILTER: {

src/duckdb_py/arrow/polars_filter_pushdown.cpp

@@ -14,12 +14,12 @@ struct PolarsBackend : public FilterBackend {
 	    : client_properties(client_properties_p), import_cache(*DuckDBPyConnection::ImportCache()) {
 	}
 
-	py::object MakeColumnRef(const vector<string> &path) override {
+	py::object MakeColumnRef(const vector<Identifier> &path) override {
 		// pl.col(path[0]).struct.field(path[1]).struct.field(...) — polars supports arbitrary
 		// chaining for nested struct access, verified empirically up to 3 levels.
 		py::object col = import_cache.polars.col()(path[0]);
 		for (idx_t i = 1; i < path.size(); i++) {
-			col = col.attr("struct").attr("field")(path[i]);
+			col = col.attr("struct").attr("field")(path[i].GetIdentifierName());
 		}
 		return col;
 	}
@@ -131,7 +131,7 @@ py::object PolarsFilterPushdown::TransformFilter(const TableFilterSet &filter_co
 		auto &column_name = columns[column_idx];
 		D_ASSERT(columns.find(column_idx) != columns.end());
 
-		vector<string> column_path = {column_name};
+		vector<Identifier> column_path = {Identifier(column_name)};
 		// Polars does not need ArrowType information — `nullptr` here propagates through the
 		// shared walker; the PolarsBackend ignores the parameter in MakeScalar.
 		py::object child_expression = duckdb::TransformFilter(entry.Filter(), std::move(column_path), backend, nullptr,

src/duckdb_py/arrow/pyarrow_filter_pushdown.cpp

@@ -187,8 +187,11 @@ struct PyArrowBackend : public FilterBackend {
 		dataset_scalar = import_cache.pyarrow.dataset().attr("scalar");
 	}
 
-	py::object MakeColumnRef(const vector<string> &path) override {
-		return field_factory(py::tuple(py::cast(path)));
+	py::object MakeColumnRef(const vector<Identifier> &path) override {
+		vector<string> str_path;
+		std::transform(path.begin(), path.end(), str_path.begin(),
+		               [](const Identifier &segment) { return segment.GetIdentifierName(); });
+		return field_factory(py::tuple(py::cast(str_path)));
 	}
 
 	py::object MakeScalar(const Value &v, const ArrowType *arrow_type, const string &timezone_config) override {
@@ -283,7 +286,7 @@ py::object PyArrowFilterPushdown::TransformFilter(TableFilterSet &filter_collect
 		auto &column_name = columns[column_idx];
 		D_ASSERT(columns.find(column_idx) != columns.end());
 
-		vector<string> column_path = {column_name};
+		vector<Identifier> column_path = {Identifier(column_name)};
 		auto &arrow_type = arrow_table.GetColumns().at(filter_to_col.at(column_idx));
 		py::object child_expression = duckdb::TransformFilter(entry.Filter(), std::move(column_path), backend,
 		                                                      arrow_type.get(), config.time_zone);

src/duckdb_py/duckdb_python.cpp

@@ -9,7 +9,6 @@
 #include "duckdb_python/pystatement.hpp"
 #include "duckdb_python/pyrelation.hpp"
 #include "duckdb_python/expression/pyexpression.hpp"
-#include "duckdb_python/pyresult.hpp"
 #include "duckdb_python/pybind11/exceptions.hpp"
 #include "duckdb_python/typing.hpp"
 #include "duckdb_python/functional.hpp"
@@ -22,8 +21,6 @@
 #include "duckdb/common/enums/statement_type.hpp"
 #include "duckdb/common/adbc/adbc-init.hpp"
 
-#include "duckdb.hpp"
-
 #ifndef DUCKDB_PYTHON_LIB_NAME
 #define DUCKDB_PYTHON_LIB_NAME _duckdb
 #endif
@@ -126,7 +123,7 @@ static void InitializeConnectionMethods(py::module_ &m) {
 	    py::arg("connection") = py::none());
 	m.def(
 	    "get_profiling_information",
-	    [](const py::str &format, shared_ptr<DuckDBPyConnection> conn = nullptr) {
+	    [](const std::string &format, shared_ptr<DuckDBPyConnection> conn = nullptr) {
 		    if (!conn) {
 			    conn = DuckDBPyConnection::DefaultConnection();
 		    }

src/duckdb_py/include/duckdb_python/arrow/filter_pushdown_visitor.hpp

@@ -38,7 +38,7 @@ struct FilterBackend {
 	// Build a column expression from an accumulated path. `path` always has
 	// at least one element (the top-level column). For nested struct
 	// references the path accumulates one entry per `struct_extract`.
-	virtual py::object MakeColumnRef(const vector<string> &path) = 0;
+	virtual py::object MakeColumnRef(const vector<Identifier> &path) = 0;
 
 	// Convert a DuckDB Value to a backend-native Python scalar. `arrow_type`
 	// may be nullptr for backends that don't need Arrow type information
@@ -77,7 +77,7 @@ struct FilterBackend {
 // - `arrow_type` is the ArrowType for the current path leaf (nullable for
 //    backends that don't track Arrow types).
 // - Returns `py::none()` if no part of the filter could be pushed.
-py::object TransformFilter(const TableFilter &filter, vector<string> column_path, FilterBackend &backend,
+py::object TransformFilter(const TableFilter &filter, const vector<Identifier> &column_path, FilterBackend &backend,
                            const ArrowType *arrow_type, const string &timezone_config);
 
 // Walk a bound Expression tree (the contents of an `ExpressionFilter`) and emit
@@ -88,7 +88,7 @@ py::object TransformFilter(const TableFilter &filter, vector<string> column_path
 // and the internal runtime filter functions (dynamic / bloom / perfect-hash-join
 // / prefix-range, which are skipped). Returns `py::none()` for an optional or
 // runtime filter that can't be pushed.
-py::object TransformExpression(const Expression &expression, const vector<string> &column_path, FilterBackend &backend,
-                               const ArrowType *arrow_type, const string &timezone_config);
+py::object TransformExpression(const Expression &expression, const vector<Identifier> &column_path,
+                               FilterBackend &backend, const ArrowType *arrow_type, const string &timezone_config);
 
 } // namespace duckdb

src/duckdb_py/include/duckdb_python/numpy/numpy_bind.hpp

@@ -9,7 +9,7 @@ struct PandasColumnBindData;
 class ClientContext;
 
 struct NumpyBind {
-	static void Bind(const ClientContext &config, py::handle df, vector<PandasColumnBindData> &out,
+	static void Bind(ClientContext &config, py::handle df, vector<PandasColumnBindData> &out,
 	                 vector<LogicalType> &return_types, vector<string> &names);
 };
 

src/duckdb_py/include/duckdb_python/numpy/numpy_scan.hpp

@@ -8,8 +8,9 @@ namespace duckdb {
 struct PandasColumnBindData;
 
 struct NumpyScan {
-	static void Scan(PandasColumnBindData &bind_data, idx_t count, idx_t offset, Vector &out);
-	static void ScanObjectColumn(PyObject **col, idx_t stride, idx_t count, idx_t offset, Vector &out);
+	static void Scan(ClientContext &context, PandasColumnBindData &bind_data, idx_t count, idx_t offset, Vector &out);
+	static void ScanObjectColumn(ClientContext &context, PyObject **col, idx_t stride, idx_t count, idx_t offset,
+	                             Vector &out);
 };
 
 } // namespace duckdb

src/duckdb_py/include/duckdb_python/pandas/pandas_analyzer.hpp

@@ -12,14 +12,13 @@
 #include "duckdb/main/config.hpp"
 #include "duckdb_python/pybind11/pybind_wrapper.hpp"
 #include "duckdb_python/pybind11/gil_wrapper.hpp"
-#include "duckdb_python/numpy/numpy_type.hpp"
 #include "duckdb_python/python_conversion.hpp"
 
 namespace duckdb {
 
 class PandasAnalyzer {
 public:
-	explicit PandasAnalyzer(const ClientContext &context) {
+	explicit PandasAnalyzer(ClientContext &context) : context(context) {
 		analyzed_type = LogicalType::SQLNULL;
 
 		Value result;
@@ -48,6 +47,7 @@ class PandasAnalyzer {
 	PythonGILWrapper gil;
 	//! The resulting analyzed type
 	LogicalType analyzed_type;
+	ClientContext &context;
 };
 
 } // namespace duckdb

src/duckdb_py/include/duckdb_python/pandas/pandas_bind.hpp

@@ -27,7 +27,7 @@ struct PandasColumnBindData {
 };
 
 struct Pandas {
-	static void Bind(const ClientContext &config, py::handle df, vector<PandasColumnBindData> &out,
+	static void Bind(ClientContext &config, py::handle df, vector<PandasColumnBindData> &out,
 	                 vector<LogicalType> &return_types, vector<string> &names);
 };
 

src/duckdb_py/include/duckdb_python/pandas/pandas_scan.hpp

@@ -51,7 +51,8 @@ struct PandasScanFunction : public TableFunction {
 	// Helper function that transform pandas df names to make them work with our binder
 	static py::object PandasReplaceCopiedNames(const py::object &original_df);
 
-	static void PandasBackendScanSwitch(PandasColumnBindData &bind_data, idx_t count, idx_t offset, Vector &out);
+	static void PandasBackendScanSwitch(ClientContext &context, PandasColumnBindData &bind_data, idx_t count,
+	                                    idx_t offset, Vector &out);
 
 	static void PandasSerialize(Serializer &serializer, const optional_ptr<FunctionData> bind_data,
 	                            const TableFunction &function);