added sqlnull support and fixed explain

Author: evertlammerts

_duckdb-stubs/__init__.pyi

@@ -537,7 +537,9 @@ class DuckDBPyRelation:
     def distinct(self) -> DuckDBPyRelation: ...
     def except_(self, other_rel: Self) -> DuckDBPyRelation: ...
     def execute(self) -> DuckDBPyRelation: ...
-    def explain(self, type: ExplainType | ExplainTypeLiteral = ExplainType.STANDARD) -> str: ...
+    def explain(
+        self, type: ExplainType | ExplainTypeLiteral = ExplainType.STANDARD, format: str | None = None
+    ) -> str: ...
     def favg(
         self, expression: str, groups: str = "", window_spec: str = "", projected_columns: str = ""
     ) -> DuckDBPyRelation: ...

src/duckdb_py/include/duckdb_python/pybind11/conversions/explain_enum.hpp

@@ -33,34 +33,18 @@ static ExplainType ExplainTypeFromInteger(int64_t value) {
 	}
 }
 
-namespace PYBIND11_NAMESPACE {
-namespace detail {
-
-template <>
-struct type_caster<ExplainType> : public type_caster_base<ExplainType> {
-	using base = type_caster_base<ExplainType>;
-	ExplainType tmp;
-
-public:
-	bool load(handle src, bool convert) {
-		if (base::load(src, convert)) {
-			return true;
-		} else if (py::isinstance<py::str>(src)) {
-			tmp = ExplainTypeFromString(py::str(src));
-			value = &tmp;
-			return true;
-		} else if (py::isinstance<py::int_>(src)) {
-			tmp = ExplainTypeFromInteger(src.cast<int64_t>());
-			value = &tmp;
-			return true;
-		}
-		return false;
+//! Resolve a Python explain-type argument (ExplainType enum, str, or int) to an ExplainType.
+//! NOTE: deliberately NOT a pybind type_caster. A custom caster inheriting type_caster_base shadows the
+//! registered py::enum_ inconsistently across translation units - it ends up accepting str/int XOR the enum
+//! instance, never both, depending on which TU sees the specialization. Explicit dispatch at the call site is
+//! robust regardless of include order.
+static ExplainType ExplainTypeFromPython(const py::object &obj) {
+	if (py::isinstance<py::str>(obj)) {
+		return ExplainTypeFromString(py::str(obj));
 	}
-
-	static handle cast(ExplainType src, return_value_policy policy, handle parent) {
-		return base::cast(src, policy, parent);
+	if (py::isinstance<py::int_>(obj)) {
+		return ExplainTypeFromInteger(obj.cast<int64_t>());
 	}
-};
-
-} // namespace detail
-} // namespace PYBIND11_NAMESPACE
+	// Fall through to the registered py::enum_ caster (handles an actual ExplainType, throws otherwise).
+	return obj.cast<ExplainType>();
+}

src/duckdb_py/include/duckdb_python/pyrelation.hpp

@@ -244,7 +244,7 @@ struct DuckDBPyRelation {
 	           const Optional<py::int_> &max_col_width, const Optional<py::str> &null_value,
 	           const py::object &render_mode);
 
-	string Explain(ExplainType type);
+	string Explain(ExplainType type, const string &format = "");
 
 	static bool IsRelation(const py::object &object);
 

src/duckdb_py/native/python_objects.cpp

@@ -462,6 +462,9 @@ static bool KeyIsHashable(const LogicalType &type) {
 	}
 	case LogicalTypeId::STRUCT:
 		return false;
+	case LogicalTypeId::SQLNULL:
+		// A SQLNULL key is always NULL, and Python's None is hashable.
+		return true;
 	default:
 		throw NotImplementedException("Unsupported type: \"%s\"", type.ToString());
 	}

src/duckdb_py/numpy/array_wrapper.cpp

@@ -180,6 +180,25 @@ struct StringConvert {
 	}
 };
 
+struct NullConvert {
+	template <class DUCKDB_T, class NUMPY_T>
+	static PyObject *ConvertValue(DUCKDB_T val, NumpyAppendData &append_data) {
+		// A SQLNULL column contains only NULLs, so ConvertValue is never reached; every row takes NullValue.
+		(void)val;
+		(void)append_data;
+		Py_RETURN_NONE;
+	}
+	template <class NUMPY_T, bool PANDAS>
+	static NUMPY_T NullValue(bool &set_mask) {
+		if (PANDAS) {
+			set_mask = false;
+			Py_RETURN_NONE;
+		}
+		set_mask = true;
+		return nullptr;
+	}
+};
+
 struct BlobConvert {
 	template <class DUCKDB_T, class NUMPY_T>
 	static PyObject *ConvertValue(string_t val, NumpyAppendData &append_data) {
@@ -703,6 +722,11 @@ void ArrayWrapper::Append(idx_t current_offset, Vector &input, idx_t source_size
 	case LogicalTypeId::UUID:
 		may_have_null = ConvertColumn<hugeint_t, PyObject *, duckdb_py_convert::UUIDConvert>(append_data);
 		break;
+	case LogicalTypeId::SQLNULL:
+		// An all-NULL column (e.g. an untyped NULL literal): emit an object column of None. SQLNULL's physical
+		// type is INT32, but its data is never read since every row is NULL.
+		may_have_null = ConvertColumn<int32_t, PyObject *, duckdb_py_convert::NullConvert>(append_data);
+		break;
 
 	default:
 		throw NotImplementedException("Unsupported type \"%s\"", input.GetType().ToString());

src/duckdb_py/numpy/raw_array_wrapper.cpp

@@ -63,6 +63,7 @@ static idx_t GetNumpyTypeWidth(const LogicalType &type) {
 	case LogicalTypeId::ARRAY:
 	case LogicalTypeId::VARIANT:
 	case LogicalTypeId::GEOMETRY:
+	case LogicalTypeId::SQLNULL:
 		return sizeof(PyObject *);
 	default:
 		throw NotImplementedException("Unsupported type \"%s\" for DuckDB -> NumPy conversion", type.ToString());
@@ -128,6 +129,7 @@ string RawArrayWrapper::DuckDBToNumpyDtype(const LogicalType &type) {
 	case LogicalTypeId::ARRAY:
 	case LogicalTypeId::VARIANT:
 	case LogicalTypeId::GEOMETRY:
+	case LogicalTypeId::SQLNULL:
 		return "object";
 	case LogicalTypeId::ENUM: {
 		auto size = EnumType::GetSize(type);

src/duckdb_py/pyrelation.cpp

@@ -1749,17 +1749,22 @@ static void DisplayHTML(const string &html) {
 	display_attr(html_object);
 }
 
-string DuckDBPyRelation::Explain(ExplainType type) {
+string DuckDBPyRelation::Explain(ExplainType type, const string &format) {
 	AssertRelation();
 	D_ASSERT(py::gil_check());
 	py::gil_scoped_release release;
 
-	auto explain_format = GetExplainFormat(type);
+	// An empty format means "auto": the default format, or HTML when running under Jupyter.
+	const bool auto_format = format.empty();
+	auto explain_format = auto_format ? GetExplainFormat(type) : ProfilerPrintFormat::FromString(format);
 	auto res = rel->Explain(type, explain_format);
 	D_ASSERT(res->type == duckdb::QueryResultType::MATERIALIZED_RESULT);
 	auto &materialized = res->Cast<MaterializedQueryResult>();
 	auto &coll = materialized.Collection();
-	if (explain_format != ProfilerPrintFormat::HTML() || !DuckDBPyConnection::IsJupyter()) {
+	// Only the implicit Jupyter path renders HTML inline; an explicitly requested format always returns a string.
+	const bool jupyter_html =
+	    auto_format && explain_format == ProfilerPrintFormat::HTML() && DuckDBPyConnection::IsJupyter();
+	if (!jupyter_html) {
 		string result_;
 		for (auto &row : coll.Rows()) {
 			// Skip the first column because it just contains 'physical plan'

src/duckdb_py/pyrelation/initialize.cpp

@@ -1,6 +1,7 @@
 #include "duckdb_python/pyrelation.hpp"
 #include "duckdb_python/pyconnection/pyconnection.hpp"
 #include "duckdb_python/pyresult.hpp"
+#include "duckdb_python/pybind11/conversions/explain_enum.hpp"
 #include "duckdb/parser/qualified_name.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb_python/numpy/numpy_type.hpp"
@@ -262,7 +263,14 @@ static void InitializeSetOperators(py::class_<DuckDBPyRelation> &m) {
 static void InitializeMetaQueries(py::class_<DuckDBPyRelation> &m) {
 	m.def("describe", &DuckDBPyRelation::Describe,
 	      "Gives basic statistics (e.g., min, max) and if NULL exists for each column of the relation.")
-	    .def("explain", &DuckDBPyRelation::Explain, py::arg("type") = "standard");
+	    .def(
+	        "explain",
+	        [](DuckDBPyRelation &self, const py::object &type, const py::object &format) {
+		        // An omitted format (None) maps to "" = auto-select (default, or HTML under Jupyter).
+		        string format_str = format.is_none() ? string() : string(py::str(format));
+		        return self.Explain(ExplainTypeFromPython(type), format_str);
+	        },
+	        py::arg("type") = "standard", py::arg("format") = py::none());
 }
 
 void DuckDBPyRelation::Initialize(py::handle &m) {

tests/fast/pandas/test_fetch_nested.py

@@ -33,12 +33,11 @@ def list_test_cases():
             )
             ]
         }),
+        # An untyped NULL list now has child type SQLNULL (previously it defaulted to INTEGER), so it
+        # converts to an object array of None rather than a masked integer array.
         ("SELECT list_value(NULL,NULL,NULL) as a", {
             'a': [
-                np.ma.array(
-                    [0, 0, 0],
-                mask=[1, 1, 1],
-            )
+                np.array([None, None, None], dtype=object)
             ]
         }),
         ("SELECT list_value() as a", {

tests/fast/test_expression.py

@@ -202,8 +202,8 @@ def test_column_expression_explain(self):
         res = rel.explain()
         assert "c0" in res
         assert "c1" in res
-        # 'c2' is not in the explain result because it shows NULL instead
-        assert "NULL" in res
+        # the physical plan now renders projection column names (c0, c1, c2) rather than literal constant values
+        assert "c2" in res
         res = rel.fetchall()
         assert res == [("a", 42, None)]