tuple field assignment wrapper

Author: evertlammerts

src/duckdb_py/include/duckdb_python/pybind11/pybind_wrapper.hpp

@@ -18,6 +18,7 @@
 #include <nanobind/stl/optional.h>
 #include <nanobind/stl/detail/nb_list.h>
 #include <nanobind/operators.h>
+#include <cassert>
 
 // nanobind has no PYBIND11_NAMESPACE; the custom type_caster specializations below (and in the
 // conversion headers) live in `namespace nanobind`. Point the legacy macro at it so those headers
@@ -108,13 +109,8 @@ bool try_cast(const handle &object, T &result) {
 	return true;
 }
 
-// pybind11's std::string caster accepted str (as-is) and bytes (decoded UTF-8) and stringified scalars; nanobind's
-// nb::cast<std::string> is stricter and surfaces a raw std::bad_cast for non-str input. This helper restores the
-// lenient behavior for the identifier / parameter-key / separator sites that relied on it:
-//   str   -> the string as-is
-//   bytes -> UTF-8 decoded (so read_csv(sep=b"|") and byte-string identifiers keep working)
-//   else  -> str(obj) (so e.g. an int parameter-dict key stringifies to "1", matching pybind11)
-// It never throws std::bad_cast.
+// Lenient string conversion matching pybind11 (nanobind's cast<std::string> rejects bytes/scalars with std::bad_cast):
+// str stays as is, bytes are UTF-8 decoded, anything else goes through str(). For identifier/param-key/separator sites.
 inline std::string cast_to_string(handle obj) {
 	// Use check_ directly: an unqualified isinstance<> here is ambiguous between this namespace's override and
 	// nanobind's (pulled in by the using-directive above).
@@ -127,6 +123,36 @@ inline std::string cast_to_string(handle obj) {
 	return cast<std::string>(str(obj));
 }
 
+// Fills a tuple of known size via PyTuple_SET_ITEM (nanobind's py::tuple is immutable). Cheaper than building a
+// py::list then copying it to a tuple. Fill every slot with append()/set_item(), then take().
+class tuple_builder {
+public:
+	explicit tuple_builder(size_t size)
+	    : tuple_(steal<tuple>(PyTuple_New(static_cast<Py_ssize_t>(size)))), size_(size) {
+	}
+	// Append to the next slot (steals item's ref).
+	void append(object item) {
+		assert(index_ < size_);
+		PyTuple_SET_ITEM(tuple_.ptr(), static_cast<Py_ssize_t>(index_++), item.release().ptr());
+	}
+	// Set slot `index` (steals item's ref).
+	void set_item(size_t index, object item) {
+		assert(index < size_);
+		PyTuple_SET_ITEM(tuple_.ptr(), static_cast<Py_ssize_t>(index), item.release().ptr());
+	}
+	size_t size() const {
+		return size_;
+	}
+	tuple take() {
+		return std::move(tuple_);
+	}
+
+private:
+	tuple tuple_;
+	size_t size_;
+	size_t index_ = 0;
+};
+
 // pybind11 compatibility shim: pybind11's py::register_exception<T>(scope, name[, base]) maps to nanobind's
 // nb::exception<T>(scope, name[, base]) (which both creates the Python exception type and registers a C++->Python
 // translator). Returns the exception object so callers can set .attr()/.doc().

src/duckdb_py/native/python_objects.cpp

@@ -393,17 +393,14 @@ py::object PythonObject::FromStruct(const Value &val, const LogicalType &type,
 
 	auto &child_types = StructType::GetChildTypes(type);
 	if (StructType::IsUnnamed(type)) {
-		// nanobind tuples are immutable; build the pre-sized tuple via the raw CPython API (SET_ITEM steals
-		// the reference) to keep the hot FromValue path allocation-light.
-		auto py_tuple = py::steal<py::tuple>(PyTuple_New((Py_ssize_t)struct_values.size()));
+		py::tuple_builder py_tuple(struct_values.size());
 		for (idx_t i = 0; i < struct_values.size(); i++) {
 			auto &child_entry = child_types[i];
 			D_ASSERT(child_entry.first.empty());
 			auto &child_type = child_entry.second;
-			PyTuple_SET_ITEM(py_tuple.ptr(), (Py_ssize_t)i,
-			                 FromValue(struct_values[i], child_type, client_properties).release().ptr());
+			py_tuple.append(FromValue(struct_values[i], child_type, client_properties));
 		}
-		return std::move(py_tuple);
+		return py_tuple.take();
 	} else {
 		py::dict py_struct;
 		for (idx_t i = 0; i < struct_values.size(); i++) {
@@ -672,13 +669,11 @@ py::object PythonObject::FromValue(const Value &val, const LogicalType &type,
 		// because the return type of ArrayType::GetSize is idx_t,
 		// which is typedef'd to uint64_t and ssize_t is 4 bytes with Emscripten
 		// and pybind11 requires that the input be castable to ssize_t
-		auto arr = py::steal<py::tuple>(PyTuple_New(static_cast<Py_ssize_t>(array_size)));
-
+		py::tuple_builder arr(array_size);
 		for (idx_t elem_idx = 0; elem_idx < array_size; elem_idx++) {
-			PyTuple_SET_ITEM(arr.ptr(), (Py_ssize_t)elem_idx,
-			                 FromValue(array_values[elem_idx], child_type, client_properties).release().ptr());
+			arr.append(FromValue(array_values[elem_idx], child_type, client_properties));
 		}
-		return std::move(arr);
+		return arr.take();
 	}
 	case LogicalTypeId::MAP: {
 		auto &list_values = ListValue::GetChildren(val);

src/duckdb_py/pyresult.cpp

@@ -136,21 +136,18 @@ Optional<py::tuple> DuckDBPyResult::Fetchone() {
 	if (!current_chunk || current_chunk->size() == 0) {
 		return py::none();
 	}
-	// nanobind tuples are immutable (no pre-sized ctor / indexed assignment); build a list sequentially
-	// and convert to a tuple at the end. Only py-object refcounts move here, no heavy C++ data is copied.
-	py::list res;
-
+	py::tuple_builder row(result->types.size());
 	for (idx_t col_idx = 0; col_idx < result->types.size(); col_idx++) {
 		auto &mask = FlatVector::Validity(current_chunk->data[col_idx]);
 		if (!mask.RowIsValid(chunk_offset)) {
-			res.append(py::none());
-			continue;
+			row.append(py::none());
+		} else {
+			auto val = current_chunk->data[col_idx].GetValue(chunk_offset);
+			row.append(PythonObject::FromValue(val, result->types[col_idx], result->client_properties));
 		}
-		auto val = current_chunk->data[col_idx].GetValue(chunk_offset);
-		res.append(PythonObject::FromValue(val, result->types[col_idx], result->client_properties));
 	}
 	chunk_offset++;
-	return py::tuple(res);
+	return row.take();
 }
 
 py::list DuckDBPyResult::Fetchmany(idx_t size) {

src/duckdb_py/python_udf.cpp

@@ -331,9 +331,7 @@ static scalar_function_t CreateNativeFunction(PyObject *function, PythonExceptio
 
 			py::object ret;
 			if (input.ColumnCount() > 0) {
-				// nanobind tuples are immutable; build a pre-sized tuple with the raw CPython API (SET_ITEM steals a
-				// reference) so the per-row UDF path keeps pybind11's allocation profile (no list-then-convert copy).
-				auto bundled_parameters = py::steal<py::tuple>(PyTuple_New((Py_ssize_t)input.ColumnCount()));
+				py::tuple_builder parameter_builder(input.ColumnCount());
 				bool contains_null = false;
 				for (idx_t i = 0; i < input.ColumnCount(); i++) {
 					// Fill the tuple with the arguments for this row
@@ -343,16 +341,15 @@ static scalar_function_t CreateNativeFunction(PyObject *function, PythonExceptio
 						contains_null = true;
 						break;
 					}
-					PyTuple_SET_ITEM(
-					    bundled_parameters.ptr(), (Py_ssize_t)i,
-					    PythonObject::FromValue(value, column.GetType(), client_properties).release().ptr());
+					parameter_builder.append(PythonObject::FromValue(value, column.GetType(), client_properties));
 				}
 				if (contains_null) {
 					// Immediately insert None, no need to call the function
 					FlatVector::SetNull(result, row, true);
 					continue;
 				}
 				// Call the function
+				auto bundled_parameters = parameter_builder.take();
 				ret = py::steal<py::object>(PyObject_CallObject(function, bundled_parameters.ptr()));
 			} else {
 				ret = py::steal<py::object>(PyObject_CallObject(function, nullptr));