Merge/v1.5 variegata into main (#507)

Author: evertlammerts

.github/workflows/packaging_wheels.yml

@@ -32,7 +32,7 @@ jobs:
       matrix:
         python: [ cp314 ]
         platform:
-          - { os: windows-2025,     arch: amd64,      cibw_system: win }
+          - { os: windows-2022,     arch: amd64,      cibw_system: win }
           - { os: windows-11-arm,   arch: ARM64,      cibw_system: win }
           - { os: ubuntu-24.04,     arch: x86_64,     cibw_system: manylinux }
           - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }

.github/workflows/targeted_test.yml

@@ -8,7 +8,7 @@ on:
         required: true
         type: choice
         options:
-          - 'windows-2025'
+          - 'windows-2022'
           - 'windows-11-arm'
           - 'ubuntu-24.04'
           - 'ubuntu-24.04-arm'

external/duckdb

@@ -14,11 +14,11 @@
 
 namespace duckdb {
 
-void TransformDuckToArrowChunk(ArrowSchema &arrow_schema, ArrowArray &data, py::list &batches) {
+void TransformDuckToArrowChunk(py::object pyarrow_schema, ArrowArray &data, py::list &batches) {
 	py::gil_assert();
 	auto pyarrow_lib_module = py::module::import("pyarrow").attr("lib");
 	auto batch_import_func = pyarrow_lib_module.attr("RecordBatch").attr("_import_from_c");
-	batches.append(batch_import_func(reinterpret_cast<uint64_t>(&data), reinterpret_cast<uint64_t>(&arrow_schema)));
+	batches.append(batch_import_func(reinterpret_cast<uint64_t>(&data), pyarrow_schema));
 }
 
 void VerifyArrowDatasetLoaded() {

src/duckdb_py/arrow/arrow_array_stream.cpp

@@ -17,18 +17,28 @@ namespace duckdb {
 
 namespace pyarrow {
 
-py::object ToArrowTable(const vector<LogicalType> &types, const vector<string> &names, const py::list &batches,
-                        ClientProperties &options) {
+py::object ToPyArrowSchema(const ArrowSchema &schema) {
 	py::gil_scoped_acquire acquire;
 
 	auto pyarrow_lib_module = py::module::import("pyarrow").attr("lib");
-	auto from_batches_func = pyarrow_lib_module.attr("Table").attr("from_batches");
 	auto schema_import_func = pyarrow_lib_module.attr("Schema").attr("_import_from_c");
+	return schema_import_func(reinterpret_cast<uint64_t>(&schema));
+}
+
+py::object ToArrowTable(const py::list &batches, py::object pyarrow_schema) {
+	py::gil_scoped_acquire acquire;
+
+	auto pyarrow_lib_module = py::module::import("pyarrow").attr("lib");
+	auto from_batches_func = pyarrow_lib_module.attr("Table").attr("from_batches");
+
+	return py::cast<duckdb::pyarrow::Table>(from_batches_func(batches, pyarrow_schema));
+}
+
+py::object ToArrowTable(const vector<LogicalType> &types, const vector<string> &names, const py::list &batches,
+                        ClientProperties &options) {
 	ArrowSchema schema;
 	ArrowConverter::ToArrowSchema(&schema, types, names, options);
-	auto schema_obj = schema_import_func(reinterpret_cast<uint64_t>(&schema));
-
-	return py::cast<duckdb::pyarrow::Table>(from_batches_func(batches, schema_obj));
+	return ToArrowTable(batches, ToPyArrowSchema(schema));
 }
 
 } // namespace pyarrow

src/duckdb_py/arrow/arrow_export_utils.cpp

@@ -62,7 +62,7 @@ enum class PyArrowObjectType {
 	PolarsLazyFrame
 };
 
-void TransformDuckToArrowChunk(ArrowSchema &arrow_schema, ArrowArray &data, py::list &batches);
+void TransformDuckToArrowChunk(py::object pyarrow_schema, ArrowArray &data, py::list &batches);
 
 PyArrowObjectType GetArrowType(const py::handle &obj);
 

src/duckdb_py/include/duckdb_python/arrow/arrow_array_stream.hpp

@@ -6,9 +6,13 @@ namespace duckdb {
 
 namespace pyarrow {
 
+py::object ToPyArrowSchema(const ArrowSchema &schema);
+
 py::object ToArrowTable(const vector<LogicalType> &types, const vector<string> &names, const py::list &batches,
                         ClientProperties &options);
 
+py::object ToArrowTable(const py::list &batches, py::object pyarrow_schema);
+
 } // namespace pyarrow
 
 } // namespace duckdb

src/duckdb_py/include/duckdb_python/arrow/arrow_export_utils.hpp

@@ -36,15 +36,13 @@ struct DuckDBPyResult {
 
 	PandasDataFrame FetchDF(bool date_as_object);
 
-	duckdb::pyarrow::Table FetchArrowTable(idx_t rows_per_batch, bool to_polars);
-
 	PandasDataFrame FetchDFChunk(const idx_t vectors_per_chunk = 1, bool date_as_object = false);
 
 	py::dict FetchPyTorch();
 
 	py::dict FetchTF();
 
-	ArrowArrayStream FetchArrowArrayStream(idx_t rows_per_batch = 1000000);
+	duckdb::pyarrow::Table FetchArrowTable(idx_t rows_per_batch, bool to_polars);
 	duckdb::pyarrow::RecordBatchReader FetchRecordBatchReader(idx_t rows_per_batch = 1000000);
 	py::object FetchArrowCapsule(idx_t rows_per_batch = 1000000);
 
@@ -71,6 +69,19 @@ struct DuckDBPyResult {
 	unique_ptr<DataChunk> FetchNextRaw(QueryResult &result);
 	unique_ptr<NumpyResultConversion> InitializeNumpyConversion(bool pandas = false);
 
+	//! Re-feed an already-MATERIALIZED result (a ColumnDataCollection, e.g. from
+	//! rel.execute()) back through the engine on the user's own context. The eager
+	//! variant installs a PhysicalArrowCollector to produce an ArrowQueryResult
+	//! (parallel); the stream variant produces a lazy StreamQueryResult that co-owns
+	//! the context (so it survives `del conn`). Never call these on a StreamQueryResult:
+	//! a lazy result already has a live context and is converted/wrapped directly.
+	void PromoteMaterializedToArrow(idx_t batch_size);
+
+	template <typename T>
+	T RunWithArrowSchema(const std::function<T(const ArrowSchema &)> &fun, bool dedup_col_names);
+	duckdb::pyarrow::Table MaterializedResultToArrowTable(const ArrowSchema &arrow_schema, idx_t rows_per_batch);
+	ArrowArrayStream FetchArrowArrayStream(idx_t rows_per_batch);
+
 private:
 	idx_t chunk_offset = 0;
 

src/duckdb_py/include/duckdb_python/pyresult.hpp

@@ -413,7 +413,7 @@ shared_ptr<DuckDBPyConnection> DuckDBPyConnection::UnregisterUDF(const string &n
 		auto &catalog = Catalog::GetCatalog(context, SYSTEM_CATALOG);
 		DropInfo info;
 		info.type = CatalogType::SCALAR_FUNCTION_ENTRY;
-		info.name = Identifier(name);
+		info.NameMutable() = Identifier(name);
 		info.allow_drop_internal = true;
 		info.cascade = false;
 		info.if_not_found = OnEntryNotFound::THROW_EXCEPTION;
@@ -1651,11 +1651,12 @@ unique_ptr<DuckDBPyRelation> DuckDBPyConnection::RunQuery(const py::object &quer
 unique_ptr<DuckDBPyRelation> DuckDBPyConnection::Table(const string &tname) {
 	auto &connection = con.GetConnection();
 	auto qualified_name = QualifiedName::Parse(tname);
-	if (qualified_name.schema.empty()) {
-		qualified_name.schema = DEFAULT_SCHEMA;
+	if (qualified_name.Schema().empty()) {
+		qualified_name.SchemaMutable() = DEFAULT_SCHEMA;
 	}
 	try {
-		return CreateRelation(connection.Table(qualified_name.catalog, qualified_name.schema, qualified_name.name));
+		return CreateRelation(
+		    connection.Table(qualified_name.Catalog(), qualified_name.Schema(), qualified_name.Name()));
 	} catch (const CatalogException &) {
 		// CatalogException will be of the type '... is not a table'
 		// Not a table in the database, make a query relation that can perform replacement scans

src/duckdb_py/pyconnection.cpp

@@ -325,13 +325,13 @@ shared_ptr<DuckDBPyExpression> DuckDBPyExpression::ColumnExpression(const py::ar
 		}
 
 		auto qualified_name = QualifiedName::Parse(column_name);
-		if (!qualified_name.catalog.empty()) {
-			column_names.push_back(qualified_name.catalog);
+		if (!qualified_name.Catalog().empty()) {
+			column_names.push_back(qualified_name.Catalog());
 		}
-		if (!qualified_name.schema.empty()) {
-			column_names.push_back(qualified_name.schema);
+		if (!qualified_name.Schema().empty()) {
+			column_names.push_back(qualified_name.Schema());
 		}
-		column_names.push_back(qualified_name.name);
+		column_names.push_back(qualified_name.Name());
 	} else {
 		for (auto &part : names) {
 			column_names.push_back(Identifier(py::str(part)));