fix regressions and get on par with main

Author: evertlammerts

_duckdb-stubs/_typing.pyi

@@ -133,7 +133,7 @@ Note:
     We use lowercase here to be able to reuse this `Literal` in the `DTypeIdentifiers` `Literal`.
 """
 
-NestedIds: TypeAlias = Literal["list", "struct", "array", "enum", "map", "decimal", "union"]
+NestedIds: TypeAlias = Literal["list", "struct", "tuple", "array", "enum", "map", "decimal", "union"]
 """Identifiers for nested types in `DuckDBPyType.id`."""
 
 PyTypeIds: TypeAlias = Builtins | NestedIds

duckdb/experimental/spark/sql/type_utils.py

@@ -94,7 +94,7 @@ def convert_nested_type(dtype: DuckDBPyType) -> DataType:  # noqa: D103
             "DuckDB union types cannot be directly mapped to PySpark types."
         )
         raise ContributionsAcceptedError(msg)
-    if id == "struct":
+    if id == "struct" or id == "tuple":
         children: list[tuple[str, DuckDBPyType]] = dtype.children
         fields = [StructField(x[0], convert_type(x[1])) for x in children]
         return StructType(fields)
@@ -105,7 +105,7 @@ def convert_nested_type(dtype: DuckDBPyType) -> DataType:  # noqa: D103
 
 def convert_type(dtype: DuckDBPyType) -> DataType:  # noqa: D103
     id = dtype.id
-    if id in ["list", "struct", "map", "array"]:
+    if id in ["list", "struct", "tuple", "map", "array"]:
         return convert_nested_type(dtype)
     if id == "decimal":
         children: list[tuple[str, DuckDBPyType]] = dtype.children

external/duckdb

@@ -79,10 +79,9 @@ nb::object MakePyArrowScalar(const Value &constant, const string &timezone_confi
 		// Cast::Operation<dtime_ns_t, int64_t> for which no specialization exists, and
 		// throws "Unimplemented type for cast (INT64 -> INT64)". Use the type-strong
 		// GetValueUnsafe<dtime_ns_t>() which reads `value_.time_ns` from the union
-		// directly. The `dtime_ns_t.micros` field name is a misnomer — it actually holds
-		// nanoseconds (see arrow_conversion.cpp:432).
+		// directly. dtime_ns_t.value holds nanoseconds (see arrow_conversion.cpp:432).
 		nb::handle date_type = import_cache.pyarrow.time64();
-		return dataset_scalar(scalar(constant.GetValueUnsafe<dtime_ns_t>().micros, date_type("ns")));
+		return dataset_scalar(scalar(constant.GetValueUnsafe<dtime_ns_t>().value, date_type("ns")));
 	}
 	case LogicalTypeId::TIMESTAMP: {
 		nb::handle date_type = import_cache.pyarrow.timestamp();

src/arrow/pyarrow_filter_pushdown.cpp

@@ -120,7 +120,7 @@ Value TransformDictionaryToStruct(optional_ptr<ClientContext> context, const PyD
                                   const LogicalType &target_type = LogicalType::UNKNOWN) {
 	auto struct_keys = TransformStructKeys(dict.keys, dict.len, target_type);
 
-	bool struct_target = target_type.id() == LogicalTypeId::STRUCT;
+	bool struct_target = target_type.id() == LogicalTypeId::STRUCT || target_type.id() == LogicalTypeId::TUPLE;
 	if (struct_target && dict.len != StructType::GetChildCount(target_type)) {
 		throw InvalidInputException("We could not convert the object %s to the desired target type (%s)",
 		                            dict.ToString(), target_type.ToString());
@@ -255,7 +255,7 @@ Value TransformTupleToStruct(optional_ptr<ClientContext> context, nb::handle ele
 	auto tuple = nb::cast<nb::tuple>(ele);
 	auto size = nb::len(tuple);
 
-	D_ASSERT(target_type.id() == LogicalTypeId::STRUCT);
+	D_ASSERT(target_type.id() == LogicalTypeId::STRUCT || target_type.id() == LogicalTypeId::TUPLE);
 	auto child_types = StructType::GetChildTypes(target_type);
 	auto child_count = child_types.size();
 	if (size != child_count) {
@@ -562,7 +562,7 @@ struct PythonValueConversion {
 
 	static void HandleTuple(optional_ptr<ClientContext> context, Value &result, const LogicalType &target_type,
 	                        nb::handle ele, idx_t list_size) {
-		if (target_type.id() == LogicalTypeId::STRUCT) {
+		if (target_type.id() == LogicalTypeId::STRUCT || target_type.id() == LogicalTypeId::TUPLE) {
 			result = TransformTupleToStruct(context, ele, target_type);
 			return;
 		}
@@ -588,6 +588,7 @@ struct PythonValueConversion {
 			PyDictionary dict = PyDictionary(nb::borrow<nb::object>(ele));
 			switch (target_type.id()) {
 			case LogicalTypeId::STRUCT:
+			case LogicalTypeId::TUPLE:
 				return TransformDictionaryToStruct(context, dict, target_type);
 			case LogicalTypeId::MAP:
 				return TransformDictionaryToMap(context, dict, target_type);
@@ -890,6 +891,7 @@ struct PythonVectorConversion {
 		auto &result_type = result.GetType();
 		switch (result_type.id()) {
 		case LogicalTypeId::STRUCT:
+		case LogicalTypeId::TUPLE:
 			ConvertTupleToStruct(context, result, result_offset, ele, tuple_size);
 			break;
 		case LogicalTypeId::ARRAY:
@@ -985,6 +987,7 @@ void TransformPythonObjectInternal(optional_ptr<ClientContext> context, nb::hand
 		auto &conversion_target = OP::ConversionTarget(result, param);
 		switch (conversion_target.id()) {
 		case LogicalTypeId::STRUCT:
+		case LogicalTypeId::TUPLE:
 		case LogicalTypeId::UNKNOWN:
 		case LogicalTypeId::LIST:
 		case LogicalTypeId::ARRAY:

src/native/python_conversion.cpp

@@ -462,6 +462,7 @@ static bool KeyIsHashable(const LogicalType &type) {
 		return true;
 	}
 	case LogicalTypeId::STRUCT:
+	case LogicalTypeId::TUPLE:
 		return false;
 	case LogicalTypeId::SQLNULL:
 		// A SQLNULL key is always NULL, and Python's None is hashable.
@@ -608,7 +609,7 @@ nb::object PythonObject::FromValue(const Value &val, const LogicalType &type,
 			time = val.GetValueUnsafe<dtime_t>();
 		} else {
 			// Python's datetime doesn't support nanoseconds, we convert to micros.
-			time = val.GetValueUnsafe<dtime_ns_t>().time();
+			time = dtime_t(val.GetValueUnsafe<dtime_ns_t>().value / 1000);
 		}
 		duckdb::Time::Convert(time, hour, min, sec, usec);
 		try {
@@ -702,7 +703,8 @@ nb::object PythonObject::FromValue(const Value &val, const LogicalType &type,
 		}
 		return std::move(py_struct);
 	}
-	case LogicalTypeId::STRUCT: {
+	case LogicalTypeId::STRUCT:
+	case LogicalTypeId::TUPLE: {
 		return FromStruct(val, type, client_properties);
 	}
 	case LogicalTypeId::UUID: {

src/native/python_objects.cpp

@@ -295,21 +295,10 @@ struct ArrayConvert {
 };
 
 struct StructConvert {
-	static nb::dict ConvertValue(Vector &input, idx_t chunk_offset, NumpyAppendData &append_data) {
-		auto &client_properties = append_data.client_properties;
-
-		nb::dict py_struct;
+	// Delegate to FromStruct so unnamed structs / TUPLE values become Python tuples (named ones stay dicts).
+	static nb::object ConvertValue(Vector &input, idx_t chunk_offset, NumpyAppendData &append_data) {
 		auto val = input.GetValue(chunk_offset);
-		auto &child_types = StructType::GetChildTypes(input.GetType());
-		auto &struct_children = StructValue::GetChildren(val);
-
-		for (idx_t i = 0; i < struct_children.size(); i++) {
-			auto &child_entry = child_types[i];
-			auto &child_name = child_entry.first;
-			auto &child_type = child_entry.second;
-			py_struct[child_name.c_str()] = PythonObject::FromValue(struct_children[i], child_type, client_properties);
-		}
-		return py_struct;
+		return PythonObject::FromStruct(val, input.GetType(), append_data.client_properties);
 	}
 };
 
@@ -716,6 +705,7 @@ void ArrayWrapper::Append(idx_t current_offset, Vector &input, idx_t source_size
 		may_have_null = ConvertNested<nb::object, duckdb_py_convert::UnionConvert>(append_data);
 		break;
 	case LogicalTypeId::STRUCT:
+	case LogicalTypeId::TUPLE:
 		may_have_null = ConvertNested<nb::object, duckdb_py_convert::StructConvert>(append_data);
 		break;
 	case LogicalTypeId::VARIANT:

src/numpy/array_wrapper.cpp

@@ -58,6 +58,7 @@ static idx_t GetNumpyTypeWidth(const LogicalType &type) {
 	case LogicalTypeId::LIST:
 	case LogicalTypeId::MAP:
 	case LogicalTypeId::STRUCT:
+	case LogicalTypeId::TUPLE:
 	case LogicalTypeId::UNION:
 	case LogicalTypeId::UUID:
 	case LogicalTypeId::ARRAY:
@@ -124,6 +125,7 @@ string RawArrayWrapper::DuckDBToNumpyDtype(const LogicalType &type) {
 	case LogicalTypeId::LIST:
 	case LogicalTypeId::MAP:
 	case LogicalTypeId::STRUCT:
+	case LogicalTypeId::TUPLE:
 	case LogicalTypeId::UNION:
 	case LogicalTypeId::UUID:
 	case LogicalTypeId::ARRAY:

src/numpy/raw_array_wrapper.cpp

@@ -436,7 +436,7 @@ std::shared_ptr<DuckDBPyConnection> DuckDBPyConnection::UnregisterUDF(const stri
 		auto &catalog = Catalog::GetCatalog(context, SYSTEM_CATALOG);
 		DropInfo info;
 		info.type = CatalogType::SCALAR_FUNCTION_ENTRY;
-		info.NameMutable() = Identifier(name);
+		info.SetName(Identifier(name));
 		info.allow_drop_internal = true;
 		info.cascade = false;
 		info.if_not_found = OnEntryNotFound::THROW_EXCEPTION;
@@ -1694,7 +1694,7 @@ std::unique_ptr<DuckDBPyRelation> DuckDBPyConnection::Table(const string &tname)
 	auto &connection = con.GetConnection();
 	auto qualified_name = QualifiedName::Parse(tname);
 	if (qualified_name.Schema().empty()) {
-		qualified_name.SchemaMutable() = DEFAULT_SCHEMA;
+		qualified_name = QualifiedName(qualified_name.Catalog(), DEFAULT_SCHEMA, qualified_name.Name());
 	}
 	try {
 		return CreateRelation(

src/pyconnection.cpp

@@ -55,7 +55,7 @@ bool DuckDBPyType::EqualsString(const string &type_str) const {
 
 std::unique_ptr<DuckDBPyType> DuckDBPyType::GetAttribute(const string &name) const {
 	auto name_identifier = Identifier(name);
-	if (type.id() == LogicalTypeId::STRUCT || type.id() == LogicalTypeId::UNION) {
+	if (type.id() == LogicalTypeId::STRUCT || type.id() == LogicalTypeId::TUPLE || type.id() == LogicalTypeId::UNION) {
 		auto &children = StructType::GetChildTypes(type);
 		for (idx_t i = 0; i < children.size(); i++) {
 			auto &child = children[i];
@@ -402,6 +402,7 @@ nb::list DuckDBPyType::Children() const {
 	switch (type.id()) {
 	case LogicalTypeId::LIST:
 	case LogicalTypeId::STRUCT:
+	case LogicalTypeId::TUPLE:
 	case LogicalTypeId::UNION:
 	case LogicalTypeId::MAP:
 	case LogicalTypeId::ARRAY:
@@ -436,7 +437,7 @@ nb::list DuckDBPyType::Children() const {
 		children.append(nb::make_tuple("values", strings_list));
 		return children;
 	}
-	if (id == LogicalTypeId::STRUCT || id == LogicalTypeId::UNION) {
+	if (id == LogicalTypeId::STRUCT || id == LogicalTypeId::TUPLE || id == LogicalTypeId::UNION) {
 		auto &struct_children = StructType::GetChildTypes(type);
 		for (idx_t i = 0; i < struct_children.size(); i++) {
 			auto &child = struct_children[i];