Merge branch 'v1.5-variegata' into fix-unionbyname-missing-columns

evertlammerts · web-flow · commit b94529dbe4ff · 2026-01-12T14:00:26.000+01:00
diff --git a/_duckdb-stubs/_sqltypes.pyi b/_duckdb-stubs/_sqltypes.pyi
@@ -59,6 +59,7 @@ INTERVAL: DuckDBPyType  # value = INTERVAL
 SMALLINT: DuckDBPyType  # value = SMALLINT
 SQLNULL: DuckDBPyType  # value = "NULL"
 TIME: DuckDBPyType  # value = TIME
+TIME_NS: DuckDBPyType  # value = TIME_NS
 TIMESTAMP: DuckDBPyType  # value = TIMESTAMP
 TIMESTAMP_MS: DuckDBPyType  # value = TIMESTAMP_MS
 TIMESTAMP_NS: DuckDBPyType  # value = TIMESTAMP_NS
diff --git a/duckdb/experimental/spark/sql/type_utils.py b/duckdb/experimental/spark/sql/type_utils.py
@@ -23,6 +23,7 @@
     StringType,
     StructField,
     StructType,
+    TimeNSType,
     TimeNTZType,
     TimestampMillisecondNTZType,
     TimestampNanosecondNTZType,
@@ -56,6 +57,7 @@
     "uuid": UUIDType,
     "date": DateType,
     "time": TimeNTZType,
+    "time_ns": TimeNSType,
     "time with time zone": TimeType,
     "timestamp": TimestampNTZType,
     "timestamp with time zone": TimestampType,
diff --git a/duckdb/experimental/spark/sql/types.py b/duckdb/experimental/spark/sql/types.py
@@ -51,6 +51,7 @@
     "StringType",
     "StructField",
     "StructType",
+    "TimeNSType",
     "TimeNTZType",
     "TimeType",
     "TimestampMillisecondNTZType",
@@ -505,6 +506,16 @@ def simpleString(self) -> str:  # noqa: D102
         return "time"
 
 
+class TimeNSType(IntegralType):
+    """Time NS (datetime.time) data type without timezone information."""
+
+    def __init__(self) -> None:  # noqa: D107
+        super().__init__(DuckDBPyType("TIME_NS"))
+
+    def simpleString(self) -> str:  # noqa: D102
+        return "time_ns"
+
+
 class DayTimeIntervalType(AtomicType):
     """DayTimeIntervalType (datetime.timedelta)."""
 
diff --git a/duckdb/sqltypes/__init__.py b/duckdb/sqltypes/__init__.py
@@ -14,6 +14,7 @@
     SMALLINT,
     SQLNULL,
     TIME,
+    TIME_NS,
     TIME_TZ,
     TIMESTAMP,
     TIMESTAMP_MS,
@@ -50,6 +51,7 @@
     "TIMESTAMP_NS",
     "TIMESTAMP_S",
     "TIMESTAMP_TZ",
+    "TIME_NS",
     "TIME_TZ",
     "TINYINT",
     "UBIGINT",
diff --git a/external/duckdb b/external/duckdb
@@ -1 +1 @@
-Subproject commit 2e305aac809ef22e818024be1e86a1d0ee0d2863
+Subproject commit ba0a7e6292c1dc1685cf6751b92e32c8f8c0c937
diff --git a/src/duckdb_py/native/python_objects.cpp b/src/duckdb_py/native/python_objects.cpp
@@ -585,13 +585,20 @@ py::object PythonObject::FromValue(const Value &val, const LogicalType &type,
 		auto tmp_datetime_with_tz = import_cache.datetime.datetime.combine()(tmp_datetime, py_time, timezone_offset);
 		return tmp_datetime_with_tz.attr("timetz")();
 	}
-	case LogicalTypeId::TIME: {
+	case LogicalTypeId::TIME:
+	case LogicalTypeId::TIME_NS: {
 		D_ASSERT(type.InternalType() == PhysicalType::INT64);
-		int32_t hour, min, sec, microsec;
-		auto time = val.GetValueUnsafe<dtime_t>();
-		duckdb::Time::Convert(time, hour, min, sec, microsec);
+		int32_t hour, min, sec, usec;
+		dtime_t time;
+		if (type.id() == LogicalTypeId::TIME) {
+			time = val.GetValueUnsafe<dtime_t>();
+		} else {
+			// Python's datetime doesn't support nanoseconds, we convert to micros.
+			time = val.GetValueUnsafe<dtime_ns_t>().time();
+		}
+		duckdb::Time::Convert(time, hour, min, sec, usec);
 		try {
-			auto pytime = PyTime_FromTime(hour, min, sec, microsec);
+			auto pytime = PyTime_FromTime(hour, min, sec, usec);
 			if (!pytime) {
 				throw py::error_already_set();
 			}
diff --git a/src/duckdb_py/numpy/array_wrapper.cpp b/src/duckdb_py/numpy/array_wrapper.cpp
@@ -140,6 +140,24 @@ struct TimeConvert {
 	}
 };
 
+struct TimeNSConvert {
+	template <class DUCKDB_T, class NUMPY_T>
+	static PyObject *ConvertValue(dtime_ns_t val, NumpyAppendData &append_data) {
+		auto &client_properties = append_data.client_properties;
+		auto value = Value::TIME_NS(val);
+		auto py_obj = PythonObject::FromValue(value, LogicalType::TIME_NS, client_properties);
+		// Release ownership of the PyObject* without decreasing refcount
+		// this returns a handle, of which we take the ptr to get the PyObject*
+		return py_obj.release().ptr();
+	}
+
+	template <class NUMPY_T, bool PANDAS>
+	static NUMPY_T NullValue(bool &set_mask) {
+		set_mask = true;
+		return nullptr;
+	}
+};
+
 struct StringConvert {
 	template <class DUCKDB_T, class NUMPY_T>
 	static PyObject *ConvertValue(string_t val, NumpyAppendData &append_data) {
@@ -639,6 +657,9 @@ void ArrayWrapper::Append(idx_t current_offset, Vector &input, idx_t source_size
 	case LogicalTypeId::TIME:
 		may_have_null = ConvertColumn<dtime_t, PyObject *, duckdb_py_convert::TimeConvert>(append_data);
 		break;
+	case LogicalTypeId::TIME_NS:
+		may_have_null = ConvertColumn<dtime_ns_t, PyObject *, duckdb_py_convert::TimeNSConvert>(append_data);
+		break;
 	case LogicalTypeId::INTERVAL:
 		may_have_null = ConvertColumn<interval_t, int64_t, duckdb_py_convert::IntervalConvert>(append_data);
 		break;
diff --git a/src/duckdb_py/numpy/raw_array_wrapper.cpp b/src/duckdb_py/numpy/raw_array_wrapper.cpp
@@ -48,6 +48,7 @@ static idx_t GetNumpyTypeWidth(const LogicalType &type) {
 	case LogicalTypeId::TIMESTAMP_TZ:
 		return sizeof(int64_t);
 	case LogicalTypeId::TIME:
+	case LogicalTypeId::TIME_NS:
 	case LogicalTypeId::TIME_TZ:
 	case LogicalTypeId::VARCHAR:
 	case LogicalTypeId::BIT:
@@ -110,6 +111,7 @@ string RawArrayWrapper::DuckDBToNumpyDtype(const LogicalType &type) {
 	case LogicalTypeId::INTERVAL:
 		return "timedelta64[ns]";
 	case LogicalTypeId::TIME:
+	case LogicalTypeId::TIME_NS:
 	case LogicalTypeId::TIME_TZ:
 	case LogicalTypeId::VARCHAR:
 	case LogicalTypeId::BIT:
diff --git a/src/duckdb_py/typing/typing.cpp b/src/duckdb_py/typing/typing.cpp
@@ -27,6 +27,7 @@ static void DefineBaseTypes(py::handle &m) {
 	m.attr("TIMESTAMP_S") = make_shared_ptr<DuckDBPyType>(LogicalType::TIMESTAMP_S);
 
 	m.attr("TIME") = make_shared_ptr<DuckDBPyType>(LogicalType::TIME);
+	m.attr("TIME_NS") = make_shared_ptr<DuckDBPyType>(LogicalType::TIME_NS);
 
 	m.attr("TIME_TZ") = make_shared_ptr<DuckDBPyType>(LogicalType::TIME_TZ);
 	m.attr("TIMESTAMP_TZ") = make_shared_ptr<DuckDBPyType>(LogicalType::TIMESTAMP_TZ);
diff --git a/tests/fast/arrow/test_time.py b/tests/fast/arrow/test_time.py
@@ -24,7 +24,7 @@ def test_time_types(self, duckdb_cursor):
         assert rel["a"] == arrow_table["c"]
         assert rel["b"] == arrow_table["c"]
         assert rel["c"] == arrow_table["c"]
-        assert rel["d"] == arrow_table["c"]
+        assert rel["d"] == arrow_table["d"]
 
     def test_time_null(self, duckdb_cursor):
         if not can_run:
@@ -40,7 +40,7 @@ def test_time_null(self, duckdb_cursor):
         assert rel["a"] == arrow_table["c"]
         assert rel["b"] == arrow_table["c"]
         assert rel["c"] == arrow_table["c"]
-        assert rel["d"] == arrow_table["c"]
+        assert rel["d"] == arrow_table["d"]
 
     def test_max_times(self, duckdb_cursor):
         if not can_run:
@@ -62,7 +62,7 @@ def test_max_times(self, duckdb_cursor):
         assert rel["a"] == result["a"]
 
         # Max NSec
-        data = pa.array([9223372036854774], type=pa.time64("us"))
+        data = pa.array([9223372036854774000], type=pa.time64("ns"))
         result = pa.Table.from_arrays([data], ["a"])
         data = pa.array([9223372036854774000], type=pa.time64("ns"))
         arrow_table = pa.Table.from_arrays([data], ["a"])
diff --git a/tests/fast/spark/test_spark_types.py b/tests/fast/spark/test_spark_types.py
@@ -28,6 +28,7 @@
     StringType,
     StructField,
     StructType,
+    TimeNSType,
     TimeNTZType,
     TimestampMillisecondNTZType,
     TimestampNanosecondNTZType,
@@ -129,5 +130,6 @@ def test_all_types_schema(self, spark):
                     True,
                 ),
                 StructField("map", MapType(StringType(), StringType(), True), True),
+                StructField("time_ns", TimeNSType(), True),
             ]
         )
diff --git a/tests/fast/types/test_time_ns.py b/tests/fast/types/test_time_ns.py
@@ -0,0 +1,75 @@
+import datetime
+
+import pytest
+
+from duckdb import ConversionException, sqltypes
+
+
+def test_time_ns_select(duckdb_cursor):
+    duckdb_cursor.execute("SELECT TIME_NS '1992-09-20 11:30:00.123456'")
+    result = duckdb_cursor.fetchone()[0]
+    assert result
+    assert isinstance(result, datetime.time)
+
+
+@pytest.mark.xfail(
+    raises=ConversionException,
+    reason="Conversion Error: Unimplemented type for cast (TIME -> TIME_NS)",
+)
+def test_time_ns_insert(duckdb_cursor):
+    """This tests that datetime.time values can be inserted as TIME_NS."""
+    duckdb_cursor.execute("SELECT TIME_NS '1992-09-20 11:30:00.123456'")
+    result1 = duckdb_cursor.fetchone()[0]
+    duckdb_cursor.execute("CREATE OR REPLACE TEMP TABLE time_ns_test (time_ns_col TIME_NS)")
+    duckdb_cursor.execute("INSERT INTO time_ns_test VALUES (?)", [result1])
+    duckdb_cursor.execute("SELECT time_ns_col FROM time_ns_test")
+    result2 = duckdb_cursor.fetchone()[0]
+    assert isinstance(result2, datetime.time)
+    assert result1 == result2
+
+
+def test_time_insert(duckdb_cursor):
+    """This tests that datetime.time values are casted to TIME when needed."""
+    duckdb_cursor.execute("SELECT TIME_NS '1992-09-20 11:30:00.123456'")
+    result1 = duckdb_cursor.fetchone()[0]
+    duckdb_cursor.execute("CREATE OR REPLACE TEMP TABLE time_test (time_col TIME)")
+    duckdb_cursor.execute("INSERT INTO time_test VALUES (?)", [result1])
+    duckdb_cursor.execute("SELECT time_col FROM time_test")
+    result2 = duckdb_cursor.fetchone()[0]
+    assert isinstance(result2, datetime.time)
+    assert result1 == result2
+
+
+def test_time_ns_arrow_roundtrip(duckdb_cursor):
+    pa = pytest.importorskip("pyarrow")
+
+    # Get a time_ns in an arrow table
+    arrow_table = duckdb_cursor.execute("SELECT TIME_NS '12:34:56.123456789' AS time_ns_col").fetch_arrow_table()
+
+    value = arrow_table.column("time_ns_col")[0]
+    assert isinstance(value, pa.lib.Time64Scalar)
+
+    # Roundtrip back into duckdb and assert the column's type is TIME_NS
+    duckdb_cursor.execute("CREATE OR REPLACE TEMP TABLE time_ns_test AS SELECT * FROM arrow_table")
+    col_type = duckdb_cursor.execute("SELECT time_ns_col FROM time_ns_test").description[0][1]
+    assert col_type == sqltypes.TIME_NS
+
+
+def test_time_ns_pandas_roundtrip(duckdb_cursor):
+    """Test that we can roundtrip using Pandas."""
+    pytest.importorskip("pandas")
+    df = duckdb_cursor.execute("SELECT TIME_NS '12:34:56.123456789' AS time_ns_col").df()
+    assert df["time_ns_col"].dtype == "object"
+    duckdb_cursor.execute("CREATE OR REPLACE TEMP TABLE time_ns_test AS SELECT * FROM df")
+    col_type = duckdb_cursor.execute("SELECT time_ns_col FROM time_ns_test").description[0][1]
+    assert col_type == sqltypes.TIME
+
+
+def test_time_pandas_roundtrip(duckdb_cursor):
+    """For Pandas, creating a table using CREATE .... AS SELECT FROM df, will create TIME_NS cols by default."""
+    pytest.importorskip("pandas")
+    df = duckdb_cursor.execute("SELECT TIME '12:34:56.123456789' AS time_col").df()
+    assert df["time_col"].dtype == "object"
+    duckdb_cursor.execute("CREATE OR REPLACE TEMP TABLE time_test AS SELECT * FROM df")
+    col_type = duckdb_cursor.execute("SELECT time_col FROM time_test").description[0][1]
+    assert col_type == sqltypes.TIME
