Skip to content

Commit 510ebd8

Browse files
evertlammertsevertlammerts
authored
Integrate TIME_NS type (duckdb#262)
Add support for TIME_NS (see duckdb/duckdb#20361). The original PR added support for the `TIME_NS` type, which represents the time of day in nanoseconds: * Arrow supports this type * Pandas does not have this type itself (but can handle it as an `object` through arrow) * Python does not have an equivalent - `datetime.time` has microsecond precision. Note that DuckDB supports `TIME_NS` -> `TIME` casting but not the other way around, meaning that you can't insert e.g. a `datetime.time` into a `TIME_NS` column.
2 parents 8221a2a + 5a67395 commit 510ebd8

11 files changed

Lines changed: 132 additions & 8 deletions

File tree

‎_duckdb-stubs/_sqltypes.pyi‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ INTERVAL: DuckDBPyType # value = INTERVAL
5959
SMALLINT: DuckDBPyType # value = SMALLINT
6060
SQLNULL: DuckDBPyType # value = "NULL"
6161
TIME: DuckDBPyType # value = TIME
62+
TIME_NS: DuckDBPyType # value = TIME_NS
6263
TIMESTAMP: DuckDBPyType # value = TIMESTAMP
6364
TIMESTAMP_MS: DuckDBPyType # value = TIMESTAMP_MS
6465
TIMESTAMP_NS: DuckDBPyType # value = TIMESTAMP_NS

‎duckdb/experimental/spark/sql/type_utils.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
StringType,
2424
StructField,
2525
StructType,
26+
TimeNSType,
2627
TimeNTZType,
2728
TimestampMillisecondNTZType,
2829
TimestampNanosecondNTZType,
@@ -56,6 +57,7 @@
5657
"uuid": UUIDType,
5758
"date": DateType,
5859
"time": TimeNTZType,
60+
"time_ns": TimeNSType,
5961
"time with time zone": TimeType,
6062
"timestamp": TimestampNTZType,
6163
"timestamp with time zone": TimestampType,

‎duckdb/experimental/spark/sql/types.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
"StringType",
5252
"StructField",
5353
"StructType",
54+
"TimeNSType",
5455
"TimeNTZType",
5556
"TimeType",
5657
"TimestampMillisecondNTZType",
@@ -505,6 +506,16 @@ def simpleString(self) -> str: # noqa: D102
505506
return "time"
506507

507508

509+
class TimeNSType(IntegralType):
510+
"""Time NS (datetime.time) data type without timezone information."""
511+
512+
def __init__(self) -> None: # noqa: D107
513+
super().__init__(DuckDBPyType("TIME_NS"))
514+
515+
def simpleString(self) -> str: # noqa: D102
516+
return "time_ns"
517+
518+
508519
class DayTimeIntervalType(AtomicType):
509520
"""DayTimeIntervalType (datetime.timedelta)."""
510521

‎duckdb/sqltypes/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
SMALLINT,
1515
SQLNULL,
1616
TIME,
17+
TIME_NS,
1718
TIME_TZ,
1819
TIMESTAMP,
1920
TIMESTAMP_MS,
@@ -50,6 +51,7 @@
5051
"TIMESTAMP_NS",
5152
"TIMESTAMP_S",
5253
"TIMESTAMP_TZ",
54+
"TIME_NS",
5355
"TIME_TZ",
5456
"TINYINT",
5557
"UBIGINT",

‎src/duckdb_py/native/python_objects.cpp‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -585,13 +585,20 @@ py::object PythonObject::FromValue(const Value &val, const LogicalType &type,
585585
auto tmp_datetime_with_tz = import_cache.datetime.datetime.combine()(tmp_datetime, py_time, timezone_offset);
586586
return tmp_datetime_with_tz.attr("timetz")();
587587
}
588-
case LogicalTypeId::TIME: {
588+
case LogicalTypeId::TIME:
589+
case LogicalTypeId::TIME_NS: {
589590
D_ASSERT(type.InternalType() == PhysicalType::INT64);
590-
int32_t hour, min, sec, microsec;
591-
auto time = val.GetValueUnsafe<dtime_t>();
592-
duckdb::Time::Convert(time, hour, min, sec, microsec);
591+
int32_t hour, min, sec, usec;
592+
dtime_t time;
593+
if (type.id() == LogicalTypeId::TIME) {
594+
time = val.GetValueUnsafe<dtime_t>();
595+
} else {
596+
// Python's datetime doesn't support nanoseconds, we convert to micros.
597+
time = val.GetValueUnsafe<dtime_ns_t>().time();
598+
}
599+
duckdb::Time::Convert(time, hour, min, sec, usec);
593600
try {
594-
auto pytime = PyTime_FromTime(hour, min, sec, microsec);
601+
auto pytime = PyTime_FromTime(hour, min, sec, usec);
595602
if (!pytime) {
596603
throw py::error_already_set();
597604
}

‎src/duckdb_py/numpy/array_wrapper.cpp‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,24 @@ struct TimeConvert {
140140
}
141141
};
142142

143+
struct TimeNSConvert {
144+
template <class DUCKDB_T, class NUMPY_T>
145+
static PyObject *ConvertValue(dtime_ns_t val, NumpyAppendData &append_data) {
146+
auto &client_properties = append_data.client_properties;
147+
auto value = Value::TIME_NS(val);
148+
auto py_obj = PythonObject::FromValue(value, LogicalType::TIME_NS, client_properties);
149+
// Release ownership of the PyObject* without decreasing refcount
150+
// this returns a handle, of which we take the ptr to get the PyObject*
151+
return py_obj.release().ptr();
152+
}
153+
154+
template <class NUMPY_T, bool PANDAS>
155+
static NUMPY_T NullValue(bool &set_mask) {
156+
set_mask = true;
157+
return nullptr;
158+
}
159+
};
160+
143161
struct StringConvert {
144162
template <class DUCKDB_T, class NUMPY_T>
145163
static PyObject *ConvertValue(string_t val, NumpyAppendData &append_data) {
@@ -639,6 +657,9 @@ void ArrayWrapper::Append(idx_t current_offset, Vector &input, idx_t source_size
639657
case LogicalTypeId::TIME:
640658
may_have_null = ConvertColumn<dtime_t, PyObject *, duckdb_py_convert::TimeConvert>(append_data);
641659
break;
660+
case LogicalTypeId::TIME_NS:
661+
may_have_null = ConvertColumn<dtime_ns_t, PyObject *, duckdb_py_convert::TimeNSConvert>(append_data);
662+
break;
642663
case LogicalTypeId::INTERVAL:
643664
may_have_null = ConvertColumn<interval_t, int64_t, duckdb_py_convert::IntervalConvert>(append_data);
644665
break;

‎src/duckdb_py/numpy/raw_array_wrapper.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ static idx_t GetNumpyTypeWidth(const LogicalType &type) {
4848
case LogicalTypeId::TIMESTAMP_TZ:
4949
return sizeof(int64_t);
5050
case LogicalTypeId::TIME:
51+
case LogicalTypeId::TIME_NS:
5152
case LogicalTypeId::TIME_TZ:
5253
case LogicalTypeId::VARCHAR:
5354
case LogicalTypeId::BIT:
@@ -110,6 +111,7 @@ string RawArrayWrapper::DuckDBToNumpyDtype(const LogicalType &type) {
110111
case LogicalTypeId::INTERVAL:
111112
return "timedelta64[ns]";
112113
case LogicalTypeId::TIME:
114+
case LogicalTypeId::TIME_NS:
113115
case LogicalTypeId::TIME_TZ:
114116
case LogicalTypeId::VARCHAR:
115117
case LogicalTypeId::BIT:

‎src/duckdb_py/typing/typing.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ static void DefineBaseTypes(py::handle &m) {
2727
m.attr("TIMESTAMP_S") = make_shared_ptr<DuckDBPyType>(LogicalType::TIMESTAMP_S);
2828

2929
m.attr("TIME") = make_shared_ptr<DuckDBPyType>(LogicalType::TIME);
30+
m.attr("TIME_NS") = make_shared_ptr<DuckDBPyType>(LogicalType::TIME_NS);
3031

3132
m.attr("TIME_TZ") = make_shared_ptr<DuckDBPyType>(LogicalType::TIME_TZ);
3233
m.attr("TIMESTAMP_TZ") = make_shared_ptr<DuckDBPyType>(LogicalType::TIMESTAMP_TZ);

‎tests/fast/arrow/test_time.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ def test_time_types(self, duckdb_cursor):
2424
assert rel["a"] == arrow_table["c"]
2525
assert rel["b"] == arrow_table["c"]
2626
assert rel["c"] == arrow_table["c"]
27-
assert rel["d"] == arrow_table["c"]
27+
assert rel["d"] == arrow_table["d"]
2828

2929
def test_time_null(self, duckdb_cursor):
3030
if not can_run:
@@ -40,7 +40,7 @@ def test_time_null(self, duckdb_cursor):
4040
assert rel["a"] == arrow_table["c"]
4141
assert rel["b"] == arrow_table["c"]
4242
assert rel["c"] == arrow_table["c"]
43-
assert rel["d"] == arrow_table["c"]
43+
assert rel["d"] == arrow_table["d"]
4444

4545
def test_max_times(self, duckdb_cursor):
4646
if not can_run:
@@ -62,7 +62,7 @@ def test_max_times(self, duckdb_cursor):
6262
assert rel["a"] == result["a"]
6363

6464
# Max NSec
65-
data = pa.array([9223372036854774], type=pa.time64("us"))
65+
data = pa.array([9223372036854774000], type=pa.time64("ns"))
6666
result = pa.Table.from_arrays([data], ["a"])
6767
data = pa.array([9223372036854774000], type=pa.time64("ns"))
6868
arrow_table = pa.Table.from_arrays([data], ["a"])

‎tests/fast/spark/test_spark_types.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
StringType,
2929
StructField,
3030
StructType,
31+
TimeNSType,
3132
TimeNTZType,
3233
TimestampMillisecondNTZType,
3334
TimestampNanosecondNTZType,
@@ -129,5 +130,6 @@ def test_all_types_schema(self, spark):
129130
True,
130131
),
131132
StructField("map", MapType(StringType(), StringType(), True), True),
133+
StructField("time_ns", TimeNSType(), True),
132134
]
133135
)

0 commit comments

Comments
 (0)