Merge v1.5-variegata and integrate the latest DuckDB main (#494)

Merges the v1.5-variegata branch into main and updates the DuckDB
submodule to latest main, which pulls in a big wave of breaking core API
changes.

The most important fixes:
- Migrating table filters to expression filters
- Migrating string identifiers to the `Identifier` type
- Rewrote the buggy enum `type_caster`s (pybind's caster machinery it
uses to cast between python objects and e.g. `ExplainType`,
`PythonUDFType`, `FunctionNullHandling`, …) to not violate the ODR
- Added timestamp_tz_ns support (incl. pyspark)
- Passing `ClientContext` to `MaxLogicalType`/`ForceMaxLogicalType`
(cascading into `TransformPythonValue`/`TransformPythonObject` calls).
- Migrating `SetCardinality` calls to `SetChildCardinality` and
`CheckCardinality` wherever needed
- Supporting `SQLNULL` in result schemas.
- Added support for `EXPLAIN` / profiler's `ProfilerPrintFormat` through
a `format=` param in `explain()`
- Adapted ADBC tests to the default `TransactionInvalidationPolicy`

One remaining follow-up: the `query_graph` HTML profiler renderer is
outdated because of the new nested profiling schema. I'm xfailing the
test now so we get CI green again, but will fix before v2.0. cc
@guillesd

Author: evertlammerts

CLAUDE.md

@@ -121,7 +121,7 @@ Supported: `3.10`, `3.11`, `3.12`, `3.13`, `3.14`. Do **not** use free-threaded
 
 Key `pyproject.toml` settings:
 
-- `BUILD_EXTENSIONS = "core_functions;json;parquet;icu;jemalloc"` — extensions built into the wheel.
+- `BUILD_EXTENSIONS = "core_functions;json;parquet;icu"` — extensions built into the wheel. (jemalloc is part of DuckDB core and is auto-enabled on supported platforms — 64-bit Linux, non-musl, non-BSD.)
 - Editable overrides: `build-dir = "build/debug/"`, `editable.rebuild = true`, `editable.mode = "redirect"`, `cmake.build-type = "Debug"`, `DISABLE_UNITY = "1"` (unity disabled for better debugging).
 - Coverage overrides: `build-dir = "build/coverage/"`, `RelWithDebInfo`, `--coverage` flags. Activate with `COVERAGE=true uv sync ...`.
 

_duckdb-stubs/__init__.pyi

@@ -309,22 +309,15 @@ class DuckDBPyConnection:
         strict_mode: bool | None = None,
     ) -> DuckDBPyRelation: ...
     def from_df(self, df: pandas.DataFrame) -> DuckDBPyRelation: ...
-    @typing.overload
-    def from_parquet(
-        self,
-        file_glob: str,
-        binary_as_string: bool = False,
-        *,
-        file_row_number: bool = False,
-        filename: bool = False,
-        hive_partitioning: bool = False,
-        union_by_name: bool = False,
-        compression: ParquetCompression | None = None,
-    ) -> DuckDBPyRelation: ...
-    @typing.overload
     def from_parquet(
         self,
-        file_globs: Sequence[str],
+        path_or_buffer: str
+        | bytes
+        | os.PathLike[str]
+        | os.PathLike[bytes]
+        | typing.IO[bytes]
+        | typing.IO[str]
+        | Sequence[str | bytes | os.PathLike[str] | os.PathLike[bytes] | typing.IO[bytes] | typing.IO[str]],
         binary_as_string: bool = False,
         *,
         file_row_number: bool = False,
@@ -433,22 +426,15 @@ class DuckDBPyConnection:
         hive_types: HiveTypes | None = None,
         hive_types_autocast: bool | None = None,
     ) -> DuckDBPyRelation: ...
-    @typing.overload
-    def read_parquet(
-        self,
-        file_glob: str,
-        binary_as_string: bool = False,
-        *,
-        file_row_number: bool = False,
-        filename: bool = False,
-        hive_partitioning: bool = False,
-        union_by_name: bool = False,
-        compression: ParquetCompression | None = None,
-    ) -> DuckDBPyRelation: ...
-    @typing.overload
     def read_parquet(
         self,
-        file_globs: Sequence[str],
+        path_or_buffer: str
+        | bytes
+        | os.PathLike[str]
+        | os.PathLike[bytes]
+        | typing.IO[bytes]
+        | typing.IO[str]
+        | Sequence[str | bytes | os.PathLike[str] | os.PathLike[bytes] | typing.IO[bytes] | typing.IO[str]],
         binary_as_string: bool = False,
         *,
         file_row_number: bool = False,
@@ -551,7 +537,9 @@ class DuckDBPyRelation:
     def distinct(self) -> DuckDBPyRelation: ...
     def except_(self, other_rel: Self) -> DuckDBPyRelation: ...
     def execute(self) -> DuckDBPyRelation: ...
-    def explain(self, type: ExplainType | ExplainTypeLiteral = ExplainType.STANDARD) -> str: ...
+    def explain(
+        self, type: ExplainType | ExplainTypeLiteral = ExplainType.STANDARD, format: str | None = None
+    ) -> str: ...
     def favg(
         self, expression: str, groups: str = "", window_spec: str = "", projected_columns: str = ""
     ) -> DuckDBPyRelation: ...
@@ -1061,21 +1049,14 @@ def from_csv_auto(
     strict_mode: bool | None = None,
 ) -> DuckDBPyRelation: ...
 def from_df(df: pandas.DataFrame, *, connection: DuckDBPyConnection | None = None) -> DuckDBPyRelation: ...
-@typing.overload
-def from_parquet(
-    file_glob: str,
-    binary_as_string: bool = False,
-    *,
-    file_row_number: bool = False,
-    filename: bool = False,
-    hive_partitioning: bool = False,
-    union_by_name: bool = False,
-    compression: ParquetCompression | None = None,
-    connection: DuckDBPyConnection | None = None,
-) -> DuckDBPyRelation: ...
-@typing.overload
 def from_parquet(
-    file_globs: Sequence[str],
+    path_or_buffer: str
+    | bytes
+    | os.PathLike[str]
+    | os.PathLike[bytes]
+    | typing.IO[bytes]
+    | typing.IO[str]
+    | Sequence[str | bytes | os.PathLike[str] | os.PathLike[bytes] | typing.IO[bytes] | typing.IO[str]],
     binary_as_string: bool = False,
     *,
     file_row_number: bool = False,
@@ -1232,21 +1213,14 @@ def read_json(
     hive_types: HiveTypes | None = None,
     hive_types_autocast: bool | None = None,
 ) -> DuckDBPyRelation: ...
-@typing.overload
-def read_parquet(
-    file_glob: str,
-    binary_as_string: bool = False,
-    *,
-    file_row_number: bool = False,
-    filename: bool = False,
-    hive_partitioning: bool = False,
-    union_by_name: bool = False,
-    compression: ParquetCompression | None = None,
-    connection: DuckDBPyConnection | None = None,
-) -> DuckDBPyRelation: ...
-@typing.overload
 def read_parquet(
-    file_globs: Sequence[str],
+    path_or_buffer: str
+    | bytes
+    | os.PathLike[str]
+    | os.PathLike[bytes]
+    | typing.IO[bytes]
+    | typing.IO[str]
+    | Sequence[str | bytes | os.PathLike[str] | os.PathLike[bytes] | typing.IO[bytes] | typing.IO[str]],
     binary_as_string: bool = False,
     *,
     file_row_number: bool = False,

cmake/duckdb_loader.cmake

@@ -3,8 +3,7 @@
 # Simple DuckDB Build Configuration Module
 #
 # Sets sensible defaults for DuckDB Python extension builds and provides a clean
-# interface for adding DuckDB as a library target. Adds jemalloc option for
-# debugging but will never allow jemalloc in a release build if not on Linux.
+# interface for adding DuckDB as a library target.
 #
 # Usage: include(cmake/duckdb_loader.cmake) # Optionally load extensions
 # set(BUILD_EXTENSIONS "json;parquet;icu")
@@ -108,37 +107,6 @@ set(DEBUG_STACKTRACE
 # Internal Functions
 # ════════════════════════════════════════════════════════════════════════════════
 
-function(_duckdb_validate_jemalloc_config)
-  # Check if jemalloc is in the extension list
-  if(NOT BUILD_EXTENSIONS MATCHES "jemalloc")
-    return()
-  endif()
-
-  # jemalloc is only enabled on 64bit x86 linux builds
-  if(CMAKE_SIZEOF_VOID_P EQUAL 8
-     AND CMAKE_SYSTEM_NAME STREQUAL "Linux"
-     AND NOT BSD)
-    set(jemalloc_allowed TRUE)
-  else()
-    set(jemalloc_allowed FALSE)
-  endif()
-
-  if(NOT jemalloc_allowed)
-    message(WARNING "jemalloc extension is only supported on Linux.\n"
-                    "Removing jemalloc from extension list.")
-    # Remove jemalloc from the extension list
-    string(REPLACE "jemalloc" "" BUILD_EXTENSIONS_FILTERED
-                   "${BUILD_EXTENSIONS}")
-    string(REGEX REPLACE ";+" ";" BUILD_EXTENSIONS_FILTERED
-                         "${BUILD_EXTENSIONS_FILTERED}")
-    string(REGEX REPLACE "^;|;$" "" BUILD_EXTENSIONS_FILTERED
-                         "${BUILD_EXTENSIONS_FILTERED}")
-    set(BUILD_EXTENSIONS
-        "${BUILD_EXTENSIONS_FILTERED}"
-        PARENT_SCOPE)
-  endif()
-endfunction()
-
 function(_duckdb_validate_source_path)
   if(NOT EXISTS "${DUCKDB_SOURCE_PATH}")
     message(
@@ -234,19 +202,13 @@ endfunction()
 
 function(duckdb_add_library target_name)
   _duckdb_validate_source_path()
-  _duckdb_validate_jemalloc_config()
   _duckdb_print_summary()
 
   # Add DuckDB subdirectory - it will use our variables
   add_subdirectory("${DUCKDB_SOURCE_PATH}" duckdb EXCLUDE_FROM_ALL)
 
   # Create clean interface target
   _duckdb_create_interface_target(${target_name})
-
-  # Propagate BUILD_EXTENSIONS back to caller scope in case it was modified
-  set(BUILD_EXTENSIONS
-      "${BUILD_EXTENSIONS}"
-      PARENT_SCOPE)
 endfunction()
 
 function(duckdb_link_extensions target_name)

duckdb/experimental/spark/sql/type_utils.py

@@ -19,6 +19,7 @@
     IntegerType,
     LongType,
     MapType,
+    NullType,
     ShortType,
     StringType,
     StructField,
@@ -27,6 +28,7 @@
     TimeNTZType,
     TimestampMillisecondNTZType,
     TimestampNanosecondNTZType,
+    TimestampNanosecondType,
     TimestampNTZType,
     TimestampSecondNTZType,
     TimestampType,
@@ -41,6 +43,7 @@
 )
 
 _sqltype_to_spark_class = {
+    "null": NullType,
     "boolean": BooleanType,
     "utinyint": UnsignedByteType,
     "tinyint": ByteType,
@@ -62,9 +65,10 @@
     "time with time zone": TimeType,
     "timestamp": TimestampNTZType,
     "timestamp with time zone": TimestampType,
-    "timestamp_ms": TimestampNanosecondNTZType,
-    "timestamp_ns": TimestampMillisecondNTZType,
+    "timestamp_ms": TimestampMillisecondNTZType,
+    "timestamp_ns": TimestampNanosecondNTZType,
     "timestamp_s": TimestampSecondNTZType,
+    "timestamptz_ns": TimestampNanosecondType,
     "interval": DayTimeIntervalType,
     "list": ArrayType,
     "struct": StructType,

duckdb/experimental/spark/sql/types.py

@@ -49,6 +49,7 @@
     "TimestampMillisecondNTZType",
     "TimestampNTZType",
     "TimestampNanosecondNTZType",
+    "TimestampNanosecondType",
     "TimestampSecondNTZType",
     "TimestampType",
     "UUIDType",
@@ -239,6 +240,26 @@ def fromInternal(self, ts: int) -> datetime.datetime:  # noqa: D102
             return datetime.datetime.fromtimestamp(ts // 1000000).replace(microsecond=ts % 1000000)
 
 
+class TimestampNanosecondType(AtomicType, metaclass=DataTypeSingleton):
+    """Timestamp (datetime.datetime) data type with timezone information with nanosecond precision."""
+
+    def __init__(self) -> None:  # noqa: D107
+        super().__init__(DuckDBPyType("TIMESTAMPTZ_NS"))
+
+    def needConversion(self) -> bool:  # noqa: D102
+        return True
+
+    @classmethod
+    def typeName(cls) -> str:  # noqa: D102
+        return "timestamptz_ns"
+
+    def toInternal(self, dt: datetime.datetime) -> int:  # noqa: D102
+        raise ContributionsAcceptedError
+
+    def fromInternal(self, ts: int) -> datetime.datetime:  # noqa: D102
+        raise ContributionsAcceptedError
+
+
 class TimestampNTZType(AtomicType, metaclass=DataTypeSingleton):
     """Timestamp (datetime.datetime) data type without timezone information with microsecond precision."""