Fwd ports of:

PR #18642: Change arrow() to export record batch reader
PR #18624: Adjust filter pushdown to latest polars release
PR #18547: Rename the Varint type to Bignum

Author: evertlammerts

duckdb/__init__.pyi

@@ -180,6 +180,7 @@ class StatementType:
     DETACH: StatementType
     MULTI: StatementType
     COPY_DATABASE: StatementType
+    MERGE_INTO: StatementType
     def __int__(self) -> int: ...
     def __index__(self) -> int: ...
     @property
@@ -320,8 +321,8 @@ class DuckDBPyConnection:
     def fetch_df_chunk(self, vectors_per_chunk: int = 1, *, date_as_object: bool = False) -> pandas.DataFrame: ...
     def pl(self, rows_per_batch: int = 1000000, *, lazy: bool = False) -> polars.DataFrame: ...
     def fetch_arrow_table(self, rows_per_batch: int = 1000000) -> pyarrow.lib.Table: ...
-    def arrow(self, rows_per_batch: int = 1000000) -> pyarrow.lib.Table: ...
     def fetch_record_batch(self, rows_per_batch: int = 1000000) -> pyarrow.lib.RecordBatchReader: ...
+    def arrow(self, rows_per_batch: int = 1000000) -> pyarrow.lib.RecordBatchReader: ...
     def torch(self) -> dict: ...
     def tf(self) -> dict: ...
     def begin(self) -> DuckDBPyConnection: ...
@@ -668,8 +669,8 @@ def df(*, date_as_object: bool = False, connection: DuckDBPyConnection = ...) ->
 def fetch_df_chunk(vectors_per_chunk: int = 1, *, date_as_object: bool = False, connection: DuckDBPyConnection = ...) -> pandas.DataFrame: ...
 def pl(rows_per_batch: int = 1000000, *, lazy: bool = False, connection: DuckDBPyConnection = ...) -> polars.DataFrame: ...
 def fetch_arrow_table(rows_per_batch: int = 1000000, *, connection: DuckDBPyConnection = ...) -> pyarrow.lib.Table: ...
-def arrow(rows_per_batch: int = 1000000, *, connection: DuckDBPyConnection = ...) -> pyarrow.lib.Table: ...
 def fetch_record_batch(rows_per_batch: int = 1000000, *, connection: DuckDBPyConnection = ...) -> pyarrow.lib.RecordBatchReader: ...
+def arrow(rows_per_batch: int = 1000000, *, connection: DuckDBPyConnection = ...) -> pyarrow.lib.RecordBatchReader: ...
 def torch(*, connection: DuckDBPyConnection = ...) -> dict: ...
 def tf(*, connection: DuckDBPyConnection = ...) -> dict: ...
 def begin(*, connection: DuckDBPyConnection = ...) -> DuckDBPyConnection: ...

duckdb/polars_io.py

@@ -123,47 +123,61 @@ def _pl_tree_to_sql(tree: dict) -> str:
         raise NotImplementedError(f"Unsupported function type: {func_dict}")
 
     if node_type == "Scalar":
-        # Handle scalar values with typed representations
-        dtype = str(subtree["dtype"])
-        value = subtree["value"]
+        # Detect format: old style (dtype/value) or new style (direct type key)
+        if "dtype" in subtree and "value" in subtree:
+            dtype = str(subtree["dtype"])
+            value = subtree["value"]
+        else:
+            # New style: dtype is the single key in the dict
+            dtype = next(iter(subtree.keys()))
+            value = subtree
 
         # Decimal support
-        if dtype.startswith("{'Decimal'"):
+        if dtype.startswith("{'Decimal'") or dtype == "Decimal":
             decimal_value = value['Decimal']
             decimal_value = Decimal(decimal_value[0]) / Decimal(10 ** decimal_value[1])
             return str(decimal_value)
 
         # Datetime with microseconds since epoch
-        if dtype.startswith("{'Datetime'"):
+        if dtype.startswith("{'Datetime'") or dtype == "Datetime":
             micros = value['Datetime'][0]
             dt_timestamp = datetime.datetime.fromtimestamp(micros / 1_000_000, tz=datetime.UTC)
             return f"'{str(dt_timestamp)}'::TIMESTAMP"
 
-        # Match simple types
-        if dtype in ("Int8", "Int16", "Int32", "Int64", "UInt8", "UInt16", "UInt32", "UInt64", "Float32", "Float64", "Boolean"):
+        # Match simple numeric/boolean types
+        if dtype in ("Int8", "Int16", "Int32", "Int64",
+                     "UInt8", "UInt16", "UInt32", "UInt64",
+                     "Float32", "Float64", "Boolean"):
             return str(value[dtype])
 
+        # Time type
         if dtype == "Time":
-            # Convert nanoseconds to TIME
             nanoseconds = value["Time"]
             seconds = nanoseconds // 1_000_000_000
             microseconds = (nanoseconds % 1_000_000_000) // 1_000
-            dt_time = (datetime.datetime.min + datetime.timedelta(seconds=seconds, microseconds=microseconds)).time()
-            return f"'{str(dt_time)}'::TIME"
+            dt_time = (datetime.datetime.min + datetime.timedelta(
+                seconds=seconds, microseconds=microseconds
+            )).time()
+            return f"'{dt_time}'::TIME"
 
+        # Date type
         if dtype == "Date":
-            # Convert days since Unix epoch to SQL DATE
             days_since_epoch = value["Date"]
             date = datetime.date(1970, 1, 1) + datetime.timedelta(days=days_since_epoch)
-            return f"'{str(date)}'::DATE"
+            return f"'{date}'::DATE"
+
+        # Binary type
         if dtype == "Binary":
-            # Convert binary data to hex string for BLOB
             binary_data = bytes(value["Binary"])
             escaped = ''.join(f'\\x{b:02x}' for b in binary_data)
             return f"'{escaped}'::BLOB"
 
-        if dtype == "String":
-            return f"'{value['StringOwned']}'"
+        # String type
+        if dtype == "String" or dtype == "StringOwned":
+            # Some new formats may store directly under StringOwned
+            string_val = value.get("StringOwned", value.get("String", None))
+            return f"'{string_val}'"
+
 
         raise NotImplementedError(f"Unsupported scalar type {str(dtype)}, with value {value}")
 

scripts/connection_methods.json

@@ -395,10 +395,7 @@
 		"return": "polars.DataFrame"
 	},
 	{
-		"name": [
-			"fetch_arrow_table",
-			"arrow"
-		],
+		"name": "fetch_arrow_table",
 		"function": "FetchArrow",
 		"docs": "Fetch a result as Arrow table following execute()",
 		"args": [
@@ -411,7 +408,11 @@
 		"return": "pyarrow.lib.Table"
 	},
 	{
-		"name": "fetch_record_batch",
+		"name": [
+			"fetch_record_batch",
+			"arrow"
+		],
+		
 		"function": "FetchRecordBatchReader",
 		"docs": "Fetch an Arrow RecordBatchReader following execute()",
 		"args": [

src/duckdb_py/duckdb_python.cpp

@@ -427,17 +427,17 @@ static void InitializeConnectionMethods(py::module_ &m) {
 	    "Fetch a result as Arrow table following execute()", py::arg("rows_per_batch") = 1000000, py::kw_only(),
 	    py::arg("connection") = py::none());
 	m.def(
-	    "arrow",
-	    [](idx_t rows_per_batch, shared_ptr<DuckDBPyConnection> conn = nullptr) {
+	    "fetch_record_batch",
+	    [](const idx_t rows_per_batch, shared_ptr<DuckDBPyConnection> conn = nullptr) {
 		    if (!conn) {
 			    conn = DuckDBPyConnection::DefaultConnection();
 		    }
-		    return conn->FetchArrow(rows_per_batch);
+		    return conn->FetchRecordBatchReader(rows_per_batch);
 	    },
-	    "Fetch a result as Arrow table following execute()", py::arg("rows_per_batch") = 1000000, py::kw_only(),
+	    "Fetch an Arrow RecordBatchReader following execute()", py::arg("rows_per_batch") = 1000000, py::kw_only(),
 	    py::arg("connection") = py::none());
 	m.def(
-	    "fetch_record_batch",
+	    "arrow",
 	    [](const idx_t rows_per_batch, shared_ptr<DuckDBPyConnection> conn = nullptr) {
 		    if (!conn) {
 			    conn = DuckDBPyConnection::DefaultConnection();
@@ -971,21 +971,21 @@ static void InitializeConnectionMethods(py::module_ &m) {
 static void RegisterStatementType(py::handle &m) {
 	auto statement_type = py::enum_<duckdb::StatementType>(m, "StatementType");
 	static const duckdb::StatementType TYPES[] = {
-	    duckdb::StatementType::INVALID_STATEMENT,      duckdb::StatementType::SELECT_STATEMENT,
-	    duckdb::StatementType::INSERT_STATEMENT,       duckdb::StatementType::UPDATE_STATEMENT,
-	    duckdb::StatementType::CREATE_STATEMENT,       duckdb::StatementType::DELETE_STATEMENT,
-	    duckdb::StatementType::PREPARE_STATEMENT,      duckdb::StatementType::EXECUTE_STATEMENT,
-	    duckdb::StatementType::ALTER_STATEMENT,        duckdb::StatementType::TRANSACTION_STATEMENT,
-	    duckdb::StatementType::COPY_STATEMENT,         duckdb::StatementType::ANALYZE_STATEMENT,
-	    duckdb::StatementType::VARIABLE_SET_STATEMENT, duckdb::StatementType::CREATE_FUNC_STATEMENT,
-	    duckdb::StatementType::EXPLAIN_STATEMENT,      duckdb::StatementType::DROP_STATEMENT,
-	    duckdb::StatementType::EXPORT_STATEMENT,       duckdb::StatementType::PRAGMA_STATEMENT,
-	    duckdb::StatementType::VACUUM_STATEMENT,       duckdb::StatementType::CALL_STATEMENT,
-	    duckdb::StatementType::SET_STATEMENT,          duckdb::StatementType::LOAD_STATEMENT,
-	    duckdb::StatementType::RELATION_STATEMENT,     duckdb::StatementType::EXTENSION_STATEMENT,
-	    duckdb::StatementType::LOGICAL_PLAN_STATEMENT, duckdb::StatementType::ATTACH_STATEMENT,
-	    duckdb::StatementType::DETACH_STATEMENT,       duckdb::StatementType::MULTI_STATEMENT,
-	    duckdb::StatementType::COPY_DATABASE_STATEMENT};
+	    duckdb::StatementType::INVALID_STATEMENT,       duckdb::StatementType::SELECT_STATEMENT,
+	    duckdb::StatementType::INSERT_STATEMENT,        duckdb::StatementType::UPDATE_STATEMENT,
+	    duckdb::StatementType::CREATE_STATEMENT,        duckdb::StatementType::DELETE_STATEMENT,
+	    duckdb::StatementType::PREPARE_STATEMENT,       duckdb::StatementType::EXECUTE_STATEMENT,
+	    duckdb::StatementType::ALTER_STATEMENT,         duckdb::StatementType::TRANSACTION_STATEMENT,
+	    duckdb::StatementType::COPY_STATEMENT,          duckdb::StatementType::ANALYZE_STATEMENT,
+	    duckdb::StatementType::VARIABLE_SET_STATEMENT,  duckdb::StatementType::CREATE_FUNC_STATEMENT,
+	    duckdb::StatementType::EXPLAIN_STATEMENT,       duckdb::StatementType::DROP_STATEMENT,
+	    duckdb::StatementType::EXPORT_STATEMENT,        duckdb::StatementType::PRAGMA_STATEMENT,
+	    duckdb::StatementType::VACUUM_STATEMENT,        duckdb::StatementType::CALL_STATEMENT,
+	    duckdb::StatementType::SET_STATEMENT,           duckdb::StatementType::LOAD_STATEMENT,
+	    duckdb::StatementType::RELATION_STATEMENT,      duckdb::StatementType::EXTENSION_STATEMENT,
+	    duckdb::StatementType::LOGICAL_PLAN_STATEMENT,  duckdb::StatementType::ATTACH_STATEMENT,
+	    duckdb::StatementType::DETACH_STATEMENT,        duckdb::StatementType::MULTI_STATEMENT,
+	    duckdb::StatementType::COPY_DATABASE_STATEMENT, duckdb::StatementType::MERGE_INTO_STATEMENT};
 	static const idx_t AMOUNT = sizeof(TYPES) / sizeof(duckdb::StatementType);
 	for (idx_t i = 0; i < AMOUNT; i++) {
 		auto &type = TYPES[i];

src/duckdb_py/native/python_objects.cpp

@@ -8,7 +8,7 @@
 #include "duckdb/common/operator/cast_operators.hpp"
 #include "duckdb_python/pyconnection/pyconnection.hpp"
 #include "duckdb/common/operator/add.hpp"
-#include "duckdb/common/types/varint.hpp"
+#include "duckdb/common/types/bignum.hpp"
 #include "duckdb/function/to_interval.hpp"
 
 #include "datetime.h" // Python datetime initialize #1
@@ -683,9 +683,9 @@ py::object PythonObject::FromValue(const Value &val, const LogicalType &type,
 		auto uuid_value = val.GetValueUnsafe<hugeint_t>();
 		return import_cache.uuid.UUID()(UUID::ToString(uuid_value));
 	}
-	case LogicalTypeId::VARINT: {
-		auto varint_value = val.GetValueUnsafe<string_t>();
-		return py::str(Varint::VarIntToVarchar(varint_value));
+	case LogicalTypeId::BIGNUM: {
+		auto bignum_value = val.GetValueUnsafe<bignum_t>();
+		return py::str(Bignum::BignumToVarchar(bignum_value));
 	}
 	case LogicalTypeId::INTERVAL: {
 		auto interval_value = val.GetValueUnsafe<interval_t>();

src/duckdb_py/pyconnection.cpp

@@ -204,10 +204,10 @@ static void InitializeConnectionMethods(py::class_<DuckDBPyConnection, shared_pt
 	      py::arg("rows_per_batch") = 1000000, py::kw_only(), py::arg("lazy") = false);
 	m.def("fetch_arrow_table", &DuckDBPyConnection::FetchArrow, "Fetch a result as Arrow table following execute()",
 	      py::arg("rows_per_batch") = 1000000);
-	m.def("arrow", &DuckDBPyConnection::FetchArrow, "Fetch a result as Arrow table following execute()",
-	      py::arg("rows_per_batch") = 1000000);
 	m.def("fetch_record_batch", &DuckDBPyConnection::FetchRecordBatchReader,
 	      "Fetch an Arrow RecordBatchReader following execute()", py::arg("rows_per_batch") = 1000000);
+	m.def("arrow", &DuckDBPyConnection::FetchRecordBatchReader, "Fetch an Arrow RecordBatchReader following execute()",
+	      py::arg("rows_per_batch") = 1000000);
 	m.def("torch", &DuckDBPyConnection::FetchPyTorch, "Fetch a result as dict of PyTorch Tensors following execute()");
 	m.def("tf", &DuckDBPyConnection::FetchTF, "Fetch a result as dict of TensorFlow Tensors following execute()");
 	m.def("begin", &DuckDBPyConnection::Begin, "Start a new transaction");

tests/fast/api/test_dbapi_fetch.py

@@ -40,11 +40,11 @@ def test_multiple_fetch_arrow(self, duckdb_cursor):
         arrow = pytest.importorskip("pyarrow")
         con = duckdb.connect()
         c = con.execute('SELECT 42::BIGINT AS a')
-        table = c.arrow()
+        table = c.fetch_arrow_table()
         df = table.to_pandas()
         pd.testing.assert_frame_equal(df, pd.DataFrame.from_dict({'a': [42]}))
-        assert c.arrow() is None
-        assert c.arrow() is None
+        assert c.fetch_arrow_table() is None
+        assert c.fetch_arrow_table() is None
 
     def test_multiple_close(self, duckdb_cursor):
         con = duckdb.connect()

tests/fast/api/test_duckdb_connection.py

@@ -52,7 +52,7 @@ def test_default_connection_from_connect(self):
     def test_arrow(self):
         pyarrow = pytest.importorskip("pyarrow")
         duckdb.execute("select [1,2,3]")
-        result = duckdb.arrow()
+        result = duckdb.fetch_arrow_table()
 
     def test_begin_commit(self):
         duckdb.begin()
@@ -299,7 +299,7 @@ def test_unregister_problematic_behavior(self, duckdb_cursor):
         assert duckdb_cursor.execute("select * from vw").fetchone() == (0,)
 
         # Create a registered object called 'vw'
-        arrow_result = duckdb_cursor.execute("select 42").arrow()
+        arrow_result = duckdb_cursor.execute("select 42").fetch_arrow_table()
         with pytest.raises(duckdb.CatalogException, match='View with name "vw" already exists'):
             duckdb_cursor.register('vw', arrow_result)
 

tests/fast/api/test_native_tz.py

@@ -59,25 +59,41 @@ def test_pandas_timestamp_time(self, duckdb_cursor):
     )
     def test_arrow_timestamp_timezone(self, duckdb_cursor):
         res = duckdb_cursor.execute("SET timezone='America/Los_Angeles';")
-        table = duckdb_cursor.execute(f"select TimeRecStart as tz  from '{filename}'").arrow()
+        table = duckdb_cursor.execute(f"select TimeRecStart as tz  from '{filename}'").fetch_arrow_table()
         res = table.to_pandas()
         assert res.dtypes["tz"].tz.zone == 'America/Los_Angeles'
         assert res['tz'][0].hour == 14 and res['tz'][0].minute == 52
 
         duckdb_cursor.execute("SET timezone='UTC';")
-        res = duckdb_cursor.execute(f"select TimeRecStart as tz  from '{filename}'").arrow().to_pandas()
+        res = duckdb_cursor.execute(f"select TimeRecStart as tz  from '{filename}'").fetch_arrow_table().to_pandas()
         assert res.dtypes["tz"].tz.zone == 'UTC'
         assert res['tz'][0].hour == 21 and res['tz'][0].minute == 52
 
     def test_arrow_timestamp_time(self, duckdb_cursor):
         duckdb_cursor.execute("SET timezone='America/Los_Angeles';")
-        res1 = duckdb_cursor.execute(f"select TimeRecStart::TIMETZ  as tz  from '{filename}'").arrow().to_pandas()
-        res2 = duckdb_cursor.execute(f"select TimeRecStart::TIMETZ::TIME  as tz  from '{filename}'").arrow().to_pandas()
+        res1 = (
+            duckdb_cursor.execute(f"select TimeRecStart::TIMETZ  as tz  from '{filename}'")
+            .fetch_arrow_table()
+            .to_pandas()
+        )
+        res2 = (
+            duckdb_cursor.execute(f"select TimeRecStart::TIMETZ::TIME  as tz  from '{filename}'")
+            .fetch_arrow_table()
+            .to_pandas()
+        )
         assert res1['tz'][0].hour == 14 and res1['tz'][0].minute == 52
         assert res2['tz'][0].hour == res2['tz'][0].hour and res2['tz'][0].minute == res1['tz'][0].minute
 
         duckdb_cursor.execute("SET timezone='UTC';")
-        res1 = duckdb_cursor.execute(f"select TimeRecStart::TIMETZ  as tz  from '{filename}'").arrow().to_pandas()
-        res2 = duckdb_cursor.execute(f"select TimeRecStart::TIMETZ::TIME  as tz  from '{filename}'").arrow().to_pandas()
+        res1 = (
+            duckdb_cursor.execute(f"select TimeRecStart::TIMETZ  as tz  from '{filename}'")
+            .fetch_arrow_table()
+            .to_pandas()
+        )
+        res2 = (
+            duckdb_cursor.execute(f"select TimeRecStart::TIMETZ::TIME  as tz  from '{filename}'")
+            .fetch_arrow_table()
+            .to_pandas()
+        )
         assert res1['tz'][0].hour == 21 and res1['tz'][0].minute == 52
         assert res2['tz'][0].hour == res2['tz'][0].hour and res2['tz'][0].minute == res1['tz'][0].minute

tests/fast/arrow/test_6584.py

@@ -7,7 +7,7 @@
 
 def f(cur, i, data):
     cur.execute(f"create table t_{i} as select * from data")
-    return cur.execute(f"select * from t_{i}").arrow()
+    return cur.execute(f"select * from t_{i}").fetch_arrow_table()
 
 
 def test_6584():