Fix transaction errors and more

Author: evertlammerts

src/duckdb_py/pyrelation.cpp

@@ -128,7 +128,8 @@ unique_ptr<DuckDBPyRelation> DuckDBPyRelation::ProjectFromTypes(const py::object
 		LogicalType type;
 		if (py::isinstance<py::str>(item)) {
 			string type_str = py::str(item);
-			type = TransformStringToLogicalType(type_str, *rel->context->GetContext());
+			rel->context->GetContext()->RunFunctionInTransaction(
+			    [&]() { type = TransformStringToLogicalType(type_str, *rel->context->GetContext().get()); });
 		} else if (py::isinstance<DuckDBPyType>(item)) {
 			auto *type_p = item.cast<DuckDBPyType *>();
 			type = type_p->Type();

src/duckdb_py/typing/pytype.cpp

@@ -122,12 +122,10 @@ static LogicalType FromString(const string &type_str, shared_ptr<DuckDBPyConnect
 		pycon = DuckDBPyConnection::DefaultConnection();
 	}
 	auto &connection = pycon->con.GetConnection();
-	if (connection.HasActiveTransaction()) {
-		return TransformStringToLogicalType(type_str, *connection.context);
-	}
-	connection.BeginTransaction();
-	auto type = TransformStringToLogicalType(type_str, *connection.context);
-	connection.Commit();
+
+	LogicalType type;
+	connection.context->RunFunctionInTransaction(
+	    [&]() { type = TransformStringToLogicalType(type_str, *connection.context); });
 	return type;
 }
 

tests/fast/arrow/test_filter_pushdown.py

@@ -2,23 +2,25 @@
 import sys
 
 import pytest
-from conftest import pandas_supports_arrow_backend
+from conftest import PANDAS_GE_3
 from packaging.version import Version
 
 import duckdb
 
 pa = pytest.importorskip("pyarrow")
-pd = pytest.importorskip("pyarrow.dataset")
+pa_ds = pytest.importorskip("pyarrow.dataset")
 pa_lib = pytest.importorskip("pyarrow.lib")
-pq = pytest.importorskip("pyarrow.parquet")
+pa_parquet = pytest.importorskip("pyarrow.parquet")
+pd = pytest.importorskip("pandas")
 np = pytest.importorskip("numpy")
 re = pytest.importorskip("re")
 
 
 def create_pyarrow_pandas(rel):
-    if not pandas_supports_arrow_backend():
-        pytest.skip(reason="Pandas version doesn't support 'pyarrow' backend")
-    return rel.df().convert_dtypes(dtype_backend="pyarrow")
+    if PANDAS_GE_3:
+        return rel.df()
+    else:
+        return rel.df().convert_dtypes(dtype_backend="pyarrow")
 
 
 def create_pyarrow_table(rel):
@@ -27,7 +29,7 @@ def create_pyarrow_table(rel):
 
 def create_pyarrow_dataset(rel):
     table = create_pyarrow_table(rel)
-    return pd.dataset(table)
+    return pa_ds.dataset(table)
 
 
 def test_decimal_filter_pushdown(duckdb_cursor):
@@ -550,7 +552,7 @@ def test_9371(self, duckdb_cursor, tmp_path):
         df = df.set_index("ts")  # SET INDEX! (It all works correctly when the index is not set)
         df.to_parquet(str(file_path))
 
-        my_arrow_dataset = pd.dataset(str(file_path))
+        my_arrow_dataset = pa_ds.dataset(str(file_path))
         res = duckdb_cursor.execute("SELECT * FROM my_arrow_dataset WHERE ts = ?", parameters=[dt]).fetch_arrow_table()
         output = duckdb_cursor.sql("select * from res").fetchall()
         expected = [(1, dt), (2, dt), (3, dt)]
@@ -708,34 +710,6 @@ def test_filter_pushdown_2145(self, duckdb_cursor, tmp_path, create_table):
         expected_df = duckdb.from_parquet(glob_pattern.as_posix()).filter("date > '2019-01-01'").df()
         pandas.testing.assert_frame_equal(expected_df, output_df)
 
-    # https://github.com/duckdb/duckdb/pull/4817/files#r1339973721
-    @pytest.mark.parametrize("create_table", [create_pyarrow_pandas, create_pyarrow_table])
-    def test_filter_column_removal(self, duckdb_cursor, create_table):
-        duckdb_cursor.execute(
-            """
-            CREATE TABLE test AS SELECT
-                range a,
-                100 - range b
-            FROM range(100)
-        """
-        )
-        duck_test_table = duckdb_cursor.table("test")
-        arrow_table = create_table(duck_test_table)
-
-        # PR 4817 - remove filter columns that are unused in the remainder of the query plan from the table function
-        query_res = duckdb_cursor.execute(
-            """
-            EXPLAIN SELECT count(*) FROM arrow_table WHERE
-                a > 25 AND b > 25
-        """
-        ).fetchall()
-
-        # scanned columns that come out of the scan are displayed like this, so we shouldn't see them
-        match = re.search("│ +a +│", query_res[0][1])
-        assert not match
-        match = re.search("│ +b +│", query_res[0][1])
-        assert not match
-
     @pytest.mark.skipif(sys.version_info < (3, 9), reason="Requires python 3.9")
     @pytest.mark.parametrize("create_table", [create_pyarrow_pandas, create_pyarrow_table])
     def test_struct_filter_pushdown(self, duckdb_cursor, create_table):
@@ -1023,36 +997,36 @@ def test_dynamic_filter(self, duckdb_cursor):
     def test_binary_view_filter(self, duckdb_cursor):
         """Filters on a view column work (without pushdown because pyarrow does not support view filters yet)."""
         table = pa.table({"col": pa.array([b"abc", b"efg"], type=pa.binary_view())})
-        dset = pd.dataset(table)
+        dset = pa_ds.dataset(table)
         res = duckdb_cursor.sql("select * from dset where col = 'abc'::binary")
         assert len(res) == 1
 
     def test_string_view_filter(self, duckdb_cursor):
         """Filters on a view column work (without pushdown because pyarrow does not support view filters yet)."""
         table = pa.table({"col": pa.array(["abc", "efg"], type=pa.string_view())})
-        dset = pd.dataset(table)
+        dset = pa_ds.dataset(table)
         res = duckdb_cursor.sql("select * from dset where col = 'abc'")
         assert len(res) == 1
 
     @pytest.mark.xfail(raises=pa_lib.ArrowNotImplementedError)
     def test_canary_for_pyarrow_string_view_filter_support(self, duckdb_cursor):
         """This canary will xpass when pyarrow implements string view filter support."""
         # predicate: field == "string value"
-        filter_expr = pd.field("col") == pd.scalar("val1")
+        filter_expr = pa_ds.field("col") == pa_ds.scalar("val1")
         # dataset with a string view column
         table = pa.table({"col": pa.array(["val1", "val2"], type=pa.string_view())})
-        dset = pd.dataset(table)
+        dset = pa_ds.dataset(table)
         # creating the scanner fails
         dset.scanner(columns=["col"], filter=filter_expr)
 
     @pytest.mark.xfail(raises=pa_lib.ArrowNotImplementedError)
     def test_canary_for_pyarrow_binary_view_filter_support(self, duckdb_cursor):
         """This canary will xpass when pyarrow implements binary view filter support."""
         # predicate: field == const
-        const = pd.scalar(pa.scalar(b"bin1", pa.binary_view()))
-        filter_expr = pd.field("col") == const
+        const = pa_ds.scalar(pa.scalar(b"bin1", pa.binary_view()))
+        filter_expr = pa_ds.field("col") == const
         # dataset with a string view column
         table = pa.table({"col": pa.array([b"bin1", b"bin2"], type=pa.binary_view())})
-        dset = pd.dataset(table)
+        dset = pa_ds.dataset(table)
         # creating the scanner fails
         dset.scanner(columns=["col"], filter=filter_expr)

tests/fast/pandas/test_pandas_arrow.py

@@ -2,7 +2,6 @@
 
 import numpy as np
 import pytest
-from conftest import pandas_supports_arrow_backend
 
 import duckdb
 
@@ -11,7 +10,6 @@
 from pandas.api.types import is_integer_dtype  # noqa: E402
 
 
-@pytest.mark.skipif(not pandas_supports_arrow_backend(), reason="pandas does not support the 'pyarrow' backend")
 class TestPandasArrow:
     def test_pandas_arrow(self, duckdb_cursor):
         pd = pytest.importorskip("pandas")

tests/fast/pandas/test_pyarrow_projection_pushdown.py

@@ -1,5 +1,4 @@
 import pytest
-from conftest import pandas_supports_arrow_backend
 
 import duckdb
 
@@ -8,7 +7,6 @@
 _ = pytest.importorskip("pandas", "2.0.0")
 
 
-@pytest.mark.skipif(not pandas_supports_arrow_backend(), reason="pandas does not support the 'pyarrow' backend")
 class TestArrowDFProjectionPushdown:
     def test_projection_pushdown_no_filter(self, duckdb_cursor):
         duckdb_conn = duckdb.connect()

tests/fast/test_replacement_scan.py

@@ -469,7 +469,8 @@ def test_replacement_disabled(self):
         with pytest.raises(duckdb.CatalogException, match="Table with name df does not exist!"):
             create_relation(con, "select * from df")
         with pytest.raises(
-            duckdb.InvalidInputException, match="Cannot change enable_external_access setting while database is running"
+            duckdb.InvalidInputException,
+            match="Invalid Input Error: Cannot enable external access while database is running",
         ):
             con.execute("set enable_external_access=true")