reorg of files and PyUtil extraction

Author: evertlammerts

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.29)
 
-project(duckdb_py LANGUAGES CXX)
+project(duckdb_python LANGUAGES CXX)
 
 # Always use C++17
 set(CMAKE_CXX_STANDARD 17)
@@ -72,8 +72,7 @@ target_link_libraries(_duckdb_dependencies INTERFACE nanobind-static
 # Also add include directory
 target_include_directories(
   _duckdb_dependencies
-  INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/duckdb_py/include>
-)
+  INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/include>)
 
 # We link duckdb_static. Without this define, duckdb.h marks C API symbols
 # __declspec(dllimport) on Windows, producing unresolvable __imp_* references at
@@ -99,7 +98,7 @@ endif()
 # ────────────────────────────────────────────
 # Descend into the real DuckDB‑Python sources
 # ────────────────────────────────────────────
-add_subdirectory(src/duckdb_py)
+add_subdirectory(src)
 
 nanobind_add_module(
   _duckdb

src/duckdb_py/CMakeLists.txt src/CMakeLists.txtsrc/duckdb_py/CMakeLists.txt renamed to src/CMakeLists.txt

@@ -15,7 +15,7 @@
 namespace duckdb {
 
 void TransformDuckToArrowChunk(py::object pyarrow_schema, ArrowArray &data, py::list &batches) {
-	py::gil_assert();
+	duckdb::PyUtil::GilAssert();
 	auto pyarrow_lib_module = py::module_::import_("pyarrow").attr("lib");
 	auto batch_import_func = pyarrow_lib_module.attr("RecordBatch").attr("_import_from_c");
 	batches.append(batch_import_func(reinterpret_cast<uint64_t>(&data), pyarrow_schema));
@@ -219,7 +219,7 @@ void PythonTableArrowArrayStreamFactory::GetSchemaInternal(py::handle arrow_obj_
 	// Scanner: use projected_schema; everything else (RecordBatchReader, Dataset): use .schema
 	VerifyArrowDatasetLoaded();
 	auto &import_cache = *DuckDBPyConnection::ImportCache();
-	if (py::isinstance(arrow_obj_handle, import_cache.pyarrow.dataset.Scanner())) {
+	if (duckdb::PyUtil::IsInstance(arrow_obj_handle, import_cache.pyarrow.dataset.Scanner())) {
 		auto obj_schema = arrow_obj_handle.attr("projected_schema");
 		obj_schema.attr("_export_to_c")(reinterpret_cast<uint64_t>(&schema.arrow_schema));
 	} else {

src/duckdb_py/arrow/CMakeLists.txt src/arrow/CMakeLists.txtsrc/duckdb_py/arrow/CMakeLists.txt renamed to src/arrow/CMakeLists.txt

@@ -6,7 +6,7 @@
 #include "duckdb/common/string_util.hpp"
 #include "duckdb_python/pybind11/pybind_wrapper.hpp"
 
-namespace py = nanobind;
+namespace nb = nanobind;
 
 namespace duckdb {
 
@@ -242,9 +242,9 @@ void PyThrowException(ErrorData &error, PyObject *http_exception) {
 	case ExceptionType::HTTP: {
 		// construct exception object
 		auto exc_msg = error.Message();
-		auto e = py::handle(http_exception)(py::str(exc_msg.c_str(), exc_msg.size()));
+		auto e = nb::handle(http_exception)(nb::str(exc_msg.c_str(), exc_msg.size()));
 
-		auto headers = py::dict();
+		auto headers = nb::dict();
 		for (auto &entry : error.ExtraInfo()) {
 			if (entry.first == "status_code") {
 				e.attr("status_code") = std::stoi(entry.second);
@@ -254,7 +254,7 @@ void PyThrowException(ErrorData &error, PyObject *http_exception) {
 				e.attr("reason") = entry.second;
 			} else if (StringUtil::StartsWith(entry.first, "header_")) {
 				auto header_name = entry.first.substr(7);
-				headers[py::str(header_name.c_str(), header_name.size())] = entry.second;
+				headers[nb::str(header_name.c_str(), header_name.size())] = entry.second;
 			}
 		}
 		e.attr("headers") = std::move(headers);
@@ -321,78 +321,78 @@ static void UnsetPythonException() {
 /**
  * @see https://peps.python.org/pep-0249/#exceptions
  */
-void RegisterExceptions(const py::module_ &m) {
+void RegisterExceptions(const nb::module_ &m) {
 	// The base class is mapped to Error in python to somewhat match the DBAPI 2.0 specifications
-	py::register_exception<Warning>(m, "Warning");
-	auto error = py::register_exception<PyError>(m, "Error").ptr();
-	auto db_error = py::register_exception<DatabaseError>(m, "DatabaseError", error).ptr();
+	nb::exception<Warning>(m, "Warning");
+	auto error = nb::exception<PyError>(m, "Error").ptr();
+	auto db_error = nb::exception<DatabaseError>(m, "DatabaseError", error).ptr();
 
 	// order of declaration matters, and this needs to be checked last
 	// Unknown
-	py::register_exception<PyFatalException>(m, "FatalException", db_error);
-	py::register_exception<PyInterruptException>(m, "InterruptException", db_error);
-	py::register_exception<PyPermissionException>(m, "PermissionException", db_error);
-	py::register_exception<PySequenceException>(m, "SequenceException", db_error);
-	py::register_exception<PyDependencyException>(m, "DependencyException", db_error);
+	nb::exception<PyFatalException>(m, "FatalException", db_error);
+	nb::exception<PyInterruptException>(m, "InterruptException", db_error);
+	nb::exception<PyPermissionException>(m, "PermissionException", db_error);
+	nb::exception<PySequenceException>(m, "SequenceException", db_error);
+	nb::exception<PyDependencyException>(m, "DependencyException", db_error);
 
 	// DataError
-	auto data_error = py::register_exception<DataError>(m, "DataError", db_error).ptr();
-	py::register_exception<PyOutOfRangeException>(m, "OutOfRangeException", data_error);
-	py::register_exception<PyConversionException>(m, "ConversionException", data_error);
+	auto data_error = nb::exception<DataError>(m, "DataError", db_error).ptr();
+	nb::exception<PyOutOfRangeException>(m, "OutOfRangeException", data_error);
+	nb::exception<PyConversionException>(m, "ConversionException", data_error);
 	// no unknown type error, or decimal type
-	py::register_exception<PyTypeMismatchException>(m, "TypeMismatchException", data_error);
+	nb::exception<PyTypeMismatchException>(m, "TypeMismatchException", data_error);
 
 	// OperationalError
-	auto operational_error = py::register_exception<OperationalError>(m, "OperationalError", db_error).ptr();
-	py::register_exception<PyTransactionException>(m, "TransactionException", operational_error);
-	py::register_exception<PyOutOfMemoryException>(m, "OutOfMemoryException", operational_error);
-	py::register_exception<PyConnectionException>(m, "ConnectionException", operational_error);
+	auto operational_error = nb::exception<OperationalError>(m, "OperationalError", db_error).ptr();
+	nb::exception<PyTransactionException>(m, "TransactionException", operational_error);
+	nb::exception<PyOutOfMemoryException>(m, "OutOfMemoryException", operational_error);
+	nb::exception<PyConnectionException>(m, "ConnectionException", operational_error);
 	// no object size error
 	// no null pointer errors
-	auto io_exception = py::register_exception<PyIOException>(m, "IOException", operational_error).ptr();
-	py::register_exception<PySerializationException>(m, "SerializationException", operational_error);
+	auto io_exception = nb::exception<PyIOException>(m, "IOException", operational_error).ptr();
+	nb::exception<PySerializationException>(m, "SerializationException", operational_error);
 
 	// Use a raw pointer to avoid destructor running after Python finalization.
 	// The module holds a reference to the exception type, keeping it alive.
 	static PyObject *HTTP_EXCEPTION = nullptr;
 	{
-		auto http_exc = py::register_exception<PyHTTPException>(m, "HTTPException", io_exception);
+		auto http_exc = nb::exception<PyHTTPException>(m, "HTTPException", io_exception);
 		HTTP_EXCEPTION = http_exc.ptr();
-		const auto string_type = (py::str("")).type();
-		const auto Dict = py::module_::import_("typing").attr("Dict");
-		// nanobind py::dict has no kwargs constructor; build the annotations dict explicitly.
-		py::dict annotations;
-		annotations["status_code"] = (py::int_(0)).type();
+		const auto string_type = (nb::str("")).type();
+		const auto Dict = nb::module_::import_("typing").attr("Dict");
+		// nanobind nb::dict has no kwargs constructor; build the annotations dict explicitly.
+		nb::dict annotations;
+		annotations["status_code"] = (nb::int_(0)).type();
 		annotations["body"] = string_type;
 		annotations["reason"] = string_type;
-		annotations["headers"] = Dict[py::make_tuple(string_type, string_type)];
+		annotations["headers"] = Dict[nb::make_tuple(string_type, string_type)];
 		http_exc.attr("__annotations__") = annotations;
 		http_exc.doc() = "Thrown when an error occurs in the httpfs extension, or whilst downloading an extension.";
 	}
 
 	// IntegrityError
-	auto integrity_error = py::register_exception<IntegrityError>(m, "IntegrityError", db_error).ptr();
-	py::register_exception<PyConstraintException>(m, "ConstraintException", integrity_error);
+	auto integrity_error = nb::exception<IntegrityError>(m, "IntegrityError", db_error).ptr();
+	nb::exception<PyConstraintException>(m, "ConstraintException", integrity_error);
 
 	// InternalError
-	auto internal_error = py::register_exception<InternalError>(m, "InternalError", db_error).ptr();
-	py::register_exception<PyInternalException>(m, "InternalException", internal_error);
+	auto internal_error = nb::exception<InternalError>(m, "InternalError", db_error).ptr();
+	nb::exception<PyInternalException>(m, "InternalException", internal_error);
 
 	//// ProgrammingError
-	auto programming_error = py::register_exception<ProgrammingError>(m, "ProgrammingError", db_error).ptr();
-	py::register_exception<PyParserException>(m, "ParserException", programming_error);
-	py::register_exception<PySyntaxException>(m, "SyntaxException", programming_error);
-	py::register_exception<PyBinderException>(m, "BinderException", programming_error);
-	py::register_exception<PyInvalidInputException>(m, "InvalidInputException", programming_error);
-	py::register_exception<PyInvalidTypeException>(m, "InvalidTypeException", programming_error);
+	auto programming_error = nb::exception<ProgrammingError>(m, "ProgrammingError", db_error).ptr();
+	nb::exception<PyParserException>(m, "ParserException", programming_error);
+	nb::exception<PySyntaxException>(m, "SyntaxException", programming_error);
+	nb::exception<PyBinderException>(m, "BinderException", programming_error);
+	nb::exception<PyInvalidInputException>(m, "InvalidInputException", programming_error);
+	nb::exception<PyInvalidTypeException>(m, "InvalidTypeException", programming_error);
 	// no type for expression exceptions?
-	py::register_exception<PyCatalogException>(m, "CatalogException", programming_error);
+	nb::exception<PyCatalogException>(m, "CatalogException", programming_error);
 
 	// NotSupportedError
-	auto not_supported_error = py::register_exception<NotSupportedError>(m, "NotSupportedError", db_error).ptr();
-	py::register_exception<PyNotImplementedException>(m, "NotImplementedException", not_supported_error);
+	auto not_supported_error = nb::exception<NotSupportedError>(m, "NotSupportedError", db_error).ptr();
+	nb::exception<PyNotImplementedException>(m, "NotImplementedException", not_supported_error);
 
-	py::register_exception_translator([](const std::exception_ptr &p, void *) {
+	nb::register_exception_translator([](const std::exception_ptr &p, void *) {
 		try {
 			if (p) {
 				std::rethrow_exception(p);
@@ -401,7 +401,7 @@ void RegisterExceptions(const py::module_ &m) {
 			duckdb::ErrorData error(ex);
 			UnsetPythonException();
 			PyThrowException(error, HTTP_EXCEPTION);
-		} catch (const py::builtin_exception &ex) {
+		} catch (const nb::builtin_exception &ex) {
 			// These represent Python exceptions, we don't want to catch these
 			throw;
 		} catch (const std::exception &ex) {