Extract DuckDBPyConnection module state into DuckDBPyModuleState (#515)

Consolidate the five process-global statics (default_connection,
instance_cache, import_cache, environment, formatted_python_version)
into one DuckDBPyModuleState struct reached only through
GetModuleState(). The accessor returns a function-local singleton for
now, so behavior is unchanged; it is the single seam to retarget for PEP
489 multi-phase init / per-module state (PyModule_GetState) on the path
to free-threading. Removes the // NOLINT: allow global statics from
pyconnection.

Author: evertlammerts

src/duckdb_py/include/duckdb_python/pyconnection/pyconnection.hpp

@@ -361,11 +361,6 @@ struct DuckDBPyConnection : public std::enable_shared_from_this<DuckDBPyConnecti
 	void EnableProfiling();
 	void DisableProfiling();
 
-	//! Default connection to an in-memory database
-	static DefaultConnectionHolder default_connection;
-	//! Caches and provides an interface to get frequently used modules+subtypes
-	static std::shared_ptr<PythonImportCache> import_cache;
-
 	static bool IsPandasDataframe(const py::object &object);
 	static PyArrowObjectType GetArrowType(const py::handle &obj);
 	static bool IsAcceptedArrowObject(const py::object &object);
@@ -383,8 +378,6 @@ struct DuckDBPyConnection : public std::enable_shared_from_this<DuckDBPyConnecti
 	                               bool side_effects);
 	vector<unique_ptr<SQLStatement>> GetStatements(const py::object &query);
 
-	static PythonEnvironmentType environment;
-	static std::string formatted_python_version;
 	static void DetectEnvironment();
 };
 

src/duckdb_py/pyconnection.cpp

@@ -47,11 +47,21 @@
 
 namespace duckdb {
 
-DefaultConnectionHolder DuckDBPyConnection::default_connection;                        // NOLINT: allow global
-DBInstanceCache instance_cache;                                                        // NOLINT: allow global
-std::shared_ptr<PythonImportCache> DuckDBPyConnection::import_cache = nullptr;         // NOLINT: allow global
-PythonEnvironmentType DuckDBPyConnection::environment = PythonEnvironmentType::NORMAL; // NOLINT: allow global
-std::string DuckDBPyConnection::formatted_python_version = "";
+// All process-global module state lives in one struct, reached only through GetModuleState().
+// This is the single seam to retarget for PEP 489 multi-phase init (per-module state via
+// PyModule_GetState); call sites never touch the storage directly.
+struct DuckDBPyModuleState {
+	DefaultConnectionHolder default_connection;
+	DBInstanceCache instance_cache;
+	std::shared_ptr<PythonImportCache> import_cache;
+	PythonEnvironmentType environment = PythonEnvironmentType::NORMAL;
+	std::string formatted_python_version;
+};
+
+static DuckDBPyModuleState &GetModuleState() {
+	static DuckDBPyModuleState state; // NOLINT: allow global - sole module-state seam (future: PyModule_GetState)
+	return state;
+}
 
 DuckDBPyConnection::~DuckDBPyConnection() {
 	try {
@@ -91,14 +101,14 @@ void DuckDBPyConnection::DetectEnvironment() {
 	py::object version_info = sys.attr("version_info");
 	int major = py::cast<int>(version_info.attr("major"));
 	int minor = py::cast<int>(version_info.attr("minor"));
-	DuckDBPyConnection::formatted_python_version = std::to_string(major) + "." + std::to_string(minor);
+	GetModuleState().formatted_python_version = std::to_string(major) + "." + std::to_string(minor);
 
 	// If __main__ does not have a __file__ attribute, we are in interactive mode
 	auto main_module = py::module_::import("__main__");
 	if (py::hasattr(main_module, "__file__")) {
 		return;
 	}
-	DuckDBPyConnection::environment = PythonEnvironmentType::INTERACTIVE;
+	GetModuleState().environment = PythonEnvironmentType::INTERACTIVE;
 	if (!ModuleIsLoaded<IpythonCacheItem>()) {
 		return;
 	}
@@ -116,7 +126,7 @@ void DuckDBPyConnection::DetectEnvironment() {
 	}
 	py::dict ipython_config = ipython.attr("config");
 	if (ipython_config.contains("IPKernelApp")) {
-		DuckDBPyConnection::environment = PythonEnvironmentType::JUPYTER;
+		GetModuleState().environment = PythonEnvironmentType::JUPYTER;
 	}
 	return;
 }
@@ -127,11 +137,11 @@ bool DuckDBPyConnection::DetectAndGetEnvironment() {
 }
 
 bool DuckDBPyConnection::IsJupyter() {
-	return DuckDBPyConnection::environment == PythonEnvironmentType::JUPYTER;
+	return GetModuleState().environment == PythonEnvironmentType::JUPYTER;
 }
 
 std::string DuckDBPyConnection::FormattedPythonVersion() {
-	return DuckDBPyConnection::formatted_python_version;
+	return GetModuleState().formatted_python_version;
 }
 
 // NOTE: this function is generated by tools/pythonpkg/scripts/generate_connection_methods.py.
@@ -2199,8 +2209,8 @@ static std::shared_ptr<DuckDBPyConnection> FetchOrCreateInstance(const string &d
 		D_ASSERT(py::gil_check());
 		py::gil_scoped_release release;
 		unique_lock<std::recursive_mutex> lock(res->py_connection_lock);
-		auto database =
-		    instance_cache.GetOrCreateInstance(database_path, config, cache_instance, InstantiateNewInstance);
+		auto database = GetModuleState().instance_cache.GetOrCreateInstance(database_path, config, cache_instance,
+		                                                                    InstantiateNewInstance);
 		res->con.SetDatabase(std::move(database));
 		res->con.SetConnection(make_uniq<Connection>(res->con.GetDatabase()));
 	}
@@ -2285,14 +2295,15 @@ identifier_map_t<BoundParameterData> DuckDBPyConnection::TransformPythonParamDic
 }
 
 std::shared_ptr<DuckDBPyConnection> DuckDBPyConnection::DefaultConnection() {
-	return default_connection.Get();
+	return GetModuleState().default_connection.Get();
 }
 
 void DuckDBPyConnection::SetDefaultConnection(std::shared_ptr<DuckDBPyConnection> connection) {
-	return default_connection.Set(std::move(connection));
+	return GetModuleState().default_connection.Set(std::move(connection));
 }
 
 PythonImportCache *DuckDBPyConnection::ImportCache() {
+	auto &import_cache = GetModuleState().import_cache;
 	if (!import_cache) {
 		import_cache = std::make_shared<PythonImportCache>();
 	}
@@ -2316,7 +2327,7 @@ ModifiedMemoryFileSystem &DuckDBPyConnection::GetObjectFileSystem() {
 }
 
 bool DuckDBPyConnection::IsInteractive() {
-	return DuckDBPyConnection::environment != PythonEnvironmentType::NORMAL;
+	return GetModuleState().environment != PythonEnvironmentType::NORMAL;
 }
 
 std::shared_ptr<DuckDBPyConnection> DuckDBPyConnection::Enter() {
@@ -2334,8 +2345,8 @@ void DuckDBPyConnection::Exit(DuckDBPyConnection &self, const py::object &exc_ty
 }
 
 void DuckDBPyConnection::Cleanup() {
-	default_connection.Set(nullptr);
-	import_cache.reset();
+	GetModuleState().default_connection.Set(nullptr);
+	GetModuleState().import_cache.reset();
 }
 
 bool DuckDBPyConnection::IsPandasDataframe(const py::object &object) {