From 1185322b465c55898b71319c89debd59473f59e2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 May 2025 16:02:57 -0400 Subject: [PATCH 1/6] Minimal support for multi-phase init --- mypyc/codegen/emitmodule.py | 40 ++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index a3970b9c181e9..d19212b1f2044 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -513,6 +513,7 @@ def __init__( self.use_shared_lib = group_name is not None self.compiler_options = compiler_options self.multi_file = compiler_options.multi_file + self.multi_phase_init = True @property def group_suffix(self) -> str: @@ -869,10 +870,24 @@ def generate_module_def(self, emitter: Emitter, module_name: str, module: Module """Emit the PyModuleDef struct for a module and the module init function.""" module_prefix = emitter.names.private_name(module_name) self.emit_module_exec_func(emitter, module_name, module_prefix, module) + if self.multi_phase_init: + self.emit_module_def_slots(emitter, module_prefix) self.emit_module_methods(emitter, module_name, module_prefix, module) self.emit_module_def_struct(emitter, module_name, module_prefix) self.emit_module_init_func(emitter, module_name, module_prefix) + def emit_module_def_slots(self, emitter: Emitter, module_prefix: str) -> None: + name = f"{module_prefix}_slots" + exec_name = f"{module_prefix}_exec" + + emitter.emit_line(f"static PyModuleDef_Slot {name}[] = {{") + emitter.emit_line(f"{{Py_mod_exec, {exec_name}}},") + emitter.emit_line( + "{Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}," + ) + emitter.emit_line("{0, NULL},") + emitter.emit_line("};") + def emit_module_methods( self, emitter: Emitter, module_name: str, module_prefix: str, module: ModuleIR ) -> None: @@ -905,11 +920,15 @@ def emit_module_def_struct( "PyModuleDef_HEAD_INIT,", f'"{module_name}",', "NULL, /* docstring */", - "-1, /* size of per-interpreter state of the module,", - " or -1 if the module keeps state in global variables. */", - f"{module_prefix}module_methods", - "};", + "0, /* size of per-interpreter state of the module */", + f"{module_prefix}module_methods,", ) + if self.multi_phase_init: + slots_name = f"{module_prefix}_slots" + emitter.emit_line(f"{slots_name}, /* m_slots */") + else: + emitter.emit_line("NULL,") + emitter.emit_line("};") emitter.emit_line() def emit_module_exec_func( @@ -927,6 +946,8 @@ def emit_module_exec_func( module_static = self.module_internal_static_name(module_name, emitter) emitter.emit_lines(declaration, "{") emitter.emit_line("PyObject* modname = NULL;") + if self.multi_phase_init: + emitter.emit_line(f"{module_static} = module;") emitter.emit_line( f'modname = PyObject_GetAttrString((PyObject *){module_static}, "__name__");' ) @@ -958,7 +979,10 @@ def emit_module_exec_func( emitter.emit_line("return 0;") emitter.emit_lines("fail:") - emitter.emit_lines(f"Py_CLEAR({module_static});", "Py_CLEAR(modname);") + if self.multi_phase_init: + emitter.emit_lines(f"{module_static} = NULL;", "Py_CLEAR(modname);") + else: + emitter.emit_lines(f"Py_CLEAR({module_static});", "Py_CLEAR(modname);") for name, typ in module.final_names: static_name = emitter.static_name(name, module_name) emitter.emit_dec_ref(static_name, typ, is_xdec=True) @@ -980,6 +1004,12 @@ def emit_module_init_func( declaration = f"PyObject *CPyInit_{exported_name(module_name)}(void)" emitter.emit_lines(declaration, "{") + if self.multi_phase_init: + def_name = f"{module_prefix}module" + emitter.emit_line(f"return PyModuleDef_Init(&{def_name});") + emitter.emit_line("}") + return + exec_func = f"{module_prefix}_exec" # Store the module reference in a static and return it when necessary. From e115a1093d6138ee50af291e6a15633d6e5bc19c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 20 May 2025 16:07:55 -0400 Subject: [PATCH 2/6] Enable free threading --- mypyc/codegen/emitmodule.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index d19212b1f2044..48a46f5ba1bab 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -885,6 +885,7 @@ def emit_module_def_slots(self, emitter: Emitter, module_prefix: str) -> None: emitter.emit_line( "{Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}," ) + emitter.emit_line("{Py_mod_gil, Py_MOD_GIL_NOT_USED},") emitter.emit_line("{0, NULL},") emitter.emit_line("};") From 5551be7ca19cd253d1e8f07439ceb7aba8ac4919 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 May 2025 11:32:27 -0400 Subject: [PATCH 3/6] Fix running on older Python versions --- mypyc/codegen/emitmodule.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 48a46f5ba1bab..9d5f40db6ab3a 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -7,6 +7,7 @@ import json import os +import sys from collections.abc import Iterable from typing import Optional, TypeVar @@ -882,10 +883,12 @@ def emit_module_def_slots(self, emitter: Emitter, module_prefix: str) -> None: emitter.emit_line(f"static PyModuleDef_Slot {name}[] = {{") emitter.emit_line(f"{{Py_mod_exec, {exec_name}}},") - emitter.emit_line( - "{Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}," - ) - emitter.emit_line("{Py_mod_gil, Py_MOD_GIL_NOT_USED},") + if sys.version_info >= (3, 12): + emitter.emit_line( + "{Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}," + ) + if sys.version_info >= (3, 13): + emitter.emit_line("{Py_mod_gil, Py_MOD_GIL_NOT_USED},") emitter.emit_line("{0, NULL},") emitter.emit_line("};") From 8f818ca1e7a865c4107f3cac4b950e113ebfd52a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 May 2025 14:12:36 +0100 Subject: [PATCH 4/6] WIP --- mypyc/common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypyc/common.py b/mypyc/common.py index 9923764720864..b5506eed89c22 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -88,6 +88,10 @@ # some details in the PEP are out of date. HAVE_IMMORTAL: Final = sys.version_info >= (3, 12) +# Are we running on a free-threaded build (GIL disabled)? This implies that +# we are on Python 3.13 or later. +IS_FREE_THREADED: Final = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + JsonDict = dict[str, Any] From d53b67868b8b5067558d57b9955edd10da202d53 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 May 2025 16:33:43 +0100 Subject: [PATCH 5/6] Only enable multi-phase init on free-threaded builds --- mypyc/codegen/emitmodule.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 9d5f40db6ab3a..7c071ddea23dd 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -39,6 +39,7 @@ ) from mypyc.codegen.literals import Literals from mypyc.common import ( + IS_FREE_THREADED, MODULE_PREFIX, PREFIX, RUNTIME_C_FILES, @@ -514,7 +515,9 @@ def __init__( self.use_shared_lib = group_name is not None self.compiler_options = compiler_options self.multi_file = compiler_options.multi_file - self.multi_phase_init = True + # Multi-phase init is needed to enable free-threading. In the future we'll + # probably want to enable it always, but we'll wait until it's stable. + self.multi_phase_init = IS_FREE_THREADED @property def group_suffix(self) -> str: From b3f09a4ca965d857589e5ccce394f6b2caabdafa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 May 2025 16:39:59 +0100 Subject: [PATCH 6/6] Add comments --- mypyc/codegen/emitmodule.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 7c071ddea23dd..8474be62579df 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -887,10 +887,14 @@ def emit_module_def_slots(self, emitter: Emitter, module_prefix: str) -> None: emitter.emit_line(f"static PyModuleDef_Slot {name}[] = {{") emitter.emit_line(f"{{Py_mod_exec, {exec_name}}},") if sys.version_info >= (3, 12): + # Multiple interpreter support requires not using any C global state, + # which we don't support yet. emitter.emit_line( "{Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}," ) if sys.version_info >= (3, 13): + # Declare support for free-threading to enable experimentation, + # even if we don't properly support it. emitter.emit_line("{Py_mod_gil, Py_MOD_GIL_NOT_USED},") emitter.emit_line("{0, NULL},") emitter.emit_line("};")