Fix bug and refactor code

- Fix bug: the lost of errno value in SemLock_impl,
- Reduce shared memory sise, used kern.posix.max.sem value
instaed of SC_SEM_NSEMS_MAX value for array size of shared memeory,
- Add a unit test that raises an error when the maximum opened sems is reached.

Last code reviews are relaized with Claude code

Author: YvesDup

Lib/test/_test_multiprocessing.py

@@ -7545,7 +7545,8 @@ def test_sharedmem_details(self):
                 with memoryview(_mmap[:16]).cast('i') as _buf: # signed int
                     header = _buf[:4].tolist()
                     self.assertEqual(header[0], n)
-                    self.assertEqual(header[-2], size)
+                    self.assertEqual(header[1], _multiprocessing._MACOSX_MAX_OPEN_SEMS)
+                    self.assertEqual(header[2], size)
                     sizeof_counter = header[-1]
 
                 start += _len
@@ -7597,6 +7598,12 @@ def test_sharedmem_details(self):
         while len(sems) and (s := sems.pop()):
             del s
 
+    def test_sharedmem_max_open_sems(self):
+        # This test should be run alone.
+        with self.assertRaisesRegex(OSError, "No space left on device"):
+            sems = []
+            for i in range(1, _multiprocessing._MACOSX_MAX_OPEN_SEMS + 1):
+                sems.append(self.Semaphore(i) if i % 2 else self.BoundedSemaphore(i))
 
 #
 # Mixins

Modules/_multiprocessing/multiprocessing.c

@@ -235,7 +235,7 @@ multiprocessing_exec(PyObject *module)
     Py_DECREF(py_sem_value_max);
 
 #ifdef HAVE_BROKEN_SEM_GETVALUE
-    if (_PyMp_init_shm_names(module) < 0)
+    if (_PyMp_init_module_constants(module) < 0)
         return -1;
 #endif
 

Modules/_multiprocessing/multiprocessing.h

@@ -101,7 +101,7 @@ PyObject *_PyMp_SetError(PyObject *Type, int num);
 extern PyType_Spec _PyMp_SemLockType_spec;
 extern PyObject *_PyMp_sem_unlink(const char *name);
 #ifdef HAVE_BROKEN_SEM_GETVALUE
-    extern int _PyMp_init_shm_names(PyObject *module);
+    extern int _PyMp_init_module_constants(PyObject *module);
     extern PyObject *_multiprocessing_set_shm_names(PyObject *module, PyObject *args);
 #endif
 

Modules/_multiprocessing/semaphore.c

@@ -390,7 +390,8 @@ shm_semlock_counters = {
     .shmlock_name = {0},
     .handle_shmlock = (SEM_HANDLE)0,
     .header = (HeaderObject *)NULL,
-    .counters = (CounterObject *)NULL
+    .counters = (CounterObject *)NULL,
+    .max_open_sems = 0
 };
 
 /*
@@ -435,12 +436,34 @@ _init_shm_names(pid_t pid)
                   "%s%ld", SHMLOCK_NAME, (long)pid);
 }
 
+static void
+_init_sem_max_from_kern_posix(int value)
+{
+    shm_semlock_counters.max_open_sems = value/2;
+}
+
 /*
 Public entry point called from multiprocessing_exec().
-Builds names from getpid() and exposes them as module attributes.
+Builds names from getpid(), get 'kern.posix.sem.max' value and exposes them as module attributes.
 */
+
 int
-_PyMp_init_shm_names(PyObject *module)
+_get_sem_max_from_kern_posix(void)
+{
+    int value;
+    size_t len = sizeof(value);
+
+    if (sysctlbyname("kern.posix.sem.max", &value, &len, NULL, 0) < 0) {
+        value = sysconf(_SC_SEM_NSEMS_MAX);
+        if (value < 0) {
+            return 5000;
+        }
+    }
+    return value;
+}
+
+int
+_PyMp_init_module_constants(PyObject *module)
 {
     _init_shm_names(getpid());
     if (PyModule_AddStringConstant(module, "_MACOSX_SHAREDMEM_NAME",
@@ -449,6 +472,14 @@ _PyMp_init_shm_names(PyObject *module)
     if (PyModule_AddStringConstant(module, "_MACOSX_SHMLOCK_NAME",
                                    shm_semlock_counters.shmlock_name) < 0)
         return -1;
+
+    // A mutex is associated to each semaphore, so we could only open
+    // value//2 MacOSX semaphores.
+    int value = _get_sem_max_from_kern_posix();
+    _init_sem_max_from_kern_posix(value);
+    if (PyModule_AddIntConstant(module, "_MACOSX_MAX_OPEN_SEMS",
+                                   value/2) < 0)
+        return -1;
     return 0;
 }
 
@@ -716,7 +747,7 @@ create_shm_semlock_counters(const char *from_sem_name) {
         header = shm_semlock_counters.header;
 
         header->size_shm = size_shm;
-        header->n_slots = CALC_NB_SLOTS(size_shm - sizeof(HeaderObject));
+        header->n_slots = shm_semlock_counters.max_open_sems;
         header->n_semlocks = 0;
         header->sizeof_counter = sizeof(CounterObject);
         res = 0;
@@ -799,7 +830,7 @@ _search_counter_from_sem_name(const char *sem_name)
     HeaderObject *header = shm_semlock_counters.header;
     CounterObject *counter = shm_semlock_counters.counters;
 
-    for(int i = 0, j = 0; i < header->n_slots && j < header->n_semlocks; i++, counter++) {
+    for(Py_ssize_t i = 0, j = 0; i < header->n_slots && j < header->n_semlocks; i++, counter++) {
         if (counter->sem_name[0] != 0) {
             if (!PyOS_stricmp(counter->sem_name, sem_name)) {
                 return counter;
@@ -819,12 +850,11 @@ _search_counter_free_slot(void)
     HeaderObject *header = shm_semlock_counters.header;
     CounterObject *counter = shm_semlock_counters.counters;
 
-    for(int i = 0; i < header->n_slots; i++, counter++) {
+    for(Py_ssize_t i = 0; i < header->n_slots; i++, counter++) {
         if(counter->sem_name[0] == 0) {
             return counter;
         }
     }
-
     /*
     Not enough memory: see NSEMS_MAX in semaphore_macosx.h.
     */
@@ -1175,6 +1205,7 @@ _multiprocessing_SemLock_impl(PyTypeObject *type, int kind, int value,
     SEM_HANDLE handle = SEM_FAILED;
     PyObject *result;
     char *name_copy = NULL;
+    int err = 0;
 #ifdef HAVE_BROKEN_SEM_GETVALUE
     char mutex_name[SIZE_MUTEX_NAME];
     SEM_HANDLE handle_mutex = SEM_FAILED;
@@ -1252,7 +1283,10 @@ _multiprocessing_SemLock_impl(PyTypeObject *type, int kind, int value,
     failure:
 #ifdef HAVE_BROKEN_SEM_GETVALUE
     if (ISSEMAPHORE_FROM_ARGS(maxvalue, kind)) {
+        // Save errno before releasing the lock, as sem_post can change it
+        err = errno;
         RELEASE_SHMLOCK;
+        errno = err;
     }
     failure_shmlock:
 #endif /* HAVE_BROKEN_SEM_GETVALUE*/

Modules/_multiprocessing/semaphore_macosx.h

@@ -2,6 +2,7 @@
 #define SEMAPHORE_MACOSX_H
 
 #include <unistd.h>     // sysconf(SC_PAGESIZE)
+#include <sys/sysctl.h>  // sysctlbyname
 
 /*
 Structures and constants in shared memory
@@ -33,13 +34,10 @@ typedef struct {
 Structure, constants and macros of static memory:
 */
 
-#define NSEMS_MAX               sysconf(_SC_SEM_NSEMS_MAX) // returns 87381 on MacOSX 15.1 and m4 pro.
-#define CALC_SIZE_SHM           (NSEMS_MAX * sizeof(CounterObject)) + sizeof(HeaderObject)
 #define SC_PAGESIZE             sysconf(_SC_PAGESIZE)
+#define CALC_SIZE_SHM           (shm_semlock_counters.max_open_sems * sizeof(CounterObject)) + sizeof(HeaderObject)
 #define ALIGN_SHM_PAGE(n)       ((int)((n)/SC_PAGESIZE)+1)*SC_PAGESIZE
 
-#define CALC_NB_SLOTS(n)        (int)((((n)) - sizeof(HeaderObject)) / sizeof(CounterObject))
-
 #define SHAREDMEM_NAME  "/psm-gh125828-"
 #define SHMLOCK_NAME    "/mp-gh125828-"
 
@@ -54,6 +52,7 @@ struct _CountersWorkaround{
     MEMORY_HANDLE handle_shm; // Memory handle.
     char shmlock_name[SIZE_SHMLOCK_NAME];
     SEM_HANDLE handle_shmlock; // Global memory lock to handle shared memory.
+    int max_open_sems;              // Max number of opened semaphores.
     /*-- Pointers to shared memory --*/
     HeaderObject *header;     // Pointer to header (shared memory).
     CounterObject*counters;   // Pointer to the first item of fixed array (shared memory).