update

Author: jaredmauch

bin/mailmanctl

@@ -127,6 +127,11 @@ LOCK_LIFETIME = mm_cfg.days(1) + mm_cfg.hours(6)
 SNOOZE = mm_cfg.days(1)
 MAX_RESTARTS = 10
 
+# Global circuit breaker settings
+GLOBAL_RESTART_WINDOW = 30  # 30 seconds window
+GLOBAL_MAX_RESTARTS = 100   # Maximum restarts in window
+_global_restart_times = []  # List of timestamps for restarts
+
 LogStdErr('error', 'mailmanctl', manual_reprime=0)
 
 
@@ -463,6 +468,29 @@ def check_status():
     return all_running
 
 
+def check_global_circuit_breaker():
+    """Check if we've exceeded the global restart limit.
+    
+    Returns:
+        bool: True if we should stop all runners, False otherwise
+    """
+    global _global_restart_times
+    now = time.time()
+    
+    # Remove old timestamps outside the window
+    _global_restart_times = [t for t in _global_restart_times if now - t < GLOBAL_RESTART_WINDOW]
+    
+    # Add current timestamp
+    _global_restart_times.append(now)
+    
+    # Check if we've exceeded the limit
+    if len(_global_restart_times) > GLOBAL_MAX_RESTARTS:
+        syslog('error', 'Global circuit breaker triggered: %d restarts in %d seconds',
+               len(_global_restart_times), GLOBAL_RESTART_WINDOW)
+        return True
+    return False
+
+
 def main():
     try:
         args = parse_args()
@@ -661,7 +689,19 @@ Master qrunner detected abnormal subprocess exit
                                pid, killsig, exitstatus, qrname,
                                slice+1, count, restarting)
 
-                        # See if we've reached the maximum number of allowable restarts
+                        # Check global circuit breaker before restarting
+                        if restarting and check_global_circuit_breaker():
+                            syslog('error', 'Global circuit breaker triggered - stopping all runners')
+                            # Stop all runners
+                            for pid in list(kids.keys()):
+                                try:
+                                    os.kill(pid, signal.SIGTERM)
+                                except OSError as e:
+                                    if e.errno != errno.ESRCH:
+                                        raise
+                            # Exit the main loop
+                            break
+                        
                         if exitstatus != signal.SIGINT:
                             restarts += 1
                         if restarts > MAX_RESTARTS: