feat(DISET): auto-restart service after configurable max throttle duration

When a service is stuck in throttle mode (all threads blocked, queue
full), it cannot recover without external intervention. Add a
configurable MaxThrottleDuration CS option (default: 0 = disabled)
that triggers a process exit after the specified number of seconds of
continuous throttling. The process supervisor (e.g. runsv) then
restarts the service cleanly, clearing all stuck state.

When enabled, a FATAL log message is emitted before exit with full
queue/thread diagnostics for post-mortem analysis.

Author: ryuwd

src/DIRAC/Core/DISET/ServiceReactor.py

@@ -244,10 +244,25 @@ def __acceptIncomingConnection(self, svcName=False):
                     duration = now - throttleStartedAt
                     diag = self.__services[svcName].throttleDiagnostics()
                     gLogger.warn(
-                        f"Service {svcName} still throttling after {duration:.0f}s",
-                        f"queue={diag['queue']}/{diag['maxQueue']}, " f"threads={diag['threads']}/{diag['maxThreads']}",
+                        f"Service {svcName} still throttling",
+                        f"duration={duration:.0f}s, queue={diag['queue']}/{diag['maxQueue']}, "
+                        f"threads={diag['threads']}/{diag['maxThreads']}",
                     )
                     lastThrottleLog = now
+                # Check if throttle has exceeded the maximum allowed duration
+                maxThrottleDuration = self.__services[svcName].getConfig().getMaxThrottleDuration()
+                if maxThrottleDuration > 0 and (now - throttleStartedAt) > maxThrottleDuration:
+                    diag = self.__services[svcName].throttleDiagnostics()
+                    gLogger.fatal(
+                        f"Service {svcName} stuck in throttle, initiating process restart",
+                        f"duration={now - throttleStartedAt:.0f}s (limit: {maxThrottleDuration}s), "
+                        f"queue={diag['queue']}/{diag['maxQueue']}, "
+                        f"threads={diag['threads']}/{diag['maxThreads']}",
+                    )
+                    clientTransport.close()
+                    self.__alive = False
+                    return
+                gLogger.warn("Rejecting client due to throttling", str(clientTransport.getRemoteAddress()))
                 clientTransport.close()
                 time.sleep(THROTTLE_SERVICE_SLEEP_SECONDS)
                 continue

src/DIRAC/Core/DISET/private/ServiceConfiguration.py

@@ -121,6 +121,18 @@ def getURL(self):
         self.setURL(serviceURL)
         return serviceURL
 
+    def getMaxThrottleDuration(self):
+        """Maximum seconds a service can remain in throttle mode before triggering a restart.
+
+        Set to 0 to disable auto-restart (default).
+        When the throttle duration exceeds this value, the service process exits
+        to allow the process supervisor (e.g. runsv) to restart it cleanly.
+        """
+        try:
+            return int(self.getOption("MaxThrottleDuration"))
+        except Exception:
+            return 0
+
     def getContextLifeTime(self):
         optionValue = self.getOption("ContextLifeTime")
         try: