ZOOKEEPER-4952:Reduce the GC overhead of Prometheus reject exception handling (#2279)

Author: Li Wang <liwang@apple.com>
Reviewers: kezhuw

Author: li4wang

zookeeper-metrics-providers/zookeeper-prometheus-metrics/src/main/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProvider.java

@@ -33,7 +33,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.RejectedExecutionHandler;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -677,7 +677,9 @@ private Optional<ExecutorService> createExecutor() {
                 numWorkerThreads,
                 0L,
                 TimeUnit.MILLISECONDS,
-                queue, new PrometheusWorkerThreadFactory());
+                queue,
+                new PrometheusWorkerThreadFactory(),
+                new PrometheusRejectedExecutionHandler());
         LOG.info("Executor service was created with numWorkerThreads {} and maxQueueSize {}",
                 numWorkerThreads,
                 maxQueueSize);
@@ -714,14 +716,18 @@ public Thread newThread(final Runnable runnable) {
         }
     }
 
+    private class PrometheusRejectedExecutionHandler implements RejectedExecutionHandler {
+        private final String queueExceededMessage = "Prometheus metrics queue size exceeded the max " + maxQueueSize;
+
+        @Override
+        public void rejectedExecution(final Runnable r, final ThreadPoolExecutor e) {
+            rateLogger.rateLimitLog(queueExceededMessage);
+        }
+    }
+
     private void reportMetrics(final Runnable task) {
         if (executorOptional.isPresent()) {
-            try {
-                executorOptional.get().submit(task);
-            } catch (final RejectedExecutionException e) {
-                rateLogger.rateLimitLog("Prometheus metrics reporting task queue size exceeded the max",
-                        String.valueOf(maxQueueSize));
-            }
+            executorOptional.get().submit(task);
         } else {
             task.run();
         }

zookeeper-server/src/main/java/org/apache/zookeeper/server/RateLogger.java

@@ -46,15 +46,20 @@ public RateLogger(Logger log, long interval) {
 
     public void flush() {
         if (msg != null && count > 0) {
-            String log = "";
+            // Optimized logging using parameterized messages to reduce string concatenation overhead
             if (count > 1) {
-                log = "[" + count + " times] ";
+                if (value != null) {
+                    LOG.warn("[{} times] Message: {} Last value:{}", count, msg, value);
+                } else {
+                    LOG.warn("[{} times] Message: {}", count, msg);
+                }
+            } else {
+                if (value != null) {
+                    LOG.warn("Message: {} Value:{}", msg, value);
+                } else {
+                    LOG.warn("Message: {}", msg);
+                }
             }
-            log += "Message: " + msg;
-            if (value != null) {
-                log += " Last value:" + value;
-            }
-            LOG.warn(log);
         }
         msg = null;
         value = null;
@@ -70,7 +75,8 @@ public void rateLimitLog(String newMsg) {
      */
     public void rateLimitLog(String newMsg, String newValue) {
         long now = Time.currentElapsedTime();
-        if (Objects.equals(newMsg, msg)) {
+        // Optimized string comparison: use reference equality first for constant strings
+        if (newMsg == msg || Objects.equals(newMsg, msg)) {
             ++count;
             value = newValue;
             if (now - timestamp >= LOG_INTERVAL) {