SOLR-18174 AsyncTracker Semaphore permit leak fix (#4236)

Also add metric solr_client_request_async_permits
Make max async requests configurable with sysprop solr.solrj.http.jetty.async_requests.max

Author: janhoy

changelog/unreleased/SOLR-18174-prevent-double-registration.yml

@@ -0,0 +1,8 @@
+title: Fix semaphore permit leaks in HttpJettySolrClient's AsyncTracker. Avoid IO-thread deadlock on connection failure retries. Add a new metric gauge solr.http.client.async_permits
+type: fixed
+authors:
+  - name: Jan Høydahl
+    url: https://home.apache.org/phonebook.html?uid=janhoy
+links:
+  - name: SOLR-18174
+    url: https://issues.apache.org/jira/browse/SOLR-18174

solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java

@@ -19,6 +19,7 @@
 import static org.apache.solr.util.stats.InstrumentedHttpListenerFactory.KNOWN_METRIC_NAME_STRATEGIES;
 
 import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.ObservableLongGauge;
 import java.lang.invoke.MethodHandles;
 import java.util.Iterator;
 import java.util.List;
@@ -85,6 +86,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory
   protected volatile HttpJettySolrClient defaultClient;
   protected InstrumentedHttpListenerFactory httpListenerFactory;
   protected LBAsyncSolrClient loadbalancer;
+  private ObservableLongGauge asyncRequestsGauge;
 
   int corePoolSize = 0;
   int maximumPoolSize = Integer.MAX_VALUE;
@@ -352,6 +354,7 @@ public void close() {
         ExecutorUtil.shutdownAndAwaitTermination(commExecutor);
       }
     }
+    IOUtils.closeQuietly(asyncRequestsGauge);
     try {
       SolrMetricProducer.super.close();
     } catch (Exception e) {
@@ -440,5 +443,20 @@ public void initializeMetrics(SolrMetricsContext parentContext, Attributes attri
     commExecutor =
         solrMetricsContext.instrumentedExecutorService(
             commExecutor, "solr.core.executor", "httpShardExecutor", SolrInfoBean.Category.QUERY);
+    if (defaultClient != null) {
+      asyncRequestsGauge =
+          solrMetricsContext.observableLongGauge(
+              "solr.client.request.async_permits",
+              "Outstanding async HTTP request permits in the Jetty SolrJ client"
+                  + " (state=max: configured ceiling; state=available: currently unused permits).",
+              measurement -> {
+                measurement.record(
+                    defaultClient.asyncTrackerMaxPermits(), Attributes.of(STATE_KEY_ATTR, "max"));
+                measurement.record(
+                    defaultClient.asyncTrackerAvailablePermits(),
+                    Attributes.of(STATE_KEY_ATTR, "available"));
+              },
+              null);
+    }
   }
 }