Bound HTTP client timeouts and reclaim pooled connections

Fixes connection-pool exhaustion: a stalled read on the end-user
backend route was held indefinitely because the pooled HTTP clients
had no socket/read timeout (Apache default SO_TIMEOUT = 0 = infinite).
The worker thread blocked inside the read and never reached
Response.close(), so the leased connection was never returned. With
connectionRequestTimeout also unset, new requests blocked forever
waiting for a lease instead of failing fast, wedging the listener.

Application.getClient()/getNoCertClient():
- set socket (read) and connect timeouts on the RequestConfig
- always set connectionRequestTimeout (default 30s) so lease waits
  fail fast instead of blocking forever
- give pooled connections a TTL and revalidate after inactivity so
  stuck/stale connections are reclaimed
- timeouts exposed as named constants for tuning

SignUp: close the PublicKey/Agent/Authorization client Responses on
all paths (defense-in-depth; these leaked on the admin signup path).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: namedgraph

src/main/java/com/atomgraph/linkeddatahub/Application.java

@@ -1505,9 +1505,24 @@ public void submitImport(RDFImport rdfImport, com.atomgraph.linkeddatahub.apps.m
         new ImportExecutor(importThreadPool).start(service, adminService, this, baseURI, gsc, rdfImport);
     }
 
+    /** Default socket (read) timeout in milliseconds for pooled HTTP client connections. Bounds stalled reads so a hung connection cannot be held indefinitely. */
+    public static final int CLIENT_SOCKET_TIMEOUT = 120000;
+
+    /** Default connection (connect) timeout in milliseconds for pooled HTTP client connections. */
+    public static final int CLIENT_CONNECT_TIMEOUT = 10000;
+
+    /** Default timeout in milliseconds to wait for a connection lease from the pool before failing fast. Used when no explicit connectionRequestTimeout is configured. */
+    public static final int CLIENT_CONNECTION_REQUEST_TIMEOUT = 30000;
+
+    /** Time-to-live in milliseconds after which pooled connections are discarded rather than reused. */
+    public static final long CLIENT_CONNECTION_TIME_TO_LIVE = 300000;
+
+    /** Period of inactivity in milliseconds after which a pooled connection is revalidated before reuse. */
+    public static final int CLIENT_VALIDATE_AFTER_INACTIVITY = 10000;
+
     /**
      * Builds JAX-RS client instance from given configuration.
-     * 
+     *
      * @param keyStore keystore
      * @param keyStorePassword keystore password
      * @param trustStore truststore
@@ -1543,7 +1558,7 @@ public static Client getClient(KeyStore keyStore, String keyStorePassword, KeySt
             register("http", new PlainConnectionSocketFactory()).
             build();
 
-        PoolingHttpClientConnectionManager conman = new PoolingHttpClientConnectionManager(socketFactoryRegistry)
+        PoolingHttpClientConnectionManager conman = new PoolingHttpClientConnectionManager(socketFactoryRegistry, null, null, null, CLIENT_CONNECTION_TIME_TO_LIVE, TimeUnit.MILLISECONDS)
         {
 
             // https://github.com/eclipse-ee4j/jersey/issues/4449
@@ -1574,7 +1589,8 @@ public void releaseConnection(final HttpClientConnection managedConn, final Obje
         };
         if (maxConnPerRoute != null) conman.setDefaultMaxPerRoute(maxConnPerRoute);
         if (maxTotalConn != null) conman.setMaxTotal(maxTotalConn);
-        
+        conman.setValidateAfterInactivity(CLIENT_VALIDATE_AFTER_INACTIVITY);
+
         ClientConfig config = new ClientConfig();
         config.connectorProvider(new ApacheConnectorProvider());
         config.register(MultiPartFeature.class);
@@ -1586,10 +1602,11 @@ public void releaseConnection(final HttpClientConnection managedConn, final Obje
         config.property(ClientProperties.FOLLOW_REDIRECTS, true);
         config.property(ClientProperties.REQUEST_ENTITY_PROCESSING, RequestEntityProcessing.BUFFERED); // https://stackoverflow.com/questions/42139436/jersey-client-throws-cannot-retry-request-with-a-non-repeatable-request-entity
         config.property(ApacheClientProperties.CONNECTION_MANAGER, conman);
-        if (connectionRequestTimeout != null)
-            config.property(ApacheClientProperties.REQUEST_CONFIG, RequestConfig.custom().
-                setConnectionRequestTimeout(connectionRequestTimeout).
-                build());
+        config.property(ApacheClientProperties.REQUEST_CONFIG, RequestConfig.custom().
+            setConnectionRequestTimeout(connectionRequestTimeout != null ? connectionRequestTimeout : CLIENT_CONNECTION_REQUEST_TIMEOUT).
+            setConnectTimeout(CLIENT_CONNECT_TIMEOUT).
+            setSocketTimeout(CLIENT_SOCKET_TIMEOUT).
+            build());
 
         if (maxRequestRetries != null)
             config.property(ApacheClientProperties.RETRY_HANDLER, (HttpRequestRetryHandler) (IOException ex, int executionCount, HttpContext context) ->
@@ -1643,11 +1660,11 @@ public static Client getNoCertClient(KeyStore trustStore, Integer maxConnPerRout
                 register("http", new PlainConnectionSocketFactory()).
                 build();
 
-            PoolingHttpClientConnectionManager conman = new PoolingHttpClientConnectionManager(socketFactoryRegistry)
+            PoolingHttpClientConnectionManager conman = new PoolingHttpClientConnectionManager(socketFactoryRegistry, null, null, null, CLIENT_CONNECTION_TIME_TO_LIVE, TimeUnit.MILLISECONDS)
             {
 
                 // https://github.com/eclipse-ee4j/jersey/issues/4449
-                
+
                 @Override
                 public void close()
                 {
@@ -1674,6 +1691,7 @@ public void releaseConnection(final HttpClientConnection managedConn, final Obje
             };
             if (maxConnPerRoute != null) conman.setDefaultMaxPerRoute(maxConnPerRoute);
             if (maxTotalConn != null) conman.setMaxTotal(maxTotalConn);
+            conman.setValidateAfterInactivity(CLIENT_VALIDATE_AFTER_INACTIVITY);
 
             ClientConfig config = new ClientConfig();
             config.connectorProvider(new ApacheConnectorProvider());
@@ -1686,10 +1704,11 @@ public void releaseConnection(final HttpClientConnection managedConn, final Obje
             config.property(ClientProperties.FOLLOW_REDIRECTS, true);
             config.property(ClientProperties.REQUEST_ENTITY_PROCESSING, RequestEntityProcessing.BUFFERED); // https://stackoverflow.com/questions/42139436/jersey-client-throws-cannot-retry-request-with-a-non-repeatable-request-entity
             config.property(ApacheClientProperties.CONNECTION_MANAGER, conman);
-            if (connectionRequestTimeout != null)
-                config.property(ApacheClientProperties.REQUEST_CONFIG, RequestConfig.custom().
-                    setConnectionRequestTimeout(connectionRequestTimeout).
-                    build());
+            config.property(ApacheClientProperties.REQUEST_CONFIG, RequestConfig.custom().
+                setConnectionRequestTimeout(connectionRequestTimeout != null ? connectionRequestTimeout : CLIENT_CONNECTION_REQUEST_TIMEOUT).
+                setConnectTimeout(CLIENT_CONNECT_TIMEOUT).
+                setSocketTimeout(CLIENT_SOCKET_TIMEOUT).
+                build());
 
             if (maxRequestRetries != null)
                 config.property(ApacheClientProperties.RETRY_HANDLER, (HttpRequestRetryHandler) (IOException ex, int executionCount, HttpContext context) ->

src/main/java/com/atomgraph/linkeddatahub/resource/admin/SignUp.java

@@ -242,67 +242,80 @@ public Response post(Model agentModel)
                     certPublicKey);
                 new Skolemizer(publicKeyGraphUri.toString()).apply(publicKeyModel);
 
-                Response publicKeyResponse = super.put(publicKeyModel, false, publicKeyGraphUri);
-                if (publicKeyResponse.getStatus() != Response.Status.CREATED.getStatusCode())
+                try (Response publicKeyResponse = super.put(publicKeyModel, false, publicKeyGraphUri))
                 {
-                    if (log.isErrorEnabled()) log.error("Cannot create PublicKey");
-                    throw new InternalServerErrorException("Cannot create PublicKey");
+                    if (publicKeyResponse.getStatus() != Response.Status.CREATED.getStatusCode())
+                    {
+                        if (log.isErrorEnabled()) log.error("Cannot create PublicKey");
+                        throw new InternalServerErrorException("Cannot create PublicKey");
+                    }
                 }
 
                 Resource publicKey = publicKeyModel.createResource(publicKeyGraphUri.toString()).getPropertyResourceValue(FOAF.primaryTopic);
                 agent.addProperty(Cert.key, publicKey); // add public key
                 agentModel.add(agentModel.createResource(getSystem().getSecretaryWebIDURI().toString()), ACL.delegates, agent); // make secretary delegate whis agent
 
                 Response agentResponse = super.put(agentModel, false, agentGraphUri);
-                if (agentResponse.getStatus() != Response.Status.CREATED.getStatusCode())
+                boolean returnAgentResponse = false;
+                try
                 {
-                    if (log.isErrorEnabled()) log.error("Cannot create Agent");
-                    throw new InternalServerErrorException("Cannot create Agent");
-                }
+                    if (agentResponse.getStatus() != Response.Status.CREATED.getStatusCode())
+                    {
+                        if (log.isErrorEnabled()) log.error("Cannot create Agent");
+                        throw new InternalServerErrorException("Cannot create Agent");
+                    }
 
-                URI authGraphUri = getUriInfo().getBaseUriBuilder().path(AUTHORIZATION_PATH).path("{slug}/").build(UUID.randomUUID().toString());
-                Model authModel = ModelFactory.createDefaultModel();
-                // creating authorizations for the Agent and PublicKey documents
-                createAuthorization(authModel,
-                    authGraphUri,
-                    authModel.createResource(getUriInfo().getBaseUri().resolve(AUTHORIZATION_PATH).toString()),
-                    agentGraphUri,
-                    publicKeyGraphUri);
-                new Skolemizer(authGraphUri.toString()).apply(authModel);
+                    URI authGraphUri = getUriInfo().getBaseUriBuilder().path(AUTHORIZATION_PATH).path("{slug}/").build(UUID.randomUUID().toString());
+                    Model authModel = ModelFactory.createDefaultModel();
+                    // creating authorizations for the Agent and PublicKey documents
+                    createAuthorization(authModel,
+                        authGraphUri,
+                        authModel.createResource(getUriInfo().getBaseUri().resolve(AUTHORIZATION_PATH).toString()),
+                        agentGraphUri,
+                        publicKeyGraphUri);
+                    new Skolemizer(authGraphUri.toString()).apply(authModel);
 
-                Response authResponse = super.put(authModel, false, authGraphUri);
-                if (authResponse.getStatus() != Response.Status.CREATED.getStatusCode())
-                {
-                    if (log.isErrorEnabled()) log.error("Cannot create Authorization");
-                    throw new InternalServerErrorException("Cannot create Authorization");
-                }
+                    try (Response authResponse = super.put(authModel, false, authGraphUri))
+                    {
+                        if (authResponse.getStatus() != Response.Status.CREATED.getStatusCode())
+                        {
+                            if (log.isErrorEnabled()) log.error("Cannot create Authorization");
+                            throw new InternalServerErrorException("Cannot create Authorization");
+                        }
+                    }
 
-                // purge agent lookup from proxy cache
-                URI agentServiceBackendProxy = getSystem().getServiceContext(getAgentService()).getBackendProxy();
-                if (agentServiceBackendProxy != null)
-                {
-                    try (Response response = ban(agentServiceBackendProxy, mbox.getURI()))
+                    // purge agent lookup from proxy cache
+                    URI agentServiceBackendProxy = getSystem().getServiceContext(getAgentService()).getBackendProxy();
+                    if (agentServiceBackendProxy != null)
                     {
-                        // Response automatically closed by try-with-resources
+                        try (Response response = ban(agentServiceBackendProxy, mbox.getURI()))
+                        {
+                            // Response automatically closed by try-with-resources
+                        }
                     }
-                }
 
-                // remove secretary WebID from cache
-                getSystem().getEventBus().post(new com.atomgraph.linkeddatahub.server.event.SignUp(getSystem().getSecretaryWebIDURI()));
+                    // remove secretary WebID from cache
+                    getSystem().getEventBus().post(new com.atomgraph.linkeddatahub.server.event.SignUp(getSystem().getSecretaryWebIDURI()));
 
-                if (download)
-                {
-                    return Response.ok(keyStoreBytes).
-                        type(PKCS12_MEDIA_TYPE).
-                        header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=cert.p12").
-                        build();
+                    if (download)
+                    {
+                        return Response.ok(keyStoreBytes).
+                            type(PKCS12_MEDIA_TYPE).
+                            header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=cert.p12").
+                            build();
+                    }
+                    else
+                    {
+                        LocalDate certExpires = LocalDate.now().plusDays(getValidityDays()); // ((X509Certificate)cert).getNotAfter();
+                        sendEmail(agent, certExpires, keyStoreBytes, keyStoreFileName);
+
+                        returnAgentResponse = true;
+                        return agentResponse; // 201 Created - ownership passes to the JAX-RS runtime
+                    }
                 }
-                else
+                finally
                 {
-                    LocalDate certExpires = LocalDate.now().plusDays(getValidityDays()); // ((X509Certificate)cert).getNotAfter(); 
-                    sendEmail(agent, certExpires, keyStoreBytes, keyStoreFileName);
-
-                    return agentResponse; // 201 Created
+                    if (!returnAgentResponse) agentResponse.close();
                 }
             }
         }