Bound HTTP client timeouts and reclaim pooled connections

Fixes connection-pool exhaustion: a stalled read on the end-user
backend route was held indefinitely because the pooled HTTP clients
had no socket/read timeout (Apache default SO_TIMEOUT = 0 = infinite).
The worker thread blocked inside the read and never reached
Response.close(), so the leased connection was never returned. With
connectionRequestTimeout also unset, new requests blocked forever
waiting for a lease instead of failing fast, wedging the listener.

Adds socket timeout, connect timeout, connection TTL and
validate-after-inactivity to the pooled HTTP clients, applied only
when configured. Values are supplied via env vars through the existing
CATALINA_OPTS -> system property -> JAX-RS app constructor convention
(as connectionRequestTimeout already is), with image defaults in the
Dockerfile -- no hardcoded values in Java:

  CLIENT_SOCKET_TIMEOUT             -> ...linkeddatahub.socketTimeout
  CLIENT_CONNECT_TIMEOUT            -> ...linkeddatahub.connectTimeout
  CLIENT_CONNECTION_TIME_TO_LIVE    -> ...linkeddatahub.connectionTimeToLive
  CLIENT_VALIDATE_AFTER_INACTIVITY  -> ...linkeddatahub.validateAfterInactivity

SignUp: close the PublicKey/Agent/Authorization client Responses on
all paths (defense-in-depth; these leaked on the admin signup path).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: namedgraph

Dockerfile

@@ -111,6 +111,14 @@ ENV MAX_REQUEST_RETRIES=3
 
 ENV CONNECTION_REQUEST_TIMEOUT=30000
 
+ENV CLIENT_SOCKET_TIMEOUT=120000
+
+ENV CLIENT_CONNECT_TIMEOUT=10000
+
+ENV CLIENT_CONNECTION_TIME_TO_LIVE=300000
+
+ENV CLIENT_VALIDATE_AFTER_INACTIVITY=10000
+
 ENV IMPORT_KEEPALIVE=
 
 ENV MAX_IMPORT_THREADS=10

platform/entrypoint.sh

@@ -1080,6 +1080,22 @@ if [ -n "$CONNECTION_REQUEST_TIMEOUT" ]; then
     export CATALINA_OPTS="$CATALINA_OPTS -Dcom.atomgraph.linkeddatahub.connectionRequestTimeout=$CONNECTION_REQUEST_TIMEOUT"
 fi
 
+if [ -n "$CLIENT_SOCKET_TIMEOUT" ]; then
+    export CATALINA_OPTS="$CATALINA_OPTS -Dcom.atomgraph.linkeddatahub.socketTimeout=$CLIENT_SOCKET_TIMEOUT"
+fi
+
+if [ -n "$CLIENT_CONNECT_TIMEOUT" ]; then
+    export CATALINA_OPTS="$CATALINA_OPTS -Dcom.atomgraph.linkeddatahub.connectTimeout=$CLIENT_CONNECT_TIMEOUT"
+fi
+
+if [ -n "$CLIENT_CONNECTION_TIME_TO_LIVE" ]; then
+    export CATALINA_OPTS="$CATALINA_OPTS -Dcom.atomgraph.linkeddatahub.connectionTimeToLive=$CLIENT_CONNECTION_TIME_TO_LIVE"
+fi
+
+if [ -n "$CLIENT_VALIDATE_AFTER_INACTIVITY" ]; then
+    export CATALINA_OPTS="$CATALINA_OPTS -Dcom.atomgraph.linkeddatahub.validateAfterInactivity=$CLIENT_VALIDATE_AFTER_INACTIVITY"
+fi
+
 if [ -n "$MAX_CONTENT_LENGTH" ]; then
     MAX_CONTENT_LENGTH_PARAM="--stringparam ldhc:maxContentLength '$MAX_CONTENT_LENGTH' "
 fi

src/main/java/com/atomgraph/linkeddatahub/Application.java

@@ -358,6 +358,10 @@ public Application(@Context ServletConfig servletConfig) throws URISyntaxExcepti
             servletConfig.getServletContext().getInitParameter(LDHC.maxRequestRetries.getURI()) != null ? Integer.valueOf(servletConfig.getServletContext().getInitParameter(LDHC.maxRequestRetries.getURI())) : null,
             System.getProperty("com.atomgraph.linkeddatahub.connectionRequestTimeout") != null ? Integer.valueOf(System.getProperty("com.atomgraph.linkeddatahub.connectionRequestTimeout")) :
             servletConfig.getServletContext().getInitParameter(LDHC.connectionRequestTimeout.getURI()) != null ? Integer.valueOf(servletConfig.getServletContext().getInitParameter(LDHC.connectionRequestTimeout.getURI())) : null,
+            System.getProperty("com.atomgraph.linkeddatahub.socketTimeout") != null ? Integer.valueOf(System.getProperty("com.atomgraph.linkeddatahub.socketTimeout")) : null,
+            System.getProperty("com.atomgraph.linkeddatahub.connectTimeout") != null ? Integer.valueOf(System.getProperty("com.atomgraph.linkeddatahub.connectTimeout")) : null,
+            System.getProperty("com.atomgraph.linkeddatahub.connectionTimeToLive") != null ? Long.valueOf(System.getProperty("com.atomgraph.linkeddatahub.connectionTimeToLive")) : null,
+            System.getProperty("com.atomgraph.linkeddatahub.validateAfterInactivity") != null ? Integer.valueOf(System.getProperty("com.atomgraph.linkeddatahub.validateAfterInactivity")) : null,
             servletConfig.getServletContext().getInitParameter(LDHC.maxImportThreads.getURI()) != null ? Integer.valueOf(servletConfig.getServletContext().getInitParameter(LDHC.maxImportThreads.getURI())) : null,
             servletConfig.getServletContext().getInitParameter(LDHC.notificationAddress.getURI()) != null ? servletConfig.getServletContext().getInitParameter(LDHC.notificationAddress.getURI()) : null,
             servletConfig.getServletContext().getInitParameter(LDHC.supportedLanguages.getURI()) != null ? servletConfig.getServletContext().getInitParameter(LDHC.supportedLanguages.getURI()) : null,
@@ -445,7 +449,8 @@ public Application(final ServletConfig servletConfig, final MediaTypes mediaType
             final String baseURIString, final String proxyScheme, final String proxyHostname, final Integer proxyPort,
             final String uploadRootString, final boolean invalidateCache,
             final Integer cookieMaxAge, final boolean enableLinkedDataProxy, final boolean allowInternalUrls, final Integer maxContentLength,
-            final Integer maxConnPerRoute, final Integer maxTotalConn, final Integer maxRequestRetries, final Integer connectionRequestTimeout, final Integer maxImportThreads,
+            final Integer maxConnPerRoute, final Integer maxTotalConn, final Integer maxRequestRetries, final Integer connectionRequestTimeout,
+            final Integer socketTimeout, final Integer connectTimeout, final Long connectionTimeToLive, final Integer validateAfterInactivity, final Integer maxImportThreads,
             final String notificationAddressString, final String supportedLanguageCodes, final boolean enableWebIDSignUp, final String oidcRefreshTokensPropertiesPath,
             final String frontendProxyString, final String backendProxyAdminString, final String backendProxyEndUserString,
             final String mailUser, final String mailPassword, final String smtpHost, final String smtpPort,
@@ -704,10 +709,10 @@ public Application(final ServletConfig servletConfig, final MediaTypes mediaType
                 trustStore.load(trustStoreInputStream, clientTrustStorePassword.toCharArray());
             }
 
-            client = getClient(keyStore, clientKeyStorePassword, trustStore, maxConnPerRoute, maxTotalConn, null, false, connectionRequestTimeout);
-            externalClient = getClient(keyStore, clientKeyStorePassword, trustStore, maxConnPerRoute, maxTotalConn, null, false, connectionRequestTimeout);
-            importClient = getClient(keyStore, clientKeyStorePassword, trustStore, maxConnPerRoute, maxTotalConn, maxRequestRetries, true, connectionRequestTimeout);
-            noCertClient = getNoCertClient(trustStore, maxConnPerRoute, maxTotalConn, maxRequestRetries, connectionRequestTimeout);
+            client = getClient(keyStore, clientKeyStorePassword, trustStore, maxConnPerRoute, maxTotalConn, null, false, connectionRequestTimeout, socketTimeout, connectTimeout, connectionTimeToLive, validateAfterInactivity);
+            externalClient = getClient(keyStore, clientKeyStorePassword, trustStore, maxConnPerRoute, maxTotalConn, null, false, connectionRequestTimeout, socketTimeout, connectTimeout, connectionTimeToLive, validateAfterInactivity);
+            importClient = getClient(keyStore, clientKeyStorePassword, trustStore, maxConnPerRoute, maxTotalConn, maxRequestRetries, true, connectionRequestTimeout, socketTimeout, connectTimeout, connectionTimeToLive, validateAfterInactivity);
+            noCertClient = getNoCertClient(trustStore, maxConnPerRoute, maxTotalConn, maxRequestRetries, connectionRequestTimeout, socketTimeout, connectTimeout, connectionTimeToLive, validateAfterInactivity);
 
             if (maxContentLength != null)
             {
@@ -1507,21 +1512,26 @@ public void submitImport(RDFImport rdfImport, com.atomgraph.linkeddatahub.apps.m
 
     /**
      * Builds JAX-RS client instance from given configuration.
-     * 
+     *
      * @param keyStore keystore
      * @param keyStorePassword keystore password
      * @param trustStore truststore
      * @param maxConnPerRoute max connections per route
      * @param maxTotalConn max total connections
      * @param maxRequestRetries maximum number of times that the HTTP client will retry a request
      * @param buffered true if request entity should be buffered
+     * @param connectionRequestTimeout timeout in milliseconds to wait for a connection lease from the pool (null to leave unset)
+     * @param socketTimeout socket (read) timeout in milliseconds (null to leave unset)
+     * @param connectTimeout connection (connect) timeout in milliseconds (null to leave unset)
+     * @param connectionTimeToLive time-to-live in milliseconds after which pooled connections are discarded (null to leave unset)
+     * @param validateAfterInactivity period of inactivity in milliseconds after which a pooled connection is revalidated before reuse (null to leave unset)
      * @return client instance
      * @throws NoSuchAlgorithmException SSL algorithm error
      * @throws KeyStoreException keystore loading error
      * @throws UnrecoverableKeyException key loading error
      * @throws KeyManagementException key loading error
      */
-    public static Client getClient(KeyStore keyStore, String keyStorePassword, KeyStore trustStore, Integer maxConnPerRoute, Integer maxTotalConn, Integer maxRequestRetries, boolean buffered, Integer connectionRequestTimeout) throws NoSuchAlgorithmException, KeyStoreException, UnrecoverableKeyException, KeyManagementException
+    public static Client getClient(KeyStore keyStore, String keyStorePassword, KeyStore trustStore, Integer maxConnPerRoute, Integer maxTotalConn, Integer maxRequestRetries, boolean buffered, Integer connectionRequestTimeout, Integer socketTimeout, Integer connectTimeout, Long connectionTimeToLive, Integer validateAfterInactivity) throws NoSuchAlgorithmException, KeyStoreException, UnrecoverableKeyException, KeyManagementException
     {
         if (keyStore == null) throw new IllegalArgumentException("KeyStore cannot be null");
         if (keyStorePassword == null) throw new IllegalArgumentException("KeyStore password string cannot be null");
@@ -1543,7 +1553,7 @@ public static Client getClient(KeyStore keyStore, String keyStorePassword, KeySt
             register("http", new PlainConnectionSocketFactory()).
             build();
 
-        PoolingHttpClientConnectionManager conman = new PoolingHttpClientConnectionManager(socketFactoryRegistry)
+        PoolingHttpClientConnectionManager conman = new PoolingHttpClientConnectionManager(socketFactoryRegistry, null, null, null, connectionTimeToLive != null ? connectionTimeToLive : -1L, TimeUnit.MILLISECONDS)
         {
 
             // https://github.com/eclipse-ee4j/jersey/issues/4449
@@ -1574,7 +1584,8 @@ public void releaseConnection(final HttpClientConnection managedConn, final Obje
         };
         if (maxConnPerRoute != null) conman.setDefaultMaxPerRoute(maxConnPerRoute);
         if (maxTotalConn != null) conman.setMaxTotal(maxTotalConn);
-        
+        if (validateAfterInactivity != null) conman.setValidateAfterInactivity(validateAfterInactivity);
+
         ClientConfig config = new ClientConfig();
         config.connectorProvider(new ApacheConnectorProvider());
         config.register(MultiPartFeature.class);
@@ -1586,10 +1597,11 @@ public void releaseConnection(final HttpClientConnection managedConn, final Obje
         config.property(ClientProperties.FOLLOW_REDIRECTS, true);
         config.property(ClientProperties.REQUEST_ENTITY_PROCESSING, RequestEntityProcessing.BUFFERED); // https://stackoverflow.com/questions/42139436/jersey-client-throws-cannot-retry-request-with-a-non-repeatable-request-entity
         config.property(ApacheClientProperties.CONNECTION_MANAGER, conman);
-        if (connectionRequestTimeout != null)
-            config.property(ApacheClientProperties.REQUEST_CONFIG, RequestConfig.custom().
-                setConnectionRequestTimeout(connectionRequestTimeout).
-                build());
+        RequestConfig.Builder requestConfig = RequestConfig.custom();
+        if (connectionRequestTimeout != null) requestConfig.setConnectionRequestTimeout(connectionRequestTimeout);
+        if (socketTimeout != null) requestConfig.setSocketTimeout(socketTimeout);
+        if (connectTimeout != null) requestConfig.setConnectTimeout(connectTimeout);
+        config.property(ApacheClientProperties.REQUEST_CONFIG, requestConfig.build());
 
         if (maxRequestRetries != null)
             config.property(ApacheClientProperties.RETRY_HANDLER, (HttpRequestRetryHandler) (IOException ex, int executionCount, HttpContext context) ->
@@ -1625,9 +1637,14 @@ public void releaseConnection(final HttpClientConnection managedConn, final Obje
      * @param maxConnPerRoute max connections per route
      * @param maxTotalConn max total connections
      * @param maxRequestRetries maximum number of times that the HTTP client will retry a request
+     * @param connectionRequestTimeout timeout in milliseconds to wait for a connection lease from the pool (null to leave unset)
+     * @param socketTimeout socket (read) timeout in milliseconds (null to leave unset)
+     * @param connectTimeout connection (connect) timeout in milliseconds (null to leave unset)
+     * @param connectionTimeToLive time-to-live in milliseconds after which pooled connections are discarded (null to leave unset)
+     * @param validateAfterInactivity period of inactivity in milliseconds after which a pooled connection is revalidated before reuse (null to leave unset)
      * @return client instance
      */
-    public static Client getNoCertClient(KeyStore trustStore, Integer maxConnPerRoute, Integer maxTotalConn, Integer maxRequestRetries, Integer connectionRequestTimeout)
+    public static Client getNoCertClient(KeyStore trustStore, Integer maxConnPerRoute, Integer maxTotalConn, Integer maxRequestRetries, Integer connectionRequestTimeout, Integer socketTimeout, Integer connectTimeout, Long connectionTimeToLive, Integer validateAfterInactivity)
     {
         try
         {
@@ -1643,11 +1660,11 @@ public static Client getNoCertClient(KeyStore trustStore, Integer maxConnPerRout
                 register("http", new PlainConnectionSocketFactory()).
                 build();
 
-            PoolingHttpClientConnectionManager conman = new PoolingHttpClientConnectionManager(socketFactoryRegistry)
+            PoolingHttpClientConnectionManager conman = new PoolingHttpClientConnectionManager(socketFactoryRegistry, null, null, null, connectionTimeToLive != null ? connectionTimeToLive : -1L, TimeUnit.MILLISECONDS)
             {
 
                 // https://github.com/eclipse-ee4j/jersey/issues/4449
-                
+
                 @Override
                 public void close()
                 {
@@ -1674,6 +1691,7 @@ public void releaseConnection(final HttpClientConnection managedConn, final Obje
             };
             if (maxConnPerRoute != null) conman.setDefaultMaxPerRoute(maxConnPerRoute);
             if (maxTotalConn != null) conman.setMaxTotal(maxTotalConn);
+            if (validateAfterInactivity != null) conman.setValidateAfterInactivity(validateAfterInactivity);
 
             ClientConfig config = new ClientConfig();
             config.connectorProvider(new ApacheConnectorProvider());
@@ -1686,10 +1704,11 @@ public void releaseConnection(final HttpClientConnection managedConn, final Obje
             config.property(ClientProperties.FOLLOW_REDIRECTS, true);
             config.property(ClientProperties.REQUEST_ENTITY_PROCESSING, RequestEntityProcessing.BUFFERED); // https://stackoverflow.com/questions/42139436/jersey-client-throws-cannot-retry-request-with-a-non-repeatable-request-entity
             config.property(ApacheClientProperties.CONNECTION_MANAGER, conman);
-            if (connectionRequestTimeout != null)
-                config.property(ApacheClientProperties.REQUEST_CONFIG, RequestConfig.custom().
-                    setConnectionRequestTimeout(connectionRequestTimeout).
-                    build());
+            RequestConfig.Builder requestConfig = RequestConfig.custom();
+            if (connectionRequestTimeout != null) requestConfig.setConnectionRequestTimeout(connectionRequestTimeout);
+            if (socketTimeout != null) requestConfig.setSocketTimeout(socketTimeout);
+            if (connectTimeout != null) requestConfig.setConnectTimeout(connectTimeout);
+            config.property(ApacheClientProperties.REQUEST_CONFIG, requestConfig.build());
 
             if (maxRequestRetries != null)
                 config.property(ApacheClientProperties.RETRY_HANDLER, (HttpRequestRetryHandler) (IOException ex, int executionCount, HttpContext context) ->