STF-322: Add bounded transport-failure retry to WebServiceClient

When the JDK HttpClient pool reuses an idle connection that an intermediary
(load balancer, proxy, NAT) has silently closed, the next send() fails with
"Connection reset" or "Broken pipe". A single retry recovers transparently
without exposing this race to callers. The default keep-alive timeout in the
JDK is longer than many intermediaries' idle timeout, so this mismatch is the
common case.

The retry predicate is intentionally narrow: SocketException
("Connection reset" / "Broken pipe"), ConnectException, and
HttpConnectTimeoutException only. Request-phase HttpTimeoutException, 4xx,
and 5xx responses are NOT retried -- the request may have been processed and
retrying could cause unwanted side effects. The predicate walks the cause
chain because the JDK frequently wraps the underlying SocketException in a
generic IOException ("HTTP/1.1 header parser received no bytes").

Customers can opt out via .maxRetries(0). Default is 1 (one retry, two total
attempts). The interrupt flag is restored before rewrapping
InterruptedException, and a pre-set interrupt short-circuits the predicate.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: oschwald

src/main/java/com/maxmind/minfraud/WebServiceClient.java

@@ -16,18 +16,23 @@
 import com.maxmind.minfraud.response.ScoreResponse;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.ConnectException;
 import java.net.ProxySelector;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.http.HttpClient;
+import java.net.http.HttpConnectTimeoutException;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
+import java.net.http.HttpTimeoutException;
+import java.nio.channels.ClosedChannelException;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
 /**
@@ -45,6 +50,7 @@ public final class WebServiceClient {
     private final boolean useHttps;
     private final List<String> locales;
     private final Duration requestTimeout;
+    private final int maxRetries;
 
     private final HttpClient httpClient;
 
@@ -63,6 +69,7 @@ private WebServiceClient(WebServiceClient.Builder builder) {
                 .getBytes(StandardCharsets.UTF_8));
 
         requestTimeout = builder.requestTimeout;
+        maxRetries = builder.maxRetries;
         if (builder.httpClient != null) {
             httpClient = builder.httpClient;
         } else {
@@ -106,6 +113,7 @@ public static final class Builder {
         List<String> locales = List.of("en");
         private ProxySelector proxy;
         private HttpClient httpClient;
+        private int maxRetries = 1;
 
         /**
          * @param accountId  Your MaxMind account ID.
@@ -119,6 +127,12 @@ public Builder(int accountId, String licenseKey) {
         /**
          * @param val Timeout duration to establish a connection to the web service. There is no
          *            timeout by default.
+         *            <p>
+         *            When {@code maxRetries > 0}, one API call may incur up to
+         *            {@code (maxRetries + 1)} connection attempts, each subject to
+         *            {@code connectTimeout} and {@code requestTimeout}. Worst-case wall-clock
+         *            duration is roughly
+         *            {@code (maxRetries + 1) x (connectTimeout + requestTimeout)}.
          * @return Builder object
          */
         public WebServiceClient.Builder connectTimeout(Duration val) {
@@ -174,6 +188,12 @@ public WebServiceClient.Builder locales(List<String> val) {
 
         /**
          * @param val Request timeout duration. here is no timeout by default.
+         *            <p>
+         *            When {@code maxRetries > 0}, one API call may incur up to
+         *            {@code (maxRetries + 1)} connection attempts, each subject to
+         *            {@code connectTimeout} and {@code requestTimeout}. Worst-case wall-clock
+         *            duration is roughly
+         *            {@code (maxRetries + 1) x (connectTimeout + requestTimeout)}.
          * @return Builder object
          */
         public Builder requestTimeout(Duration val) {
@@ -195,13 +215,33 @@ public Builder proxy(ProxySelector val) {
          * @param val the HttpClient to use when making requests. When provided,
          *            connectTimeout and proxy settings will be ignored as the
          *            custom client should handle these configurations.
+         *            <p>
+         *            The SDK applies its own transport-failure retry on top of any supplied
+         *            client; customers can disable it via {@link #maxRetries(int)} with
+         *            {@code .maxRetries(0)}.
          * @return Builder object
          */
         public Builder httpClient(HttpClient val) {
             httpClient = val;
             return this;
         }
 
+        /**
+         * @param val Maximum number of retries on transport-level failures
+         *            (connection reset, broken pipe, connect timeout, ...).
+         *            Applies uniformly to all endpoints. Defaults to 1.
+         *            Set to 0 to disable.
+         * @return Builder.
+         * @throws IllegalArgumentException if {@code val} is negative.
+         */
+        public Builder maxRetries(int val) {
+            if (val < 0) {
+                throw new IllegalArgumentException("maxRetries must not be negative");
+            }
+            maxRetries = val;
+            return this;
+        }
+
         /**
          * @return an instance of {@code WebServiceClient} created from the fields set on this
          *     builder.
@@ -311,10 +351,11 @@ public void reportTransaction(TransactionReport transaction) throws IOException,
 
         HttpResponse<InputStream> response = null;
         try {
-            response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
+            response = sendWithRetry(request);
             maybeThrowException(response, uri);
             exhaustBody(response);
         } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
             throw new MinFraudException("Interrupted sending request", e);
         } finally {
             if (response != null) {
@@ -333,9 +374,10 @@ private <T> T responseFor(String service, AbstractModel transaction, Class<T> cl
 
         HttpResponse<InputStream> response = null;
         try {
-            response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
+            response = sendWithRetry(request);
             return handleResponse(response, uri, cls);
         } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
             throw new MinFraudException("Interrupted sending request", e);
         } finally {
             if (response != null) {
@@ -344,6 +386,61 @@ private <T> T responseFor(String service, AbstractModel transaction, Class<T> cl
         }
     }
 
+    private HttpResponse<InputStream> sendWithRetry(HttpRequest request)
+        throws IOException, InterruptedException {
+        int attempts = 0;
+        while (true) {
+            try {
+                return httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
+            } catch (IOException e) {
+                if (!isRetriableTransportFailure(e) || attempts >= maxRetries) {
+                    throw e;
+                }
+                attempts++;
+            }
+        }
+    }
+
+    private static boolean isRetriableTransportFailure(IOException e) {
+        if (Thread.currentThread().isInterrupted()) {
+            return false;
+        }
+        // Walk the cause chain: the JDK HttpClient wraps the underlying transport
+        // failure in different ways depending on the protocol path. Over HTTP/1.1
+        // a "Connection reset" surfaces as a SocketException; over HTTP/2 a
+        // SETTINGS-frame write failure may surface as a plain IOException with
+        // the same message; an HTTP/2 upgrade against an already-closed socket
+        // surfaces as a (message-less) IOException caused by a
+        // ClosedChannelException. Match by class for the framing types and by
+        // message (case-insensitive) for the resets, while keeping the explicit
+        // checks for connect-phase timeouts and request-phase timeouts (which
+        // must NEVER be retried).
+        Throwable t = e;
+        while (t != null) {
+            if (t instanceof HttpConnectTimeoutException) {
+                return true; // subclass of HttpTimeoutException - must be checked first
+            }
+            if (t instanceof HttpTimeoutException) {
+                return false; // request-phase timeout: NOT retriable
+            }
+            if (t instanceof ConnectException) {
+                return true;
+            }
+            if (t instanceof ClosedChannelException) {
+                return true;
+            }
+            String msg = t.getMessage();
+            if (msg != null) {
+                String lower = msg.toLowerCase(Locale.ROOT);
+                if (lower.contains("connection reset") || lower.contains("broken pipe")) {
+                    return true;
+                }
+            }
+            t = t.getCause();
+        }
+        return false;
+    }
+
     private HttpRequest requestFor(AbstractModel transaction, URI uri)
         throws MinFraudException, IOException {
         var builder = HttpRequest.newBuilder()
@@ -354,6 +451,7 @@ private HttpRequest requestFor(AbstractModel transaction, URI uri)
             .header("User-Agent", userAgent)
             // XXX - creating this JSON string is somewhat wasteful. We
             // could use an input stream instead.
+            // BodyPublishers.ofString() is replayable; safe for retry attempts
             .POST(HttpRequest.BodyPublishers.ofString(transaction.toJson()));
 
         if (requestTimeout != null) {