STF-322: Add bounded transport-failure retry to WebServiceClient

When the JDK HttpClient pool reuses an idle connection that an intermediary
(load balancer, proxy, NAT) has silently closed, the next send() fails with
"Connection reset", "Broken pipe", or related transport errors. A single
retry recovers transparently without exposing this race to callers. The
default JDK keep-alive timeout exceeds many intermediaries' idle timeout,
making this mismatch the common case.

The retry predicate is permissive by exclusion: any IOException from
httpClient.send() is retried EXCEPT HttpTimeoutException (covering both
request-phase and connect-phase timeouts, since HttpConnectTimeoutException
is a subclass) and InterruptedIOException. Both timeouts are customer-set
budgets that retrying would silently extend; InterruptedIOException is a
user-cancellation signal. HTTP 4xx and 5xx responses are surfaced as
HttpException (and subclasses) from a separate code path -- they come back
as HttpResponse objects rather than IOExceptions, so the predicate is
structurally unable to retry them.

Customers can opt out via .maxRetries(0). Default is 1 (one retry, two total
attempts). The interrupt flag is restored before rewrapping
InterruptedException, and a pre-set interrupt short-circuits the predicate.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: oschwald

src/main/java/com/maxmind/geoip2/WebServiceClient.java

@@ -20,20 +20,26 @@
 import com.maxmind.geoip2.model.InsightsResponse;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.ProxySelector;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.net.UnknownHostException;
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
+import java.net.http.HttpTimeoutException;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.Base64;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.SSLPeerUnverifiedException;
 
 /**
  * <p>
@@ -112,6 +118,7 @@ public class WebServiceClient implements WebServiceProvider {
     private final boolean useHttps;
     private final int port;
     private final Duration requestTimeout;
+    private final int maxRetries;
     private final String userAgent = "GeoIP2/"
         + getClass().getPackage().getImplementationVersion()
         + " (Java/" + System.getProperty("java.version") + ")";
@@ -125,6 +132,7 @@ private WebServiceClient(Builder builder) {
         this.port = builder.port;
         this.useHttps = builder.useHttps;
         this.locales = builder.locales;
+        this.maxRetries = builder.maxRetries;
 
         // HttpClient supports basic auth, but it will only send it after the
         // server responds with an unauthorized. As such, we just make the
@@ -182,6 +190,7 @@ public static final class Builder {
         List<String> locales = List.of("en");
         private ProxySelector proxy = null;
         private HttpClient httpClient = null;
+        private int maxRetries = 1;
 
         /**
          * @param accountId  Your MaxMind account ID.
@@ -197,6 +206,7 @@ public Builder(int accountId, String licenseKey) {
          * @param val Timeout duration to establish a connection to the
          *            web service. The default is 3 seconds.
          * @return Builder object
+         * @apiNote See {@link #maxRetries(int)} for how this timeout interacts with retries.
          */
         public Builder connectTimeout(Duration val) {
             this.connectTimeout = val;
@@ -251,6 +261,7 @@ public Builder locales(List<String> val) {
         /**
          * @param val Request timeout duration. The default is 20 seconds.
          * @return Builder object
+         * @apiNote See {@link #maxRetries(int)} for how this timeout interacts with retries.
          */
         public Builder requestTimeout(Duration val) {
             this.requestTimeout = val;
@@ -271,13 +282,42 @@ public Builder proxy(ProxySelector val) {
          * @param val the custom HttpClient to use for requests. When providing a
          *            custom HttpClient, you cannot also set connectTimeout or proxy
          *            parameters as these should be configured on the provided client.
+         *            <p>
+         *            The SDK applies its own transport-failure retry on top of any
+         *            supplied client; customers can disable it via
+         *            {@link #maxRetries(int)} with {@code .maxRetries(0)}.
          * @return Builder object
          */
         public Builder httpClient(HttpClient val) {
             this.httpClient = val;
             return this;
         }
 
+        /**
+         * @param val Maximum number of retries on transport-level failures
+         *            (connection reset, broken pipe, EOF, ...).
+         *            Applies uniformly to all endpoints. Defaults to 1.
+         *            Set to 0 to disable.
+         * @return Builder.
+         * @throws IllegalArgumentException if {@code val} is negative.
+         * @apiNote Timeouts are not retried ({@link java.net.http.HttpTimeoutException},
+         *     including the connect-phase subclass
+         *     {@link java.net.http.HttpConnectTimeoutException}). When
+         *     {@code maxRetries > 0}, retries are triggered only by fast transport
+         *     failures, so each attempt is independently bounded by
+         *     {@link #connectTimeout(Duration)} and {@link #requestTimeout(Duration)}.
+         *     The multiplied worst-case wall clock a naive reading suggests is
+         *     unreachable in practice, since hitting the timeout aborts the call
+         *     rather than triggering a retry.
+         */
+        public Builder maxRetries(int val) {
+            if (val < 0) {
+                throw new IllegalArgumentException("maxRetries must not be negative");
+            }
+            maxRetries = val;
+            return this;
+        }
+
         /**
          * @return an instance of {@code WebServiceClient} created from the
          * fields set on this builder.
@@ -371,8 +411,7 @@ private <T> T responseFor(String path, InetAddress ipAddress, Class<T> cls)
             .GET()
             .build();
         try {
-            var response = this.httpClient
-                .send(request, HttpResponse.BodyHandlers.ofInputStream());
+            var response = sendWithRetry(request);
             try {
                 return handleResponse(response, cls);
             } finally {
@@ -383,6 +422,62 @@ private <T> T responseFor(String path, InetAddress ipAddress, Class<T> cls)
         }
     }
 
+    private HttpResponse<InputStream> sendWithRetry(HttpRequest request)
+        throws IOException, InterruptedException {
+        int attempts = 0;
+        IOException prior = null;
+        while (true) {
+            try {
+                return httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
+            } catch (IOException e) {
+                if (prior != null) {
+                    e.addSuppressed(prior);
+                }
+                if (!isRetriableTransportFailure(e) || attempts >= maxRetries) {
+                    throw e;
+                }
+                prior = e;
+                attempts++;
+            }
+        }
+    }
+
+    private static boolean isRetriableTransportFailure(IOException e) {
+        if (Thread.currentThread().isInterrupted()) {
+            return false;
+        }
+        // Both connect-phase and request-phase timeouts are customer-set
+        // budgets that retrying would silently extend.
+        // HttpConnectTimeoutException extends HttpTimeoutException, so this
+        // single check covers both.
+        if (e instanceof HttpTimeoutException) {
+            return false;
+        }
+        // The thread was interrupted during I/O; honor the cancellation.
+        if (e instanceof InterruptedIOException) {
+            return false;
+        }
+        // Typically deterministic failures: retrying just delays surfacing the
+        // config bug without recovering the request.
+        if (e instanceof UnknownHostException) {
+            return false;
+        }
+        if (e instanceof ConnectException) {
+            return false;
+        }
+        if (e instanceof SSLHandshakeException) {
+            return false;
+        }
+        if (e instanceof SSLPeerUnverifiedException) {
+            return false;
+        }
+        // Everything else from httpClient.send() is a transport failure
+        // (connection reset, broken pipe, EOF, closed channel, ...).
+        // HTTP 4xx and 5xx responses do not reach this predicate -- they come
+        // back as HttpResponse objects rather than IOExceptions.
+        return true;
+    }
+
     private <T> T handleResponse(HttpResponse<InputStream> response, Class<T> cls)
         throws GeoIp2Exception, IOException {
         var status = response.statusCode();