STF-322: Bounded transport-failure retry in WebServiceClient

CHANGELOG.md

@@ -1,6 +1,14 @@
 CHANGELOG
 =========
 
+5.0.3 (unreleased)
+------------------
+
+* Added `WebServiceClient.Builder.maxRetries(int)` to configure transport-failure
+  retry behavior. Defaults to 1 (one retry on connection reset, broken pipe,
+  or connect timeout). Set to 0 to disable. Request-phase timeouts and HTTP
+  4xx/5xx responses are never retried.
+
 5.0.2 (2025-12-08)
 ------------------
 

README.md

@@ -72,6 +72,41 @@ are not created for each request.
 See the [API documentation](https://maxmind.github.io/GeoIP2-java/) for
 more details.
 
+### Connection pooling and transport retries ###
+
+`WebServiceClient` is thread-safe and reuses a pooled `HttpClient` across
+requests. Idle connections in the pool can be silently closed by load
+balancers or other intermediaries. When the next request reuses one of these
+half-closed connections, the JDK reports the failure as a `Connection reset`
+(or `Broken pipe`) `IOException`.
+
+To smooth over these intermittent transport failures, the SDK retries once by
+default. The retry covers:
+
+* `SocketException` with message `Connection reset` or `Broken pipe`,
+* `ConnectException`,
+* `HttpConnectTimeoutException`.
+
+Retries are **not** applied to request-phase timeouts (`HttpTimeoutException`)
+or to HTTP 4xx / 5xx responses. Web service requests are idempotent GETs, so
+retried requests are byte-identical to the original.
+
+You can change the retry budget via the builder:
+
+```java
+WebServiceClient client = new WebServiceClient.Builder(42, "license_key")
+    .maxRetries(2) // up to two retries (three total attempts)
+    .build();
+```
+
+Set `.maxRetries(0)` to disable the retry entirely. Negative values throw
+`IllegalArgumentException`.
+
+If you frequently see `Connection reset` errors, you can also reduce the
+JDK's keep-alive timeout via the system property
+`jdk.httpclient.keepalive.timeout` (in seconds) to evict pooled connections
+before any intermediary does so.
+
 ## Web Service Example ##
 
 ### Country Service ###

mise.lock

@@ -0,0 +1,30 @@
+# @generated - this file is auto-generated by `mise lock` https://mise.jdx.dev/dev-tools/mise-lock.html
+
+[[tools.java]]
+version = "26.0.0"
+backend = "core:java"
+
+[[tools.maven]]
+version = "3.9.15"
+backend = "aqua:apache/maven"
+
+[tools.maven."platforms.linux-arm64"]
+url = "https://archive.apache.org/dist/maven/maven-3/3.9.15/binaries/apache-maven-3.9.15-bin.tar.gz"
+
+[tools.maven."platforms.linux-arm64-musl"]
+url = "https://archive.apache.org/dist/maven/maven-3/3.9.15/binaries/apache-maven-3.9.15-bin.tar.gz"
+
+[tools.maven."platforms.linux-x64"]
+url = "https://archive.apache.org/dist/maven/maven-3/3.9.15/binaries/apache-maven-3.9.15-bin.tar.gz"
+
+[tools.maven."platforms.linux-x64-musl"]
+url = "https://archive.apache.org/dist/maven/maven-3/3.9.15/binaries/apache-maven-3.9.15-bin.tar.gz"
+
+[tools.maven."platforms.macos-arm64"]
+url = "https://archive.apache.org/dist/maven/maven-3/3.9.15/binaries/apache-maven-3.9.15-bin.tar.gz"
+
+[tools.maven."platforms.macos-x64"]
+url = "https://archive.apache.org/dist/maven/maven-3/3.9.15/binaries/apache-maven-3.9.15-bin.tar.gz"
+
+[tools.maven."platforms.windows-x64"]
+url = "https://archive.apache.org/dist/maven/maven-3/3.9.15/binaries/apache-maven-3.9.15-bin.tar.gz"

mise.toml

@@ -0,0 +1,18 @@
+[settings]
+experimental = true
+lockfile = true
+disable_backends = [
+    "asdf",
+    "vfox",
+]
+
+[tools]
+java = "latest"
+maven = "latest"
+
+[hooks]
+enter = "mise install --quiet --locked"
+
+[[watch_files]]
+patterns = ["mise.toml", "mise.lock"]
+run = "mise install --quiet --locked"

src/main/java/com/maxmind/geoip2/WebServiceClient.java

@@ -20,14 +20,17 @@
 import com.maxmind.geoip2.model.InsightsResponse;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.ConnectException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.ProxySelector;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.http.HttpClient;
+import java.net.http.HttpConnectTimeoutException;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
+import java.net.http.HttpTimeoutException;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.Base64;
@@ -112,6 +115,7 @@ public class WebServiceClient implements WebServiceProvider {
     private final boolean useHttps;
     private final int port;
     private final Duration requestTimeout;
+    private final int maxRetries;
     private final String userAgent = "GeoIP2/"
         + getClass().getPackage().getImplementationVersion()
         + " (Java/" + System.getProperty("java.version") + ")";
@@ -125,6 +129,7 @@ private WebServiceClient(Builder builder) {
         this.port = builder.port;
         this.useHttps = builder.useHttps;
         this.locales = builder.locales;
+        this.maxRetries = builder.maxRetries;
 
         // HttpClient supports basic auth, but it will only send it after the
         // server responds with an unauthorized. As such, we just make the
@@ -182,6 +187,7 @@ public static final class Builder {
         List<String> locales = List.of("en");
         private ProxySelector proxy = null;
         private HttpClient httpClient = null;
+        private int maxRetries = 1;
 
         /**
          * @param accountId  Your MaxMind account ID.
@@ -196,6 +202,12 @@ public Builder(int accountId, String licenseKey) {
         /**
          * @param val Timeout duration to establish a connection to the
          *            web service. The default is 3 seconds.
+         *            <p>
+         *            When {@code maxRetries > 0}, one API call may incur up to
+         *            {@code (maxRetries + 1)} connection attempts, each subject to
+         *            {@code connectTimeout} and {@code requestTimeout}. Worst-case
+         *            wall-clock duration is roughly
+         *            {@code (maxRetries + 1) x (connectTimeout + requestTimeout)}.
          * @return Builder object
          */
         public Builder connectTimeout(Duration val) {
@@ -250,6 +262,12 @@ public Builder locales(List<String> val) {
 
         /**
          * @param val Request timeout duration. The default is 20 seconds.
+         *            <p>
+         *            When {@code maxRetries > 0}, one API call may incur up to
+         *            {@code (maxRetries + 1)} connection attempts, each subject to
+         *            {@code connectTimeout} and {@code requestTimeout}. Worst-case
+         *            wall-clock duration is roughly
+         *            {@code (maxRetries + 1) x (connectTimeout + requestTimeout)}.
          * @return Builder object
          */
         public Builder requestTimeout(Duration val) {
@@ -271,13 +289,33 @@ public Builder proxy(ProxySelector val) {
          * @param val the custom HttpClient to use for requests. When providing a
          *            custom HttpClient, you cannot also set connectTimeout or proxy
          *            parameters as these should be configured on the provided client.
+         *            <p>
+         *            The SDK applies its own transport-failure retry on top of any
+         *            supplied client; customers can disable it via
+         *            {@link #maxRetries(int)} with {@code .maxRetries(0)}.
          * @return Builder object
          */
         public Builder httpClient(HttpClient val) {
             this.httpClient = val;
             return this;
         }
 
+        /**
+         * @param val Maximum number of retries on transport-level failures
+         *            (connection reset, broken pipe, connect timeout, ...).
+         *            Applies uniformly to all endpoints. Defaults to 1.
+         *            Set to 0 to disable.
+         * @return Builder.
+         * @throws IllegalArgumentException if {@code val} is negative.
+         */
+        public Builder maxRetries(int val) {
+            if (val < 0) {
+                throw new IllegalArgumentException("maxRetries must not be negative");
+            }
+            maxRetries = val;
+            return this;
+        }
+
         /**
          * @return an instance of {@code WebServiceClient} created from the
          * fields set on this builder.
@@ -371,18 +409,68 @@ private <T> T responseFor(String path, InetAddress ipAddress, Class<T> cls)
             .GET()
             .build();
         try {
-            var response = this.httpClient
-                .send(request, HttpResponse.BodyHandlers.ofInputStream());
+            var response = sendWithRetry(request);
             try {
                 return handleResponse(response, cls);
             } finally {
                 response.body().close();
             }
         } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
             throw new GeoIp2Exception("Interrupted sending request", e);
         }
     }
 
+    private HttpResponse<InputStream> sendWithRetry(HttpRequest request)
+        throws IOException, InterruptedException {
+        IOException lastException = null;
+        int attempts = maxRetries + 1;
+        for (int i = 0; i < attempts; i++) {
+            try {
+                return httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
+            } catch (IOException e) {
+                if (!isRetriableTransportFailure(e) || i == attempts - 1) {
+                    throw e;
+                }
+                lastException = e;
+            }
+        }
+        // Unreachable: loop either returns or throws.
+        throw lastException;
+    }
+
+    private static boolean isRetriableTransportFailure(IOException e) {
+        if (Thread.currentThread().isInterrupted()) {
+            return false;
+        }
+        // Walk the cause chain: the JDK HttpClient wraps the underlying transport
+        // failure in different ways depending on the protocol path. Over HTTP/1.1
+        // a "Connection reset" surfaces as a SocketException; over HTTP/2 (e.g.
+        // a SETTINGS-frame write failure) it may surface as a plain IOException
+        // with the same message. Match by message regardless of class to handle
+        // both, while keeping the type checks for connect-phase timeouts and
+        // request-phase timeouts (which must NEVER be retried).
+        Throwable t = e;
+        while (t != null) {
+            if (t instanceof HttpConnectTimeoutException) {
+                return true; // subclass of HttpTimeoutException - must be checked first
+            }
+            if (t instanceof HttpTimeoutException) {
+                return false; // request-phase timeout: NOT retriable
+            }
+            if (t instanceof ConnectException) {
+                return true;
+            }
+            String msg = t.getMessage();
+            if (msg != null
+                && (msg.contains("Connection reset") || msg.contains("Broken pipe"))) {
+                return true;
+            }
+            t = t.getCause();
+        }
+        return false;
+    }
+
     private <T> T handleResponse(HttpResponse<InputStream> response, Class<T> cls)
         throws GeoIp2Exception, IOException {
         var status = response.statusCode();