Make apache5-client the default sync HTTP client (#6732)

* Swap apache-client with apache5-client

This commit swaps the default apache-client dependency of service
modules with apache5-client. Applications just relying on the default
HTTP client will now use `Apache5HttpClient` as a result.

* Fix CloudFrontUtilitiesIntegrationTest

* Fix S3 tests

* Update version in changelog entry

* Revert to 4.x for test

Expect100ContinueHeaderTest tests 4.x specific behavior (default
100-continue).

* Add apache-client to aws-sdk-java module to prevent NoClassDefFoundError for customers using ApacheHttpClient in environments that bundle the whole SDK (e.g., EMR) after apache5-client default switch (#6964)

* fix(integ-test): Fix Integ test issue after making Apache5 as default (#6974)

* fix(dynamodb): Add TCP keep-alive permissions to SecurityManager test policy for Apache 5 compatibility. Apache 5.6 sets TCP_KEEPIDLE/INTERVAL/COUNT by default which requires jdk.net.NetworkPermission.

* Fix ResponseInputStreamTimeoutIntegrationTest to update it to Apache5 exception assertion

* Remove apache5-client test scope from s3/pom.xml to restore runtime transitivity. The test scope override prevented downstream modules like s3-transfer-manager from discovering the HTTP client via SPI

* Dumy commit to trigger integ test

* because we use  org.apache.hc.core5.http.ConnectionRequestTimeoutException in ResponseInputStreamTimeoutIntegrationTest

* Removing Apache5 specific execption from ResponseInputStreamTimeoutIntegrationTest and asserting just on error message

* remove apache 5 as test dependency since it present as runtime

* Updated change logs to have notable changes in the change logs

* Upgrade httpcomponents.client5 to 5.6.1

---------

Co-authored-by: John Viegas <70235430+joviegas@users.noreply.github.com>
Co-authored-by: John Viegas <joviegas@amazon.com>

Author: dagnir

.changes/next-release/feature-AWSSDKforJavav2-890d71f.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "This update replaces the default `apache-client` runtime dependency of service clients with the new `apache5-client`. This means that service clients will now use the `Apache5HttpClient` by default if no HTTP client is explicitly configured on the service client builder.\\n Notable changes:\\n - Apache 5 uses different logger names than Apache 4\\n - Expect: 100-Continue is disabled by default\\n - TCP keep-alive socket options require `jdk.net.NetworkPermission` when SecurityManager is active"
+}

aws-sdk-java/pom.xml

@@ -2188,6 +2188,11 @@ Amazon AutoScaling, etc).</description>
             <artifactId>resiliencehubv2</artifactId>
             <version>${awsjavasdk.version}</version>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>apache-client</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
     </dependencies>
     <build>
         <finalName>${project.artifactId}-${project.version}</finalName>

services/cloudfront/src/test/java/software/amazon/awssdk/services/cloudfront/CloudFrontUtilitiesIntegrationTest.java

@@ -50,7 +50,7 @@
 import software.amazon.awssdk.http.SdkHttpClient;
 import software.amazon.awssdk.http.SdkHttpMethod;
 import software.amazon.awssdk.http.SdkHttpRequest;
-import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.http.apache5.Apache5HttpClient;
 import software.amazon.awssdk.services.cloudfront.cookie.CookiesForCannedPolicy;
 import software.amazon.awssdk.services.cloudfront.cookie.CookiesForCustomPolicy;
 import software.amazon.awssdk.services.cloudfront.internal.utils.SigningUtils;
@@ -142,7 +142,7 @@ static Stream<KeyTestCase> keyCases() throws Exception {
 
     @Test
     void unsignedUrl_shouldReturn403Response() throws Exception {
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response =
             client.prepareRequest(HttpExecuteRequest.builder()
                                                     .request(SdkHttpRequest.builder()
@@ -167,7 +167,7 @@ void getSignedUrlWithCannedPolicy_producesValidUrl(KeyTestCase testCase) throws
                                                          .keyPairId(testCase.keyPairId)
                                                          .expirationDate(expirationDate).build();
         SignedUrl signedUrl = cloudFrontUtilities.getSignedUrlWithCannedPolicy(request);
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response = client.prepareRequest(HttpExecuteRequest.builder()
                                                                                .request(signedUrl.createHttpGetRequest())
                                                                                .build()).call();
@@ -186,7 +186,7 @@ void getSignedUrlWithCannedPolicy_withExpiredDate_shouldReturn403Response(KeyTes
                                                                                       .privateKey(testCase.privateKey)
                                                                                       .keyPairId(testCase.keyPairId)
                                                                                       .expirationDate(expirationDate));
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response = client.prepareRequest(HttpExecuteRequest.builder()
                                                                                .request(signedUrl.createHttpGetRequest())
                                                                                .build()).call();
@@ -207,7 +207,7 @@ void getSignedUrlWithCustomPolicy_producesValidUrl(KeyTestCase testCase) throws
                                                          .expirationDate(expirationDate)
                                                          .activeDate(activeDate).build();
         SignedUrl signedUrl = cloudFrontUtilities.getSignedUrlWithCustomPolicy(request);
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response = client.prepareRequest(HttpExecuteRequest.builder()
                                                                                .request(signedUrl.createHttpGetRequest())
                                                                                .build()).call();
@@ -227,7 +227,7 @@ void getSignedUrlWithCustomPolicy_withFutureActiveDate_shouldReturn403Response()
                                                                                       .keyPairId(rsaKeyPairId)
                                                                                       .expirationDate(expirationDate)
                                                                                       .activeDate(activeDate));
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response = client.prepareRequest(HttpExecuteRequest.builder()
                                                                                .request(signedUrl.createHttpGetRequest())
                                                                                .build()).call();
@@ -245,7 +245,7 @@ void getCookiesForCannedPolicy_producesValidCookies(KeyTestCase testCase) throws
                                                                                              .keyPairId(testCase.keyPairId)
                                                                                              .expirationDate(expirationDate));
 
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response = client.prepareRequest(HttpExecuteRequest.builder()
                                                                                .request(cookies.createHttpGetRequest())
                                                                                .build()).call();
@@ -266,7 +266,7 @@ void getCookiesForCannedPolicy_withExpiredDate_shouldReturn403Response() throws
                                                          .expirationDate(expirationDate).build();
         CookiesForCannedPolicy cookies = cloudFrontUtilities.getCookiesForCannedPolicy(request);
 
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response = client.prepareRequest(HttpExecuteRequest.builder()
                                                                                .request(cookies.createHttpGetRequest())
                                                                                .build()).call();
@@ -286,7 +286,7 @@ void getCookiesForCustomPolicy_producesValidCookies(KeyTestCase testCase) throws
                                                                                              .expirationDate(expirationDate)
                                                                                              .activeDate(activeDate));
 
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response = client.prepareRequest(HttpExecuteRequest.builder()
                                                                                .request(cookies.createHttpGetRequest())
                                                                                .build()).call();
@@ -309,7 +309,7 @@ void getCookiesForCustomPolicy_withFutureActiveDate_shouldReturn403Response() th
                                                          .activeDate(activeDate).build();
         CookiesForCustomPolicy cookies = cloudFrontUtilities.getCookiesForCustomPolicy(request);
 
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response = client.prepareRequest(HttpExecuteRequest.builder()
                                                                                .request(cookies.createHttpGetRequest())
                                                                                .build()).call();
@@ -337,7 +337,7 @@ void getCookiesForCustomPolicy_shouldAllowQueryParametersWhenUsingWildcard(KeyTe
 
         // Request the same resource with an additional query parameter - should still be allowed by the wildcard policy
         URI uri = URI.create(resourceUrl + "?foo=bar");
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response = client.prepareRequest(HttpExecuteRequest.builder()
                                                                                .request(SdkHttpRequest.builder()
                                                                                                       .uri(uri)
@@ -372,7 +372,7 @@ void getCookiesForCustomPolicy_wildCardPath(KeyTestCase testCase) throws Excepti
 
         // Use the cookies to access a different file under the same wildcard path
         URI otherFileUri = URI.create(resourceUri + "/foo/other-file");
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response = client.prepareRequest(HttpExecuteRequest.builder()
                                                                                .request(SdkHttpRequest.builder()
                                                                                                       .uri(otherFileUri)
@@ -406,7 +406,7 @@ void getCookiesForCustomPolicy_wildCardPolicyResource_allowsAnyPath(KeyTestCase
 
         // Use the cookies to access a completely different path - the "*" pattern should allow any path
         URI differentPathUri = URI.create(resourceUrl.replace("/s3ObjectKey", "/foo/other-file"));
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response = client.prepareRequest(HttpExecuteRequest.builder()
                                                                                .request(SdkHttpRequest.builder()
                                                                                                       .uri(differentPathUri)
@@ -445,7 +445,7 @@ void getSignedUrlWithCustomPolicy_shouldAllowQueryParametersWhenUsingWildcard(Ke
         URI modifiedUri = URI.create(urlWithDynamicParam);
 
 
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response = client.prepareRequest(HttpExecuteRequest.builder()
                                                                                .request(SdkHttpRequest.builder()
                                                                                                       .encodedPath(modifiedUri.getRawPath() + "?" + modifiedUri.getRawQuery())
@@ -482,7 +482,7 @@ void getSignedUrlWithCustomPolicy_wildCardPath(KeyTestCase testCase) throws Exce
 
 
         URI modifiedUri = URI.create(signedUrl.url().replace("/specific-file","/other-file"));
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response = client.prepareRequest(HttpExecuteRequest.builder()
                                                                                .request(SdkHttpRequest.builder()
                                                                                                       .encodedPath(modifiedUri.getRawPath() + "?" + modifiedUri.getRawQuery())
@@ -518,7 +518,7 @@ void getSignedUrlWithCustomPolicy_wildCardPolicyResource_allowsAnyPath(KeyTestCa
 
 
         URI modifiedUri = URI.create(signedUrl.url().replace("/s3ObjectKey","/foo/other-file"));
-        SdkHttpClient client = ApacheHttpClient.create();
+        SdkHttpClient client = Apache5HttpClient.create();
         HttpExecuteResponse response = client.prepareRequest(HttpExecuteRequest.builder()
                                                                                .request(SdkHttpRequest.builder()
                                                                                                       .encodedPath(modifiedUri.getRawPath() + "?" + modifiedUri.getRawQuery())

services/dynamodb/src/it/java/software/amazon/awssdk/services/dynamodb/SignersIntegrationTest.java

@@ -37,7 +37,7 @@
 import software.amazon.awssdk.http.SdkHttpClient;
 import software.amazon.awssdk.http.SdkHttpFullRequest;
 import software.amazon.awssdk.http.SdkHttpMethod;
-import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.http.apache5.Apache5HttpClient;
 import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
 import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
 import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
@@ -132,7 +132,7 @@ public void sign_WithoutUsingSdkClient_ThroughExecutionAttributes() throws Excep
         // sign the request
         SdkHttpFullRequest signedRequest = signer.sign(httpFullRequest, constructExecutionAttributes());
 
-        SdkHttpClient httpClient = ApacheHttpClient.builder().build();
+        SdkHttpClient httpClient = Apache5HttpClient.builder().build();
 
         HttpExecuteRequest request = HttpExecuteRequest.builder()
                                                        .request(signedRequest)
@@ -156,7 +156,7 @@ public void test_SignMethod_WithModeledParam_And_WithoutUsingSdkClient() throws
         // sign the request
         SdkHttpFullRequest signedRequest = signer.sign(httpFullRequest, constructSignerParams());
 
-        SdkHttpClient httpClient = ApacheHttpClient.builder().build();
+        SdkHttpClient httpClient = Apache5HttpClient.builder().build();
 
         HttpExecuteRequest request = HttpExecuteRequest.builder()
                                                        .request(signedRequest)

services/dynamodb/src/test/resources/software/amazon/awssdk/services/dynamodb/security-manager-integ-test.policy

@@ -6,6 +6,11 @@ grant {
     permission "javax.net.ssl.SSLPermission" "setDefaultSSLContext";
     permission "java.net.SocketPermission" "*", "connect,resolve";
 
+    // Needed for Apache5 HTTP Client TCP keep-alive socket options
+    permission jdk.net.NetworkPermission "setOption.TCP_KEEPIDLE";
+    permission jdk.net.NetworkPermission "setOption.TCP_KEEPINTERVAL";
+    permission jdk.net.NetworkPermission "setOption.TCP_KEEPCOUNT";
+
     // Needed for test to remove the security manager
     permission java.lang.RuntimePermission "setSecurityManager";
 

services/pom.xml

@@ -532,7 +532,7 @@
             <version>${awsjavasdk.version}</version>
         </dependency>
         <dependency>
-            <artifactId>apache-client</artifactId>
+            <artifactId>apache5-client</artifactId>
             <groupId>software.amazon.awssdk</groupId>
             <version>${awsjavasdk.version}</version>
             <scope>runtime</scope>

services/s3/pom.xml

@@ -203,6 +203,12 @@
             <version>${awsjavasdk.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>apache-client</artifactId>
+            <version>${awsjavasdk.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-transport</artifactId>

services/s3/src/it/java/software/amazon/awssdk/services/s3/ResponseInputStreamTimeoutIntegrationTest.java

@@ -21,15 +21,14 @@
 
 import java.io.IOException;
 import java.time.Duration;
-import org.apache.http.conn.ConnectionPoolTimeoutException;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import software.amazon.awssdk.core.ResponseInputStream;
 import software.amazon.awssdk.core.sync.RequestBody;
 import software.amazon.awssdk.core.sync.ResponseTransformer;
-import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.http.apache5.Apache5HttpClient;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectResponse;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
@@ -48,7 +47,7 @@ public class ResponseInputStreamTimeoutIntegrationTest extends S3IntegrationTest
     @Before
     public void init() {
         s3Client = s3ClientBuilder()
-            .httpClientBuilder(ApacheHttpClient.builder().maxConnections(1))
+            .httpClientBuilder(Apache5HttpClient.builder().maxConnections(1))
             .overrideConfiguration(o -> o.retryStrategy(r -> r.maxAttempts(1)))
             .build();
     }
@@ -74,8 +73,7 @@ public void defaultTimeout_firstStreamNotConsumed_secondRequestTimesOut() {
         s3Client.getObject(getObjectRequest);
 
         assertThatThrownBy(() -> s3Client.getObject(getObjectRequest))
-            .hasRootCauseInstanceOf(ConnectionPoolTimeoutException.class)
-            .hasMessageContaining("Timeout waiting for connection from pool");
+            .hasMessageContaining("Timeout deadline");
     }
 
     @Test