split upload tests per client type: sync, async, crt, multipart-java. Lower large non-file content from 80mib to 60mib to attempt lower build time.

Author: L-Applin

.github/workflows/codebuild-ci.yml

@@ -167,11 +167,46 @@ jobs:
         uses: aws-actions/aws-codebuild-run-build@v1
         with:
           project-name: aws-sdk-java-v2-migration-test
-  s3-regression-tests-upload:
+
+  s3-regression-tests-download:
+    if: github.repository == 'aws/aws-sdk-java-v2'
+    runs-on: ubuntu-latest
+    env:
+      REGRESSION_TEST: DownloadStreamingRegressionTesting
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
+          aws-region: us-west-2
+          role-duration-seconds: 7200
+      - name: Run s3 regression tests for downloads
+        uses: aws-actions/aws-codebuild-run-build@v1
+        with:
+          project-name: aws-sdk-java-v2-s3-regression-tests
+          env-vars-for-codebuild: REGRESSION_TEST
+  s3-regression-tests-control-plane:
+    if: github.repository == 'aws/aws-sdk-java-v2'
+    runs-on: ubuntu-latest
+    env:
+      REGRESSION_TEST: ControlPlaneOperationRegressionTesting
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
+          aws-region: us-west-2
+          role-duration-seconds: 7200
+      - name: Run s3 regression tests for control plane
+        uses: aws-actions/aws-codebuild-run-build@v1
+        with:
+          project-name: aws-sdk-java-v2-s3-regression-tests
+          env-vars-for-codebuild: REGRESSION_TEST
+  s3-regression-tests-upload-sync:
     if: github.repository == 'aws/aws-sdk-java-v2'
     runs-on: ubuntu-latest
     env:
-      REGRESSION_TEST: UploadStreamingRegressionTesting
+      REGRESSION_TEST: UploadSyncRegressionTesting
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4
@@ -184,36 +219,53 @@ jobs:
         with:
           project-name: aws-sdk-java-v2-s3-regression-tests
           env-vars-for-codebuild: REGRESSION_TEST
-  s3-regression-tests-download:
+  s3-regression-tests-upload-async:
     if: github.repository == 'aws/aws-sdk-java-v2'
     runs-on: ubuntu-latest
     env:
-      REGRESSION_TEST: DownloadStreamingRegressionTesting
+      REGRESSION_TEST: UploadAsyncRegressionTesting
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
           aws-region: us-west-2
           role-duration-seconds: 7200
-      - name: Run s3 regression tests for downloads
+      - name: Run s3 regression tests for uploads
         uses: aws-actions/aws-codebuild-run-build@v1
         with:
           project-name: aws-sdk-java-v2-s3-regression-tests
           env-vars-for-codebuild: REGRESSION_TEST
-  s3-regression-tests-control-plane:
+  s3-regression-tests-upload-crt:
     if: github.repository == 'aws/aws-sdk-java-v2'
     runs-on: ubuntu-latest
     env:
-      REGRESSION_TEST: ControlPlaneOperationRegressionTesting
+      REGRESSION_TEST: UploadCrtRegressionTesting
     steps:
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
           aws-region: us-west-2
           role-duration-seconds: 7200
-      - name: Run s3 regression tests for control plane
+      - name: Run s3 regression tests for uploads
+        uses: aws-actions/aws-codebuild-run-build@v1
+        with:
+          project-name: aws-sdk-java-v2-s3-regression-tests
+          env-vars-for-codebuild: REGRESSION_TEST
+  s3-regression-tests-upload-multi:
+    if: github.repository == 'aws/aws-sdk-java-v2'
+    runs-on: ubuntu-latest
+    env:
+      REGRESSION_TEST: UploadTransferManagerRegressionTesting
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
+          aws-region: us-west-2
+          role-duration-seconds: 7200
+      - name: Run s3 regression tests for uploads
         uses: aws-actions/aws-codebuild-run-build@v1
         with:
           project-name: aws-sdk-java-v2-s3-regression-tests

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ChecksumsTestUtils.java

@@ -41,6 +41,7 @@
 import software.amazon.awssdk.services.s3.model.LocationInfo;
 import software.amazon.awssdk.services.s3.model.LocationType;
 import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.regression.upload.FlattenUploadConfig;
 import software.amazon.awssdk.services.s3control.S3ControlClient;
 import software.amazon.awssdk.services.s3control.model.CreateMultiRegionAccessPointRequest;
 import software.amazon.awssdk.services.s3control.model.GetMultiRegionAccessPointResponse;
@@ -89,7 +90,7 @@ public static void createBucket(S3Client s3, String bucketName, int retryCount,
                 }
             } else if ("OperationAborted".equals(e.awsErrorDetails().errorCode())) {
                 log.warn(() -> e.awsErrorDetails().errorMessage() + " --- Likely another operation is creating the bucket, "
-                                + "just wait for the bucket to be available");
+                               + "just wait for the bucket to be available");
             } else {
                 throw e;
             }
@@ -194,6 +195,12 @@ public static void assumeNotAccessPointWithPathStyle(TestConfig config) {
                                 "Path style doesn't work with ARN type buckets");
     }
 
+    public static void assumeNotAccessPointWithPathStyle(FlattenUploadConfig config) {
+        BucketType bucketType = config.getBucketType();
+        Assumptions.assumeFalse(config.isForcePathStyle() && bucketType.isArnType(),
+                                "Path style doesn't work with ARN type buckets");
+    }
+
     public static String crc32(String s) {
         return crc32(s.getBytes(StandardCharsets.UTF_8));
     }
@@ -266,25 +273,22 @@ public static S3AsyncClient makeAsyncClient(TestConfig config, Region region, Aw
         }
     }
 
-    public static S3Client makeSyncClient(TestConfig config, ClientOverrideConfiguration overrideConfiguration,
-                                   Region region, AwsCredentialsProvider provider) {
-        switch (config.getFlavor()) {
-            case STANDARD_SYNC:
-                return S3Client.builder()
-                               .overrideConfiguration(overrideConfiguration)
-                               .forcePathStyle(config.isForcePathStyle())
-                               .requestChecksumCalculation(config.getRequestChecksumValidation())
-                               .region(region)
-                               .credentialsProvider(provider)
-                               .build();
-            default:
-                throw new RuntimeException("Unsupported sync flavor: " + config.getFlavor());
-        }
+    public static S3Client makeSyncClient(FlattenUploadConfig config, ClientOverrideConfiguration overrideConfiguration,
+                                          Region region, AwsCredentialsProvider provider) {
+        return S3Client.builder()
+                       .overrideConfiguration(overrideConfiguration)
+                       .forcePathStyle(config.isForcePathStyle())
+                       .requestChecksumCalculation(config.getRequestChecksumValidation())
+                       .region(region)
+                       .credentialsProvider(provider)
+                       .build();
     }
 
-    public static S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfiguration overrideConfiguration,
-                                         Region region, AwsCredentialsProvider provider) {
-        switch (config.getFlavor()) {
+    public static S3AsyncClient makeAsyncClient(FlattenUploadConfig config,
+                                                S3ClientFlavor flavor,
+                                                ClientOverrideConfiguration overrideConfiguration,
+                                                Region region, AwsCredentialsProvider provider) {
+        switch (flavor) {
             case STANDARD_ASYNC:
                 return S3AsyncClient.builder()
                                     .overrideConfiguration(overrideConfiguration)
@@ -311,13 +315,15 @@ public static S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideCon
                                     .build();
             }
             default:
-                throw new RuntimeException("Unsupported async flavor: " + config.getFlavor());
+                throw new RuntimeException("Unsupported async flavor: " + flavor);
         }
     }
 
-    public static S3TransferManager makeTm(TestConfig config, ClientOverrideConfiguration overrideConfiguration,
-                                            Region region, AwsCredentialsProvider provider) {
-        S3AsyncClient s3AsyncClient = makeAsyncClient(config, overrideConfiguration, region, provider);
+    public static S3TransferManager makeTm(FlattenUploadConfig config,
+                                           S3ClientFlavor flavor,
+                                           ClientOverrideConfiguration overrideConfiguration,
+                                           Region region, AwsCredentialsProvider provider) {
+        S3AsyncClient s3AsyncClient = makeAsyncClient(config, flavor, overrideConfiguration, region, provider);
         return S3TransferManager.builder().s3Client(s3AsyncClient).build();
     }
 

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/TestCallable.java

@@ -22,7 +22,7 @@ public class TestCallable<ResponseT> {
     private SdkAutoCloseable client;
     private Callable<ResponseT> runnable;
 
-    TestCallable(SdkAutoCloseable client, Callable<ResponseT> runnable) {
+    public TestCallable(SdkAutoCloseable client, Callable<ResponseT> runnable) {
         this.client = client;
         this.runnable = runnable;
     }

test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/upload/FlattenUploadConfig.java

@@ -0,0 +1,122 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.services.s3.regression.upload;
+
+import java.util.ArrayList;
+import java.util.List;
+import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
+import software.amazon.awssdk.services.s3.regression.BucketType;
+import software.amazon.awssdk.utils.ToString;
+
+public class FlattenUploadConfig {
+    private BucketType bucketType;
+    private boolean forcePathStyle;
+    private RequestChecksumCalculation requestChecksumValidation;
+    private UploadStreamingRegressionTesting.BodyType bodyType;
+    private UploadStreamingRegressionTesting.ContentSize contentSize;
+    private boolean payloadSigning;
+
+    public static List<FlattenUploadConfig> testConfigs() {
+        List<FlattenUploadConfig> configs = new ArrayList<>();
+
+        boolean[] forcePathStyle = {true, false};
+        boolean[] payloadSign = {true, false};
+        RequestChecksumCalculation[] checksumValidations = {RequestChecksumCalculation.WHEN_REQUIRED,
+                                                            RequestChecksumCalculation.WHEN_SUPPORTED};
+        for (boolean pathStyle : forcePathStyle) {
+            for (RequestChecksumCalculation checksumValidation : checksumValidations) {
+                for (BucketType bucketType : BucketType.values()) {
+                    for (UploadStreamingRegressionTesting.BodyType bodType : UploadStreamingRegressionTesting.BodyType.values()) {
+                        for (UploadStreamingRegressionTesting.ContentSize cs : UploadStreamingRegressionTesting.ContentSize.values()) {
+                            for (boolean ps : payloadSign) {
+                                FlattenUploadConfig testConfig = new FlattenUploadConfig();
+                                testConfig.setBucketType(bucketType);
+                                testConfig.setForcePathStyle(pathStyle);
+                                testConfig.setRequestChecksumValidation(checksumValidation);
+                                testConfig.setBodyType(bodType);
+                                testConfig.setContentSize(cs);
+                                testConfig.setPayloadSigning(ps);
+                                configs.add(testConfig);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        return configs;
+    }
+
+    public BucketType getBucketType() {
+        return bucketType;
+    }
+
+    public void setBucketType(BucketType bucketType) {
+        this.bucketType = bucketType;
+    }
+
+    public boolean isForcePathStyle() {
+        return forcePathStyle;
+    }
+
+    public void setForcePathStyle(boolean forcePathStyle) {
+        this.forcePathStyle = forcePathStyle;
+    }
+
+    public RequestChecksumCalculation getRequestChecksumValidation() {
+        return requestChecksumValidation;
+    }
+
+    public void setRequestChecksumValidation(RequestChecksumCalculation requestChecksumValidation) {
+        this.requestChecksumValidation = requestChecksumValidation;
+    }
+
+    public UploadStreamingRegressionTesting.BodyType getBodyType() {
+        return bodyType;
+    }
+
+    public void setBodyType(UploadStreamingRegressionTesting.BodyType bodyType) {
+        this.bodyType = bodyType;
+    }
+
+    public UploadStreamingRegressionTesting.ContentSize getContentSize() {
+        return contentSize;
+    }
+
+    public void setContentSize(UploadStreamingRegressionTesting.ContentSize contentSize) {
+        this.contentSize = contentSize;
+    }
+
+    public boolean isPayloadSigning() {
+        return payloadSigning;
+    }
+
+    public void setPayloadSigning(boolean payloadSigning) {
+        this.payloadSigning = payloadSigning;
+    }
+
+    @Override
+    public String toString() {
+        return ToString.builder("FlattenUploadConfig")
+                       .add("bucketType", bucketType)
+                       .add("forcePathStyle", forcePathStyle)
+                       .add("requestChecksumValidation", requestChecksumValidation)
+                       .add("bodyType", bodyType)
+                       .add("contentSize", contentSize)
+                       .add("payloadSigning", payloadSigning)
+                       .build();
+    }
+}