Better checksum support

codebuild/cd/fips-where-available-jar-build.yml

@@ -2,7 +2,6 @@ version: 0.2
 phases:
   install:
     commands:
-      - sudo add-apt-repository ppa:openjdk-r/ppa
       - sudo apt-get update -y
       - sudo apt-get install openjdk-11-jdk-headless maven wget unzip -y -f
   build:

codebuild/cd/release-android-maven-central.yml

@@ -4,7 +4,6 @@ version: 0.2
 phases:
   install:
     commands:
-      - sudo add-apt-repository ppa:openjdk-r/ppa
       - sudo apt-get update -y
 
       # Java 17 for AGP 8.x

codebuild/cd/release-maven-central.yml

@@ -4,7 +4,6 @@ version: 0.2
 phases:
   install:
     commands:
-      - sudo add-apt-repository ppa:openjdk-r/ppa
       - sudo apt-get update -y
       - sudo apt-get install openjdk-11-jdk-headless maven wget unzip -y -f
   pre_build:

codebuild/linux-integration-tests.yml

@@ -9,8 +9,6 @@ env:
 phases:
   install:
     commands:
-      - sudo add-apt-repository ppa:openjdk-r/ppa
-      - sudo add-apt-repository ppa:ubuntu-toolchain-r/test
       - sudo apt-get update -y
       - sudo apt-get install cmake -y
   build:

codebuild/mqtt5-java-canary-test.yml

@@ -15,8 +15,6 @@ env:
 phases:
   install:
     commands:
-      - add-apt-repository ppa:openjdk-r/ppa
-      - add-apt-repository ppa:ubuntu-toolchain-r/test
       - apt-get update -y
       - sudo apt-get install cmake -y
       - git submodule update --init --recursive

codebuild/test-aws-java-sdk-v2.yml

@@ -5,8 +5,6 @@ phases:
     runtime-versions:
       java: "corretto8"
     commands:
-      - add-apt-repository ppa:openjdk-r/ppa
-      - add-apt-repository ppa:ubuntu-toolchain-r/test
       - apt-get update -y
       - apt-get install gcc-7 cmake ninja-build -y
       - git submodule update --init --recursive

src/main/java/software/amazon/awssdk/crt/checksums/XXHash.java

@@ -120,7 +120,7 @@ public void update(int b) {
     }
 
     /**
-     * Update xxhash state with a subrange of input
+     * Update xxhash state with a subrange of input.
      * @param input input to update with
      * @param offset to start update with
      * @param length of data

src/main/java/software/amazon/awssdk/crt/s3/ChecksumAlgorithm.java

@@ -22,7 +22,17 @@ public enum ChecksumAlgorithm {
 
     SHA256(4),
 
-    CRC64NVME(5);
+    CRC64NVME(5),
+
+    SHA512(6),
+
+    XXHASH64(7),
+
+    XXHASH3(8),
+
+    XXHASH128(9),
+
+    MD5(10);
 
     ChecksumAlgorithm(int nativeValue) {
         this.nativeValue = nativeValue;
@@ -49,6 +59,11 @@ private static Map<Integer, ChecksumAlgorithm> buildEnumMapping() {
         enumMapping.put(SHA1.getNativeValue(), SHA1);
         enumMapping.put(SHA256.getNativeValue(), SHA256);
         enumMapping.put(CRC64NVME.getNativeValue(), CRC64NVME);
+        enumMapping.put(SHA512.getNativeValue(), SHA512);
+        enumMapping.put(XXHASH64.getNativeValue(), XXHASH64);
+        enumMapping.put(XXHASH3.getNativeValue(), XXHASH3);
+        enumMapping.put(XXHASH128.getNativeValue(), XXHASH128);
+        enumMapping.put(MD5.getNativeValue(), MD5);
         return enumMapping;
     }
 

src/main/java/software/amazon/awssdk/crt/s3/ChecksumConfig.java

@@ -114,7 +114,7 @@ public boolean getValidateChecksum() {
      * The list of algorithms for user to pick up when validate the checksum. Client
      * will pick up the algorithm from the list with the priority based on
      * performance, and the algorithm sent by server. The priority based on
-     * performance is [CRC64NVME, CRC32C, CRC32, SHA1, SHA256].
+     * performance is [XXHASH128, XXHASH3,CRC64NVME, CRC32C, CRC32, XXHASH64, SHA512, SHA256, SHA1].
      *
      * If the response checksum was validated by client, the result will indicate
      * which algorithm was picked.

src/main/java/software/amazon/awssdk/crt/s3/S3Client.java

@@ -168,6 +168,12 @@ public S3MetaRequest makeMetaRequest(S3MetaRequestOptions options) {
             throw new IllegalArgumentException("S3Client.makeMetaRequest has invalid options; Operation name must be set for MetaRequestType.DEFAULT.");
         }
 
+        if (options.getChecksumConfig() != null && options.getChecksumAlgorithm() == ChecksumAlgorithm.MD5) {
+            Log.log(Log.LogLevel.Error, Log.LogSubject.S3Client,
+                    "S3Client.makeMetaRequest has invalid options; MD5 not supported as checksum algorithm.");
+            throw new IllegalArgumentException("S3Client.makeMetaRequest has invalid options; MD5 not supported as checksum algorithm.");
+        }
+
         S3MetaRequest metaRequest = new S3MetaRequest();
         S3MetaRequestResponseHandlerNativeAdapter responseHandlerNativeAdapter = new S3MetaRequestResponseHandlerNativeAdapter(
                 options.getResponseHandler());

src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java

@@ -307,6 +307,39 @@ public void testS3ClientCreateDestroyHttpProxyEnvironmentVariableSetting() {
         }
     }
 
+    @Test
+    public void testS3ClientCreateMD5MetaRequest() {
+        skipIfAndroid();
+        skipIfNetworkUnavailable();
+
+        S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION)
+                .withComputeContentMd5(true)
+                .withMemoryLimitInBytes(5L * 1024 * 1024 * 1024);
+        try (S3Client client = createS3Client(clientOptions)) {
+            S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() {
+                @Override
+                public int onResponseBody(ByteBuffer bodyBytesIn, long objectRangeStart, long objectRangeEnd) {
+                    return 0;
+                }
+
+                @Override
+                public void onFinished(S3FinishedResponseContext context) {
+                }
+            };
+
+            HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT) };
+            HttpRequest httpRequest = new HttpRequest("GET", PRE_EXIST_1MB_PATH, headers, null);
+
+            S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions()
+                    .withMetaRequestType(MetaRequestType.GET_OBJECT)
+                    .withChecksumAlgorithm(ChecksumAlgorithm.MD5)
+                    .withHttpRequest(httpRequest)
+                    .withResponseHandler(responseHandler);
+
+            Assert.assertThrows(IllegalArgumentException.class, () -> { client.makeMetaRequest(metaRequestOptions); });
+        }
+    }
+
     @Test
     public void testS3Get() {
         skipIfAndroid();
@@ -1360,9 +1393,13 @@ public void onFinished(S3FinishedResponseContext context) {
                 }
             };
             ArrayList<ChecksumAlgorithm> algorList = new ArrayList<ChecksumAlgorithm>();
+            algorList.add(ChecksumAlgorithm.XXHASH128);
+            algorList.add(ChecksumAlgorithm.XXHASH3);
             algorList.add(ChecksumAlgorithm.CRC64NVME);
             algorList.add(ChecksumAlgorithm.CRC32C);
             algorList.add(ChecksumAlgorithm.CRC32);
+            algorList.add(ChecksumAlgorithm.XXHASH64);
+            algorList.add(ChecksumAlgorithm.SHA512);
             algorList.add(ChecksumAlgorithm.SHA1);
             algorList.add(ChecksumAlgorithm.SHA256);
             ChecksumConfig validateChecksumConfig = new ChecksumConfig().withValidateChecksum(true)
@@ -1381,13 +1418,29 @@ public void onFinished(S3FinishedResponseContext context) {
     }
 
     @Test
-    public void testS3PutTrailerChecksums() throws Exception {
+    public void testS3PutTrailerChecksumsCRC64() throws Exception {
         skipIfAndroid();
         skipIfNetworkUnavailable();
         Assume.assumeTrue(hasAwsCredentials());
         testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.CRC64NVME, ChecksumLocation.TRAILER, false, false);
     }
 
+    @Test
+    public void testS3PutTrailerChecksumsSHA512() throws Exception {
+        skipIfAndroid();
+        skipIfNetworkUnavailable();
+        Assume.assumeTrue(hasAwsCredentials());
+        testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.SHA512, ChecksumLocation.TRAILER, false, false);
+    }
+
+    @Test
+    public void testS3PutTrailerChecksumsXXHASH3_64() throws Exception {
+        skipIfAndroid();
+        skipIfNetworkUnavailable();
+        Assume.assumeTrue(hasAwsCredentials());
+        testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.XXHASH3, ChecksumLocation.TRAILER, false, false);
+    }
+
     @Test
     public void testS3PutHeaderChecksums() throws Exception {
         skipIfAndroid();