Skip to content

Commit 887313c

Browse files
zoewanggzoewangg
committed
Add additional checksum algorithms (#6742)
* Add additional checksum algorithms * Update checksum order based on perf test and update benchmark test code * Address feedback * Fix codegen tests * Fix codegen generated classes tests
1 parent 837715e commit 887313c

40 files changed

Lines changed: 1065 additions & 358 deletions

File tree

‎codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/traits/HttpChecksumTrait.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,16 @@ public class HttpChecksumTrait {
4141

4242
private static final Map<String, Integer> CHECKSUM_ALGORITHM_PRIORITY =
4343
ImmutableMap.<String, Integer>builder()
44-
.put(DefaultChecksumAlgorithm.CRC32C.algorithmId(), 1)
45-
.put(DefaultChecksumAlgorithm.CRC32.algorithmId(), 2)
44+
.put(DefaultChecksumAlgorithm.XXHASH3.algorithmId(), 1)
45+
.put(DefaultChecksumAlgorithm.XXHASH128.algorithmId(), 2)
4646
.put(DefaultChecksumAlgorithm.CRC64NVME.algorithmId(), 3)
47-
.put(DefaultChecksumAlgorithm.SHA1.algorithmId(), 4)
48-
.put(DefaultChecksumAlgorithm.SHA256.algorithmId(), 5)
47+
.put(DefaultChecksumAlgorithm.CRC32C.algorithmId(), 4)
48+
.put(DefaultChecksumAlgorithm.CRC32.algorithmId(), 5)
49+
.put(DefaultChecksumAlgorithm.XXHASH64.algorithmId(), 6)
50+
.put(DefaultChecksumAlgorithm.SHA512.algorithmId(), 7)
51+
.put(DefaultChecksumAlgorithm.SHA256.algorithmId(), 8)
52+
.put(DefaultChecksumAlgorithm.SHA1.algorithmId(), 9)
53+
.put(DefaultChecksumAlgorithm.MD5.algorithmId(), 10)
4954
.build();
5055

5156
private HttpChecksumTrait() {

‎codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/query/service-2.json‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,7 @@
189189
"output":{"shape":"ChecksumStructureWithStreaming"},
190190
"httpChecksum" : {
191191
"requestValidationModeMember": "ChecksumMode",
192-
"responseAlgorithms": ["CRC32C", "CRC32", "SHA1", "SHA256"]
192+
"responseAlgorithms": ["CRC32C", "CRC32", "CRC64NVME", "MD5", "SHA1", "SHA256", "SHA512", "XXHASH64", "XXHASH3", "XXHASH128"]
193193
}
194194
},
195195
"GetOperationWithChecksum":{
@@ -399,8 +399,14 @@
399399
"enum":[
400400
"CRC32",
401401
"CRC32C",
402+
"CRC64NVME",
403+
"MD5",
402404
"SHA1",
403-
"SHA256"
405+
"SHA256",
406+
"SHA512",
407+
"XXHASH64",
408+
"XXHASH3",
409+
"XXHASH128"
404410
]
405411
},
406412
"ChecksumMode":{

‎codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/rest-json/service-2.json‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@
210210
"output":{"shape":"ChecksumStructureWithStreaming"},
211211
"httpChecksum" : {
212212
"requestValidationModeMember": "ChecksumMode",
213-
"responseAlgorithms": ["CRC32C", "CRC32", "SHA1", "SHA256"],
213+
"responseAlgorithms": ["CRC32C", "CRC32", "CRC64NVME", "MD5", "SHA1", "SHA256", "SHA512", "XXHASH64", "XXHASH3", "XXHASH128"],
214214
"requestAlgorithmMember": "ChecksumAlgorithm"
215215
}
216216
},
@@ -564,8 +564,14 @@
564564
"enum":[
565565
"CRC32",
566566
"CRC32C",
567+
"CRC64NVME",
568+
"MD5",
567569
"SHA1",
568-
"SHA256"
570+
"SHA256",
571+
"SHA512",
572+
"XXHASH64",
573+
"XXHASH3",
574+
"XXHASH128"
569575
]
570576
},
571577
"ChecksumMode":{

‎codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/c2j/xml/service-2.json‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@
134134
"output":{"shape":"ChecksumStructureWithStreaming"},
135135
"httpChecksum" : {
136136
"requestValidationModeMember": "ChecksumMode",
137-
"responseAlgorithms": ["CRC32C", "CRC32", "SHA1", "SHA256"]
137+
"responseAlgorithms": ["CRC32C", "CRC32", "CRC64NVME", "MD5", "SHA1", "SHA256", "SHA512", "XXHASH64", "XXHASH3", "XXHASH128"]
138138
}
139139
},
140140
"GetOperationWithChecksum":{
@@ -264,8 +264,14 @@
264264
"enum":[
265265
"CRC32",
266266
"CRC32C",
267+
"CRC64NVME",
268+
"MD5",
267269
"SHA1",
268-
"SHA256"
270+
"SHA256",
271+
"SHA512",
272+
"XXHASH64",
273+
"XXHASH3",
274+
"XXHASH128"
269275
]
270276
},
271277
"ChecksumMode":{

‎codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-json-async-client-class.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1251,9 +1251,12 @@ public <ReturnT> CompletableFuture<ReturnT> putOperationWithChecksum(
12511251
.requestAlgorithm(putOperationWithChecksumRequest.checksumAlgorithmAsString())
12521252
.requestAlgorithmHeader("x-amz-sdk-checksum-algorithm")
12531253
.requestValidationMode(putOperationWithChecksumRequest.checksumModeAsString())
1254-
.responseAlgorithmsV2(DefaultChecksumAlgorithm.CRC32C,
1255-
DefaultChecksumAlgorithm.CRC32, DefaultChecksumAlgorithm.SHA1,
1256-
DefaultChecksumAlgorithm.SHA256).build())
1254+
.responseAlgorithmsV2(DefaultChecksumAlgorithm.XXHASH3,
1255+
DefaultChecksumAlgorithm.XXHASH128, DefaultChecksumAlgorithm.CRC64NVME,
1256+
DefaultChecksumAlgorithm.CRC32C, DefaultChecksumAlgorithm.CRC32,
1257+
DefaultChecksumAlgorithm.XXHASH64, DefaultChecksumAlgorithm.SHA512,
1258+
DefaultChecksumAlgorithm.SHA256, DefaultChecksumAlgorithm.SHA1,
1259+
DefaultChecksumAlgorithm.MD5).build())
12571260
.withAsyncResponseTransformer(asyncResponseTransformer).withInput(putOperationWithChecksumRequest),
12581261
asyncResponseTransformer);
12591262
AsyncResponseTransformer<PutOperationWithChecksumResponse, ReturnT> finalAsyncResponseTransformer = asyncResponseTransformer;

‎codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-json-client-class.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -785,9 +785,12 @@ public <ReturnT> ReturnT putOperationWithChecksum(PutOperationWithChecksumReques
785785
.requestAlgorithm(putOperationWithChecksumRequest.checksumAlgorithmAsString())
786786
.requestAlgorithmHeader("x-amz-sdk-checksum-algorithm")
787787
.requestValidationMode(putOperationWithChecksumRequest.checksumModeAsString())
788-
.responseAlgorithmsV2(DefaultChecksumAlgorithm.CRC32C,
789-
DefaultChecksumAlgorithm.CRC32, DefaultChecksumAlgorithm.SHA1,
790-
DefaultChecksumAlgorithm.SHA256).build())
788+
.responseAlgorithmsV2(DefaultChecksumAlgorithm.XXHASH3,
789+
DefaultChecksumAlgorithm.XXHASH128, DefaultChecksumAlgorithm.CRC64NVME,
790+
DefaultChecksumAlgorithm.CRC32C, DefaultChecksumAlgorithm.CRC32,
791+
DefaultChecksumAlgorithm.XXHASH64, DefaultChecksumAlgorithm.SHA512,
792+
DefaultChecksumAlgorithm.SHA256, DefaultChecksumAlgorithm.SHA1,
793+
DefaultChecksumAlgorithm.MD5).build())
791794
.withResponseTransformer(responseTransformer)
792795
.withRequestBody(requestBody)
793796
.withMarshaller(

‎codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-query-async-client-class.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -973,9 +973,12 @@ public <ReturnT> CompletableFuture<ReturnT> putOperationWithChecksum(
973973
.requestChecksumRequired(false)
974974
.isRequestStreaming(true)
975975
.requestValidationMode(putOperationWithChecksumRequest.checksumModeAsString())
976-
.responseAlgorithmsV2(DefaultChecksumAlgorithm.CRC32C,
977-
DefaultChecksumAlgorithm.CRC32, DefaultChecksumAlgorithm.SHA1,
978-
DefaultChecksumAlgorithm.SHA256).build())
976+
.responseAlgorithmsV2(DefaultChecksumAlgorithm.XXHASH3,
977+
DefaultChecksumAlgorithm.XXHASH128, DefaultChecksumAlgorithm.CRC64NVME,
978+
DefaultChecksumAlgorithm.CRC32C, DefaultChecksumAlgorithm.CRC32,
979+
DefaultChecksumAlgorithm.XXHASH64, DefaultChecksumAlgorithm.SHA512,
980+
DefaultChecksumAlgorithm.SHA256, DefaultChecksumAlgorithm.SHA1,
981+
DefaultChecksumAlgorithm.MD5).build())
979982
.withAsyncResponseTransformer(asyncResponseTransformer).withAsyncRequestBody(requestBody)
980983
.withInput(putOperationWithChecksumRequest), asyncResponseTransformer);
981984
CompletableFuture<ReturnT> whenCompleteFuture = null;

‎codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-query-client-class.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -835,9 +835,12 @@ public <ReturnT> ReturnT putOperationWithChecksum(PutOperationWithChecksumReques
835835
.requestChecksumRequired(false)
836836
.isRequestStreaming(true)
837837
.requestValidationMode(putOperationWithChecksumRequest.checksumModeAsString())
838-
.responseAlgorithmsV2(DefaultChecksumAlgorithm.CRC32C,
839-
DefaultChecksumAlgorithm.CRC32, DefaultChecksumAlgorithm.SHA1,
840-
DefaultChecksumAlgorithm.SHA256).build())
838+
.responseAlgorithmsV2(DefaultChecksumAlgorithm.XXHASH3,
839+
DefaultChecksumAlgorithm.XXHASH128, DefaultChecksumAlgorithm.CRC64NVME,
840+
DefaultChecksumAlgorithm.CRC32C, DefaultChecksumAlgorithm.CRC32,
841+
DefaultChecksumAlgorithm.XXHASH64, DefaultChecksumAlgorithm.SHA512,
842+
DefaultChecksumAlgorithm.SHA256, DefaultChecksumAlgorithm.SHA1,
843+
DefaultChecksumAlgorithm.MD5).build())
841844
.withResponseTransformer(responseTransformer)
842845
.withRequestBody(requestBody)
843846
.withMarshaller(

‎codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-xml-async-client-class.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -698,9 +698,12 @@ public <ReturnT> CompletableFuture<ReturnT> putOperationWithChecksum(
698698
.requestChecksumRequired(false)
699699
.isRequestStreaming(true)
700700
.requestValidationMode(putOperationWithChecksumRequest.checksumModeAsString())
701-
.responseAlgorithmsV2(DefaultChecksumAlgorithm.CRC32C,
702-
DefaultChecksumAlgorithm.CRC32, DefaultChecksumAlgorithm.SHA1,
703-
DefaultChecksumAlgorithm.SHA256).build())
701+
.responseAlgorithmsV2(DefaultChecksumAlgorithm.XXHASH3,
702+
DefaultChecksumAlgorithm.XXHASH128, DefaultChecksumAlgorithm.CRC64NVME,
703+
DefaultChecksumAlgorithm.CRC32C, DefaultChecksumAlgorithm.CRC32,
704+
DefaultChecksumAlgorithm.XXHASH64, DefaultChecksumAlgorithm.SHA512,
705+
DefaultChecksumAlgorithm.SHA256, DefaultChecksumAlgorithm.SHA1,
706+
DefaultChecksumAlgorithm.MD5).build())
704707
.withAsyncResponseTransformer(asyncResponseTransformer).withAsyncRequestBody(requestBody)
705708
.withInput(putOperationWithChecksumRequest), asyncResponseTransformer);
706709
CompletableFuture<ReturnT> whenCompleteFuture = null;

‎codegen/src/test/resources/software/amazon/awssdk/codegen/poet/client/test-xml-client-class.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -516,9 +516,12 @@ public <ReturnT> ReturnT putOperationWithChecksum(PutOperationWithChecksumReques
516516
.requestChecksumRequired(false)
517517
.isRequestStreaming(true)
518518
.requestValidationMode(putOperationWithChecksumRequest.checksumModeAsString())
519-
.responseAlgorithmsV2(DefaultChecksumAlgorithm.CRC32C,
520-
DefaultChecksumAlgorithm.CRC32, DefaultChecksumAlgorithm.SHA1,
521-
DefaultChecksumAlgorithm.SHA256).build())
519+
.responseAlgorithmsV2(DefaultChecksumAlgorithm.XXHASH3,
520+
DefaultChecksumAlgorithm.XXHASH128, DefaultChecksumAlgorithm.CRC64NVME,
521+
DefaultChecksumAlgorithm.CRC32C, DefaultChecksumAlgorithm.CRC32,
522+
DefaultChecksumAlgorithm.XXHASH64, DefaultChecksumAlgorithm.SHA512,
523+
DefaultChecksumAlgorithm.SHA256, DefaultChecksumAlgorithm.SHA1,
524+
DefaultChecksumAlgorithm.MD5).build())
522525
.withResponseTransformer(responseTransformer)
523526
.withRequestBody(requestBody)
524527
.withMarshaller(

0 commit comments

Comments
 (0)