Add support for sha512, xxhash algorithms, and MD5 for httpchecksums (boto#3637)

* Add support for sha512, CRT supported xxhash algorithms, and MD5 when modeled by a service

Co-authored-by: jonathan343 <43360731+jonathan343@users.noreply.github.com>

Author: SamRemis

.changes/next-release/enhancement-httpchecksums.json

@@ -0,0 +1,6 @@
+{
+  "type": "enhancement",
+  "category": "``checksums``",
+  "description": "Added support for the SHA512 checksum algorithm. When the optional AWS CRT (``awscrt``) dependency is installed, support is also enabled for the XXHASH64, XXHASH3, and XXHASH128 checksum algorithms. Also added pass-through support for customer-provided MD5 checksum headers (without SDK-side MD5 calculation or validation)."
+}
+

botocore/httpchecksum.py

@@ -23,7 +23,7 @@
 import io
 import logging
 from binascii import crc32
-from hashlib import sha1, sha256
+from hashlib import sha1, sha256, sha512
 
 from botocore.compat import HAS_CRT, has_minimum_crt_version, urlparse
 from botocore.exceptions import (
@@ -37,6 +37,7 @@
 from botocore.utils import (
     conditionally_calculate_md5,
     determine_content_length,
+    get_checksum_header_algorithms,
     has_checksum_header,
 )
 
@@ -127,6 +128,42 @@ def digest(self):
         return self._int_crc64nvme.to_bytes(8, byteorder="big")
 
 
+class CrtXxhash64Checksum(BaseChecksum):
+    # Note: This class is only used if the CRT is available
+    def __init__(self):
+        self._xxhash = crt_checksums.XXHash.new_xxhash64()
+
+    def update(self, chunk):
+        self._xxhash.update(chunk)
+
+    def digest(self):
+        return self._xxhash.finalize()
+
+
+class CrtXxhash3Checksum(BaseChecksum):
+    # Note: This class is only used if the CRT is available
+    def __init__(self):
+        self._xxhash = crt_checksums.XXHash.new_xxhash3_64()
+
+    def update(self, chunk):
+        self._xxhash.update(chunk)
+
+    def digest(self):
+        return self._xxhash.finalize()
+
+
+class CrtXxhash128Checksum(BaseChecksum):
+    # Note: This class is only used if the CRT is available
+    def __init__(self):
+        self._xxhash = crt_checksums.XXHash.new_xxhash3_128()
+
+    def update(self, chunk):
+        self._xxhash.update(chunk)
+
+    def digest(self):
+        return self._xxhash.finalize()
+
+
 class Sha1Checksum(BaseChecksum):
     def __init__(self):
         self._checksum = sha1()
@@ -149,6 +186,17 @@ def digest(self):
         return self._checksum.digest()
 
 
+class Sha512Checksum(BaseChecksum):
+    def __init__(self):
+        self._checksum = sha512()
+
+    def update(self, chunk):
+        self._checksum.update(chunk)
+
+    def digest(self):
+        return self._checksum.digest()
+
+
 class AwsChunkedWrapper:
     _DEFAULT_CHUNK_SIZE = 1024 * 1024
 
@@ -267,6 +315,7 @@ def _validate_checksum(self):
 def resolve_checksum_context(request, operation_model, params):
     resolve_request_checksum_algorithm(request, operation_model, params)
     resolve_response_checksum_algorithms(request, operation_model, params)
+    _register_checksum_feature_ids(request)
 
 
 def resolve_request_checksum_algorithm(
@@ -398,7 +447,6 @@ def _apply_request_header_checksum(request):
     checksum_cls = _CHECKSUM_CLS.get(algorithm["algorithm"])
     digest = checksum_cls().handle(request["body"])
     request["headers"][location_name] = digest
-    _register_checksum_algorithm_feature_id(algorithm)
 
 
 def _apply_request_trailer_checksum(request):
@@ -422,7 +470,6 @@ def _apply_request_trailer_checksum(request):
     else:
         headers["Content-Encoding"] = "aws-chunked"
     headers["X-Amz-Trailer"] = location_name
-    _register_checksum_algorithm_feature_id(algorithm)
 
     content_length = determine_content_length(body)
     if content_length is not None:
@@ -446,8 +493,22 @@ def _apply_request_trailer_checksum(request):
     )
 
 
+def _register_checksum_feature_ids(request):
+    """Register feature IDs for checksum algorithms used in the request."""
+    if algorithm_list := get_checksum_header_algorithms(request):
+        for algorithm_name in algorithm_list:
+            _register_checksum_algorithm_feature_id(algorithm_name)
+        return
+    # If no checksum header exists yet, check the resolved context for
+    # an algorithm that will be applied later by apply_request_checksum.
+    checksum_context = request.get("context", {}).get("checksum", {})
+    algorithm = checksum_context.get("request_algorithm")
+    if algorithm and isinstance(algorithm, dict):
+        _register_checksum_algorithm_feature_id(algorithm["algorithm"])
+
+
 def _register_checksum_algorithm_feature_id(algorithm):
-    checksum_algorithm_name = algorithm["algorithm"].upper()
+    checksum_algorithm_name = algorithm.upper()
     if checksum_algorithm_name == "CRC64NVME":
         checksum_algorithm_name = "CRC64"
     checksum_algorithm_name_feature_id = (
@@ -552,8 +613,16 @@ def _handle_bytes_response(http_response, response, algorithm):
     "crc32": Crc32Checksum,
     "sha1": Sha1Checksum,
     "sha256": Sha256Checksum,
+    "sha512": Sha512Checksum,
 }
-_CRT_CHECKSUM_ALGORITHMS = ["crc32", "crc32c", "crc64nvme"]
+_CRT_CHECKSUM_ALGORITHMS = [
+    "crc32",
+    "crc32c",
+    "crc64nvme",
+    "xxhash64",
+    "xxhash3",
+    "xxhash128",
+]
 if HAS_CRT:
     # Use CRT checksum implementations if available
     _CRT_CHECKSUM_CLS = {
@@ -565,10 +634,25 @@ def _handle_bytes_response(http_response, response, algorithm):
         # CRC64NVME support wasn't officially added until 0.23.4
         _CRT_CHECKSUM_CLS["crc64nvme"] = CrtCrc64NvmeChecksum
 
+    if has_minimum_crt_version((0, 31, 2)):
+        _CRT_CHECKSUM_CLS["xxhash64"] = CrtXxhash64Checksum
+        _CRT_CHECKSUM_CLS["xxhash3"] = CrtXxhash3Checksum
+        _CRT_CHECKSUM_CLS["xxhash128"] = CrtXxhash128Checksum
+
     _CHECKSUM_CLS.update(_CRT_CHECKSUM_CLS)
-    # Validate this list isn't out of sync with _CRT_CHECKSUM_CLS keys
+    # Validate this list isn't out of sync with _CRT_CHECKSUM_ALGORITHMS keys
     assert all(
         name in _CRT_CHECKSUM_ALGORITHMS for name in _CRT_CHECKSUM_CLS.keys()
     )
 _SUPPORTED_CHECKSUM_ALGORITHMS = list(_CHECKSUM_CLS.keys())
-_ALGORITHMS_PRIORITY_LIST = ['crc64nvme', 'crc32c', 'crc32', 'sha1', 'sha256']
+_ALGORITHMS_PRIORITY_LIST = [
+    'xxhash128',
+    'xxhash3',
+    'crc64nvme',
+    'xxhash64',
+    'crc32c',
+    'crc32',
+    'sha1',
+    'sha256',
+    'sha512',
+]

botocore/useragent.py

@@ -100,6 +100,11 @@
     'CLI_V1_TO_V2_MIGRATION_DEBUG_MODE': '-',
     'CREDENTIALS_PROFILE_LOGIN': 'AC',
     'CREDENTIALS_LOGIN': 'AD',
+    'FLEXIBLE_CHECKSUMS_REQ_MD5': 'AE',
+    'FLEXIBLE_CHECKSUMS_REQ_SHA512': 'AF',
+    'FLEXIBLE_CHECKSUMS_REQ_XXHASH3': 'AG',
+    'FLEXIBLE_CHECKSUMS_REQ_XXHASH64': 'AH',
+    'FLEXIBLE_CHECKSUMS_REQ_XXHASH128': 'AI',
 }
 
 

botocore/utils.py

@@ -3301,22 +3301,34 @@ def _is_s3express_request(params):
     return endpoint_properties.get('backend') == 'S3Express'
 
 
-def has_checksum_header(params):
+def get_checksum_header_algorithms(params):
     """
-    Checks if a header starting with "x-amz-checksum-" is provided in a request.
+    Returns the a list of algorithm name if a headers starting with "x-amz-checksum-"
+    are provided in a request, otherwise returns an empty list.
 
     This function is considered private and subject to abrupt breaking changes or
     removal without prior announcement. Please do not use it directly.
     """
     headers = params['headers']
+    checksum_headers = []
 
     # If a header matching the x-amz-checksum-* pattern is present, we
-    # assume a checksum has already been provided by the user.
+    # extract and return the algorithm name.
     for header in headers:
-        if CHECKSUM_HEADER_PATTERN.match(header):
-            return True
+        match = CHECKSUM_HEADER_PATTERN.match(header)
+        if match:
+            checksum_headers.append(match.group(1))
+    return checksum_headers
 
-    return False
+
+def has_checksum_header(params):
+    """
+    Checks if a header starting with "x-amz-checksum-" is provided in a request.
+
+    This function is considered private and subject to abrupt breaking changes or
+    removal without prior announcement. Please do not use it directly.
+    """
+    return bool(get_checksum_header_algorithms(params))
 
 
 def conditionally_calculate_checksum(params, **kwargs):