Add support for additional HTTP checksum algorithms and user-specified MD5s (aws#10099)

Author: ashovlin

.changes/next-release/enhancement-checksums-42602.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "``checksums``",
+  "description": "Add support for SHA512, XXHASH64, XXHASH3, and XXHASH128 HTTP checksum algorithms. Also added pass-through support for user-provided MD5 checksum headers (without client-side MD5 calculation or validation)."
+}

awscli/botocore/httpchecksum.py

@@ -23,15 +23,19 @@
 import io
 import logging
 from binascii import crc32
-from hashlib import sha1, sha256
+from hashlib import sha1, sha256, sha512
 
 from awscrt import checksums as crt_checksums
 from botocore.compat import urlparse
 from botocore.exceptions import AwsChunkedWrapperError, FlexibleChecksumError
 from botocore.model import StructureShape
 from botocore.response import StreamingBody
 from botocore.useragent import register_feature_id
-from botocore.utils import determine_content_length, has_checksum_header
+from botocore.utils import (
+    determine_content_length,
+    get_checksum_algorithm_headers,
+    has_checksum_header,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -115,6 +119,42 @@ def digest(self):
         return self._int_crc64nvme.to_bytes(8, byteorder="big")
 
 
+class CrtXxhash64Checksum(BaseChecksum):
+    # Note: This class is only used if the CRT is available
+    def __init__(self):
+        self._xxhash = crt_checksums.XXHash.new_xxhash64()
+
+    def update(self, chunk):
+        self._xxhash.update(chunk)
+
+    def digest(self):
+        return self._xxhash.finalize()
+
+
+class CrtXxhash3Checksum(BaseChecksum):
+    # Note: This class is only used if the CRT is available
+    def __init__(self):
+        self._xxhash = crt_checksums.XXHash.new_xxhash3_64()
+
+    def update(self, chunk):
+        self._xxhash.update(chunk)
+
+    def digest(self):
+        return self._xxhash.finalize()
+
+
+class CrtXxhash128Checksum(BaseChecksum):
+    # Note: This class is only used if the CRT is available
+    def __init__(self):
+        self._xxhash = crt_checksums.XXHash.new_xxhash3_128()
+
+    def update(self, chunk):
+        self._xxhash.update(chunk)
+
+    def digest(self):
+        return self._xxhash.finalize()
+
+
 class Sha1Checksum(BaseChecksum):
     def __init__(self):
         self._checksum = sha1()
@@ -137,6 +177,17 @@ def digest(self):
         return self._checksum.digest()
 
 
+class Sha512Checksum(BaseChecksum):
+    def __init__(self):
+        self._checksum = sha512()
+
+    def update(self, chunk):
+        self._checksum.update(chunk)
+
+    def digest(self):
+        return self._checksum.digest()
+
+
 class AwsChunkedWrapper:
     _DEFAULT_CHUNK_SIZE = 1024 * 1024
 
@@ -241,6 +292,7 @@ def _validate_checksum(self):
 def resolve_checksum_context(request, operation_model, params):
     resolve_request_checksum_algorithm(request, operation_model, params)
     resolve_response_checksum_algorithms(request, operation_model, params)
+    _register_checksum_feature_ids(request)
 
 
 def resolve_request_checksum_algorithm(
@@ -361,7 +413,6 @@ def _apply_request_header_checksum(request):
     checksum_cls = _CHECKSUM_CLS.get(algorithm["algorithm"])
     digest = checksum_cls().handle(request["body"])
     request["headers"][location_name] = digest
-    _register_checksum_algorithm_feature_id(algorithm)
 
 
 def _apply_request_trailer_checksum(request):
@@ -385,7 +436,6 @@ def _apply_request_trailer_checksum(request):
     else:
         headers["Content-Encoding"] = "aws-chunked"
     headers["X-Amz-Trailer"] = location_name
-    _register_checksum_algorithm_feature_id(algorithm)
 
     content_length = determine_content_length(body)
     if content_length is not None:
@@ -409,8 +459,29 @@ def _apply_request_trailer_checksum(request):
     )
 
 
+def _register_checksum_feature_ids(request):
+    """Register feature IDs for checksum algorithms used in the request."""
+    if algorithm_headers := get_checksum_algorithm_headers(request):
+        for header in algorithm_headers:
+            header = header.upper()
+            if header not in (
+                "X-AMZ-CHECKSUM-ALGORITHM",
+                "X-AMZ-CHECKSUM-MODE",
+                "X-AMZ-CHECKSUM-TYPE",
+            ):
+                algorithm_name = header.removeprefix("X-AMZ-CHECKSUM-")
+                _register_checksum_algorithm_feature_id(algorithm_name)
+        return
+    # If no checksum header exists yet, check the resolved context for
+    # an algorithm that will be applied later by apply_request_checksum.
+    checksum_context = request.get("context", {}).get("checksum", {})
+    algorithm = checksum_context.get("request_algorithm")
+    if algorithm and isinstance(algorithm, dict):
+        _register_checksum_algorithm_feature_id(algorithm["algorithm"])
+
+
 def _register_checksum_algorithm_feature_id(algorithm):
-    checksum_algorithm_name = algorithm["algorithm"].upper()
+    checksum_algorithm_name = algorithm.upper()
     if checksum_algorithm_name == "CRC64NVME":
         checksum_algorithm_name = "CRC64"
     checksum_algorithm_name_feature_id = (
@@ -514,8 +585,22 @@ def _handle_bytes_response(http_response, response, algorithm):
     "crc32": CrtCrc32Checksum,
     "sha1": Sha1Checksum,
     "sha256": Sha256Checksum,
+    'sha512': Sha512Checksum,
+    'xxhash64': CrtXxhash64Checksum,
+    'xxhash3': CrtXxhash3Checksum,
+    'xxhash128': CrtXxhash128Checksum,
 }
 
 
 _SUPPORTED_CHECKSUM_ALGORITHMS = list(_CHECKSUM_CLS.keys())
-_ALGORITHMS_PRIORITY_LIST = ['crc64nvme', 'crc32c', 'crc32', 'sha1', 'sha256']
+_ALGORITHMS_PRIORITY_LIST = [
+    'xxhash128',
+    'xxhash3',
+    'crc64nvme',
+    'xxhash64',
+    'crc32c',
+    'crc32',
+    'sha1',
+    'sha256',
+    'sha512',
+]

awscli/botocore/useragent.py

@@ -99,6 +99,11 @@
     'LOGIN_CROSS_DEVICE': 'AB',
     'CREDENTIALS_PROFILE_LOGIN': 'AC',
     'CREDENTIALS_LOGIN': 'AD',
+    'FLEXIBLE_CHECKSUMS_REQ_MD5': 'AE',
+    'FLEXIBLE_CHECKSUMS_REQ_SHA512': 'AF',
+    'FLEXIBLE_CHECKSUMS_REQ_XXHASH3': 'AG',
+    'FLEXIBLE_CHECKSUMS_REQ_XXHASH64': 'AH',
+    'FLEXIBLE_CHECKSUMS_REQ_XXHASH128': 'AI',
 }
 
 

awscli/botocore/utils.py

@@ -3119,22 +3119,31 @@ def _is_s3express_request(params):
     return endpoint_properties.get('backend') == 'S3Express'
 
 
-def has_checksum_header(params):
+def get_checksum_algorithm_headers(params):
     """
-    Checks if a header starting with "x-amz-checksum-" is provided in a request.
-
-    This function is considered private and subject to abrupt breaking changes or
-    removal without prior announcement. Please do not use it directly.
+    Returns the list of header names from the request which start with
+    "x-amz-checksum-", otherwise returns an empty list.
     """
     headers = params['headers']
+    checksum_headers = []
 
     # If a header matching the x-amz-checksum-* pattern is present, we
-    # assume a checksum has already been provided by the user.
+    # extract and return the algorithm name.
     for header in headers:
-        if CHECKSUM_HEADER_PATTERN.match(header):
-            return True
+        match = CHECKSUM_HEADER_PATTERN.match(header)
+        if match:
+            checksum_headers.append(header)
+    return checksum_headers
 
-    return False
+
+def has_checksum_header(params):
+    """
+    Checks if a header starting with "x-amz-checksum-" is provided in a request.
+
+    This function is considered private and subject to abrupt breaking changes or
+    removal without prior announcement. Please do not use it directly.
+    """
+    return bool(get_checksum_algorithm_headers(params))
 
 
 def conditionally_calculate_checksum(params, **kwargs):