[Storage] Add support for CRC64 content validation (#47001)

* [Storage] [STG 102] Create File with Data (#44901)

* [Storage][102] CRC64 content validation - part 1 (#45096)

* [Storage][102] CRC64 content validation - part 2 (#45567)

* [Storage][102] CRC64 content validation - part 3 (#45861)

* [Storage][102] CRC64 content validation - part 4 (#45949)

* [Storage] [STG 102] Added Support for IPv6 Accounts + Data Lake Tag Tests (#45766)

* Fixed test collection bug

* [Storage][102] CRC64 content validation - part 5 - datalake (#46034)

* [Storage][102] CRC64 content validation - part 6 - file-share (#46262)

* [Storage][102] CRC64 content validation - part 7 - record tests (#46461)

* [Storage] Simplify Encoder seek, fix SM streaming retry (#46564)

* [Storage] Cleanup and prepare content validation for merge to main (#46971)

* Fix changelogs after merge

* Tools black, Copilot feedback

---------

Co-authored-by: Peter Wu <162184229+weirongw23-msft@users.noreply.github.com>
Co-authored-by: Peter Wu <weirongwu@microsoft.com>

Author: jalauzon-msft

eng/tools/azure-sdk-tools/devtools_testutils/storage/__init__.py

@@ -1,9 +1,12 @@
 from .api_version_policy import ApiVersionAssertPolicy
+from .decorators import GenericTestProxyParametrize1, GenericTestProxyParametrize2
 from .service_versions import service_version_map, ServiceVersion, is_version_before
 from .testcase import StorageRecordedTestCase, LogCaptured
 
 __all__ = [
     "ApiVersionAssertPolicy",
+    "GenericTestProxyParametrize1",
+    "GenericTestProxyParametrize2",
     "service_version_map",
     "StorageRecordedTestCase",
     "ServiceVersion",

eng/tools/azure-sdk-tools/devtools_testutils/storage/aio/__init__.py

@@ -1,3 +1,4 @@
 from .asynctestcase import AsyncStorageRecordedTestCase
+from .asyncdecorators import GenericTestProxyParametrize1, GenericTestProxyParametrize2
 
-__all__ = ["AsyncStorageRecordedTestCase"]
+__all__ = ["AsyncStorageRecordedTestCase", "GenericTestProxyParametrize1", "GenericTestProxyParametrize2"]

eng/tools/azure-sdk-tools/devtools_testutils/storage/aio/asyncdecorators.py

@@ -0,0 +1,21 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+
+class GenericTestProxyParametrize1:
+    def __call__(self, fn):
+        async def _wrapper(test_class, a, **kwargs):
+            await fn(test_class, a, **kwargs)
+
+        return _wrapper
+
+
+class GenericTestProxyParametrize2:
+    def __call__(self, fn):
+        async def _wrapper(test_class, a, b, **kwargs):
+            await fn(test_class, a, b, **kwargs)
+
+        return _wrapper

eng/tools/azure-sdk-tools/devtools_testutils/storage/decorators.py

@@ -0,0 +1,21 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+
+class GenericTestProxyParametrize1:
+    def __call__(self, fn):
+        def _wrapper(test_class, a, **kwargs):
+            return fn(test_class, a, **kwargs)
+
+        return _wrapper
+
+
+class GenericTestProxyParametrize2:
+    def __call__(self, fn):
+        def _wrapper(test_class, a, b, **kwargs):
+            return fn(test_class, a, b, **kwargs)
+
+        return _wrapper

sdk/storage/azure-storage-blob/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/storage/azure-storage-blob",
-  "Tag": "python/storage/azure-storage-blob_28cfcca089"
+  "Tag": "python/storage/azure-storage-blob_b09e37b521"
 }

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py

@@ -65,6 +65,7 @@
 from ._quick_query_helper import BlobQueryReader
 from ._shared.base_client import parse_connection_str, StorageAccountHostsMixin, TransportWrapper
 from ._shared.response_handlers import process_storage_error, return_response_headers
+from ._shared.validation import is_crc64_validation, parse_validation_option
 from ._serialize import (
     get_access_conditions,
     get_api_version,
@@ -505,15 +506,11 @@ def upload_blob(
         :keyword ~azure.storage.blob.ContentSettings content_settings:
             ContentSettings object used to set blob properties. Used to set content type, encoding,
             language, disposition, md5, and cache control.
-        :keyword bool validate_content:
-            If true, calculates an MD5 hash for each chunk of the blob. The storage
-            service checks the hash of the content that has arrived with the hash
-            that was sent. This is primarily valuable for detecting bitflips on
-            the wire if using http instead of https, as https (the default), will
-            already validate. Note that this MD5 hash is not stored with the
-            blob. Also note that if enabled, the memory-efficient upload algorithm
-            will not be used because computing the MD5 hash requires buffering
-            entire blocks, and doing so defeats the purpose of the memory-efficient algorithm.
+        :keyword validate_content:
+            Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
+            Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
+            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+        :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword lease:
             Required if the blob has an active lease. If specified, upload_blob only succeeds if the
             blob's lease is active and matches this ID. Value can be a BlobLeaseClient object
@@ -616,6 +613,9 @@ def upload_blob(
             raise ValueError("Encryption required but no key was provided.")
         if kwargs.get('cpk') and self.scheme.lower() != 'https':
             raise ValueError("Customer provided encryption key must be used over HTTPS.")
+        validate_content = parse_validation_option(kwargs.pop('validate_content', None))
+        if is_crc64_validation(validate_content) and self.key_encryption_key:
+            raise ValueError("Using encryption and content validation together is not currently supported.")
         options = _upload_blob_options(
             data=data,
             blob_type=blob_type,
@@ -627,6 +627,7 @@ def upload_blob(
                 'key': self.key_encryption_key,
                 'resolver': self.key_resolver_function
             },
+            validate_content=validate_content,
             config=self._config,
             sdk_moniker=self._sdk_moniker,
             client=self._client,
@@ -683,15 +684,11 @@ def download_blob(
 
             This keyword argument was introduced in API version '2019-12-12'.
 
-        :keyword bool validate_content:
-            If true, calculates an MD5 hash for each chunk of the blob. The storage
-            service checks the hash of the content that has arrived with the hash
-            that was sent. This is primarily valuable for detecting bitflips on
-            the wire if using http instead of https, as https (the default), will
-            already validate. Note that this MD5 hash is not stored with the
-            blob. Also note that if enabled, the memory-efficient upload algorithm
-            will not be used because computing the MD5 hash requires buffering
-            entire blocks, and doing so defeats the purpose of the memory-efficient algorithm.
+        :keyword validate_content:
+            Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
+            Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
+            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+        :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword lease:
             Required if the blob has an active lease. If specified, download_blob only
             succeeds if the blob's lease is active and matches this ID. Value can be a
@@ -765,6 +762,9 @@ def download_blob(
             raise ValueError("Offset value must not be None if length is set.")
         if kwargs.get('cpk') and self.scheme.lower() != 'https':
             raise ValueError("Customer provided encryption key must be used over HTTPS.")
+        validate_content = parse_validation_option(kwargs.pop('validate_content', None))
+        if is_crc64_validation(validate_content) and self.key_encryption_key:
+            raise ValueError("Using encryption and content validation together is not currently supported.")
         options = _download_blob_options(
             blob_name=self.blob_name,
             container_name=self.container_name,
@@ -778,6 +778,7 @@ def download_blob(
                 'key': self.key_encryption_key,
                 'resolver': self.key_resolver_function
             },
+            validate_content=validate_content,
             config=self._config,
             sdk_moniker=self._sdk_moniker,
             client=self._client,
@@ -2009,15 +2010,11 @@ def stage_block(
         :param int length:
             Size of the block. Optional if the length of data can be determined. For Iterable and IO, if the
             length is not provided and cannot be determined, all data will be read into memory.
-        :keyword bool validate_content:
-            If true, calculates an MD5 hash for each chunk of the blob. The storage
-            service checks the hash of the content that has arrived with the hash
-            that was sent. This is primarily valuable for detecting bitflips on
-            the wire if using http instead of https, as https (the default), will
-            already validate. Note that this MD5 hash is not stored with the
-            blob. Also note that if enabled, the memory-efficient upload algorithm
-            will not be used because computing the MD5 hash requires buffering
-            entire blocks, and doing so defeats the purpose of the memory-efficient algorithm.
+        :keyword validate_content:
+            Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
+            Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
+            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+        :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword lease:
             Required if the blob has an active lease. Value can be a BlobLeaseClient object
             or the lease ID as a string.
@@ -2850,13 +2847,11 @@ def upload_page(
             Required if the blob has an active lease. Value can be a BlobLeaseClient object
             or the lease ID as a string.
         :paramtype lease: ~azure.storage.blob.BlobLeaseClient or str
-        :keyword bool validate_content:
-            If true, calculates an MD5 hash of the page content. The storage
-            service checks the hash of the content that has arrived
-            with the hash that was sent. This is primarily valuable for detecting
-            bitflips on the wire if using http instead of https, as https (the default),
-            will already validate. Note that this MD5 hash is not stored with the
-            blob.
+        :keyword validate_content:
+            Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
+            Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
+            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+        :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword int if_sequence_number_lte:
             If the blob's sequence number is less than or equal to
             the specified value, the request proceeds; otherwise it fails.
@@ -3157,13 +3152,11 @@ def append_block(
         :param int length:
             Size of the block. Optional if the length of data can be determined. For Iterable and IO, if the
             length is not provided and cannot be determined, all data will be read into memory.
-        :keyword bool validate_content:
-            If true, calculates an MD5 hash of the block content. The storage
-            service checks the hash of the content that has arrived
-            with the hash that was sent. This is primarily valuable for detecting
-            bitflips on the wire if using http instead of https, as https (the default),
-            will already validate. Note that this MD5 hash is not stored with the
-            blob.
+        :keyword validate_content:
+            Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
+            Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
+            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+        :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword int maxsize_condition:
             Optional conditional header. The max length in bytes permitted for
             the append blob. If the Append Block operation would cause the blob

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.pyi

@@ -173,7 +173,7 @@ class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin):
         tags: Optional[Dict[str, str]] = None,
         overwrite: bool = False,
         content_settings: Optional[ContentSettings] = None,
-        validate_content: bool = False,
+        validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None,
         lease: Optional[BlobLeaseClient] = None,
         if_modified_since: Optional[datetime] = None,
         if_unmodified_since: Optional[datetime] = None,
@@ -200,7 +200,7 @@ class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin):
         length: Optional[int] = None,
         *,
         version_id: Optional[str] = None,
-        validate_content: bool = False,
+        validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None,
         lease: Optional[Union[BlobLeaseClient, str]] = None,
         if_modified_since: Optional[datetime] = None,
         if_unmodified_since: Optional[datetime] = None,
@@ -222,7 +222,7 @@ class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin):
         length: Optional[int] = None,
         *,
         version_id: Optional[str] = None,
-        validate_content: bool = False,
+        validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None,
         lease: Optional[Union[BlobLeaseClient, str]] = None,
         if_modified_since: Optional[datetime] = None,
         if_unmodified_since: Optional[datetime] = None,
@@ -244,7 +244,7 @@ class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin):
         length: Optional[int] = None,
         *,
         version_id: Optional[str] = None,
-        validate_content: bool = False,
+        validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None,
         lease: Optional[Union[BlobLeaseClient, str]] = None,
         if_modified_since: Optional[datetime] = None,
         if_unmodified_since: Optional[datetime] = None,
@@ -486,7 +486,7 @@ class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin):
         data: Union[bytes, Iterable[bytes], IO[bytes]],
         length: Optional[int] = None,
         *,
-        validate_content: Optional[bool] = None,
+        validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None,
         lease: Optional[Union[BlobLeaseClient, str]] = None,
         encoding: Optional[str] = None,
         cpk: Optional[CustomerProvidedEncryptionKey] = None,
@@ -671,7 +671,7 @@ class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin):
         length: int,
         *,
         lease: Optional[Union[BlobLeaseClient, str]] = None,
-        validate_content: Optional[bool] = None,
+        validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None,
         if_sequence_number_lte: Optional[int] = None,
         if_sequence_number_lt: Optional[int] = None,
         if_sequence_number_eq: Optional[int] = None,
@@ -741,7 +741,7 @@ class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin):
         data: Union[bytes, Iterable[bytes], IO[bytes]],
         length: Optional[int] = None,
         *,
-        validate_content: Optional[bool] = None,
+        validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None,
         maxsize_condition: Optional[int] = None,
         appendpos_condition: Optional[int] = None,
         lease: Optional[Union[BlobLeaseClient, str]] = None,

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py

@@ -58,6 +58,7 @@
 from ._shared.response_handlers import return_headers_and_deserialized, return_response_headers
 from ._shared.uploads import IterStreamer
 from ._shared.uploads_async import AsyncIterStreamer
+from ._shared.validation import CV_TYPE_PARSED, parse_validation_option
 from ._upload_helpers import _any_conditions
 
 if TYPE_CHECKING:
@@ -110,6 +111,7 @@ def _upload_blob_options(  # pylint:disable=too-many-statements
     length: Optional[int],
     metadata: Optional[Dict[str, str]],
     encryption_options: Dict[str, Any],
+    validate_content: CV_TYPE_PARSED,
     config: "StorageConfiguration",
     sdk_moniker: str,
     client: "AzureBlobStorage",
@@ -135,7 +137,6 @@ def _upload_blob_options(  # pylint:disable=too-many-statements
     else:
         raise TypeError(f"Unsupported data type: {type(data)}")
 
-    validate_content = kwargs.pop('validate_content', False)
     content_settings = kwargs.pop('content_settings', None)
     overwrite = kwargs.pop('overwrite', False)
     max_concurrency = kwargs.pop('max_concurrency', None)
@@ -258,42 +259,16 @@ def _download_blob_options(
     length: Optional[int],
     encoding: Optional[str],
     encryption_options: Dict[str, Any],
+    validate_content: CV_TYPE_PARSED,
     config: "StorageConfiguration",
     sdk_moniker: str,
     client: "AzureBlobStorage",
     **kwargs
 ) -> Dict[str, Any]:
-    """Creates a dictionary containing the options for a download blob operation.
-
-    :param str blob_name:
-        The name of the blob.
-    :param str container_name:
-        The name of the container.
-    :param Optional[str] version_id:
-        The version id parameter is a value that, when present, specifies the version of the blob to download.
-    :param Optional[int] offset:
-        Start of byte range to use for downloading a section of the blob. Must be set if length is provided.
-    :param Optional[int] length:
-        Number of bytes to read from the stream. This is optional, but should be supplied for optimal performance.
-    :param Optional[str] encoding:
-        Encoding to decode the downloaded bytes. Default is None, i.e. no decoding.
-    :param Dict[str, Any] encryption_options:
-        The options for encryption, if enabled.
-    :param StorageConfiguration config:
-        The Storage configuration options.
-    :param str sdk_moniker:
-        The string representing the SDK package version.
-    :param AzureBlobStorage client:
-        The generated Blob Storage client.
-    :return: A dictionary containing the download blob options.
-    :rtype: Dict[str, Any]
-    """
     if length is not None:
         if offset is None:
             raise ValueError("Offset must be provided if length is provided.")
         length = offset + length - 1  # Service actually uses an end-range inclusive index
-
-    validate_content = kwargs.pop('validate_content', False)
     access_conditions = get_access_conditions(kwargs.pop('lease', None))
     mod_conditions = get_modify_conditions(kwargs)
 
@@ -721,7 +696,7 @@ def _stage_block_options(
     if isinstance(data, bytes):
         data = data[:length]
 
-    validate_content = kwargs.pop('validate_content', False)
+    validate_content = parse_validation_option(kwargs.pop('validate_content', None))
     cpk_scope_info = get_cpk_scope_info(kwargs)
     cpk = kwargs.pop('cpk', None)
     cpk_info = None
@@ -1004,7 +979,7 @@ def _upload_page_options(
     )
     mod_conditions = get_modify_conditions(kwargs)
     cpk_scope_info = get_cpk_scope_info(kwargs)
-    validate_content = kwargs.pop('validate_content', False)
+    validate_content = parse_validation_option(kwargs.pop('validate_content', None))
     cpk = kwargs.pop('cpk', None)
     cpk_info = None
     if cpk:
@@ -1149,7 +1124,7 @@ def _append_block_options(
 
     appendpos_condition = kwargs.pop('appendpos_condition', None)
     maxsize_condition = kwargs.pop('maxsize_condition', None)
-    validate_content = kwargs.pop('validate_content', False)
+    validate_content = parse_validation_option(kwargs.pop('validate_content', None))
     append_conditions = None
     if maxsize_condition or appendpos_condition is not None:
         append_conditions = AppendPositionAccessConditions(