[Storage][102] CRC64 content validation - part 4 (#45949)

Author: jalauzon-msft

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py

@@ -65,6 +65,7 @@
 from ._quick_query_helper import BlobQueryReader
 from ._shared.base_client import parse_connection_str, StorageAccountHostsMixin, TransportWrapper
 from ._shared.response_handlers import process_storage_error, return_response_headers
+from ._shared.validation import ChecksumAlgorithm, parse_validation_option
 from ._serialize import (
     get_access_conditions,
     get_api_version,
@@ -505,15 +506,11 @@ def upload_blob(
         :keyword ~azure.storage.blob.ContentSettings content_settings:
             ContentSettings object used to set blob properties. Used to set content type, encoding,
             language, disposition, md5, and cache control.
-        :keyword bool validate_content:
-            If true, calculates an MD5 hash for each chunk of the blob. The storage
-            service checks the hash of the content that has arrived with the hash
-            that was sent. This is primarily valuable for detecting bitflips on
-            the wire if using http instead of https, as https (the default), will
-            already validate. Note that this MD5 hash is not stored with the
-            blob. Also note that if enabled, the memory-efficient upload algorithm
-            will not be used because computing the MD5 hash requires buffering
-            entire blocks, and doing so defeats the purpose of the memory-efficient algorithm.
+        :keyword validate_content:
+            Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
+            Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
+            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+        :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword lease:
             Required if the blob has an active lease. If specified, upload_blob only succeeds if the
             blob's lease is active and matches this ID. Value can be a BlobLeaseClient object
@@ -616,6 +613,9 @@ def upload_blob(
             raise ValueError("Encryption required but no key was provided.")
         if kwargs.get('cpk') and self.scheme.lower() != 'https':
             raise ValueError("Customer provided encryption key must be used over HTTPS.")
+        validate_content = parse_validation_option(kwargs.pop('validate_content', None))
+        if validate_content == ChecksumAlgorithm.CRC64 and self.key_encryption_key:
+            raise ValueError("Using encryption and content validation together is not currently supported.")
         options = _upload_blob_options(
             data=data,
             blob_type=blob_type,
@@ -627,6 +627,7 @@ def upload_blob(
                 'key': self.key_encryption_key,
                 'resolver': self.key_resolver_function
             },
+            validate_content=validate_content,
             config=self._config,
             sdk_moniker=self._sdk_moniker,
             client=self._client,
@@ -683,15 +684,11 @@ def download_blob(
 
             This keyword argument was introduced in API version '2019-12-12'.
 
-        :keyword bool validate_content:
-            If true, calculates an MD5 hash for each chunk of the blob. The storage
-            service checks the hash of the content that has arrived with the hash
-            that was sent. This is primarily valuable for detecting bitflips on
-            the wire if using http instead of https, as https (the default), will
-            already validate. Note that this MD5 hash is not stored with the
-            blob. Also note that if enabled, the memory-efficient upload algorithm
-            will not be used because computing the MD5 hash requires buffering
-            entire blocks, and doing so defeats the purpose of the memory-efficient algorithm.
+        :keyword validate_content:
+            Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
+            Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
+            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+        :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword lease:
             Required if the blob has an active lease. If specified, download_blob only
             succeeds if the blob's lease is active and matches this ID. Value can be a
@@ -765,6 +762,9 @@ def download_blob(
             raise ValueError("Offset value must not be None if length is set.")
         if kwargs.get('cpk') and self.scheme.lower() != 'https':
             raise ValueError("Customer provided encryption key must be used over HTTPS.")
+        validate_content = parse_validation_option(kwargs.pop('validate_content', None))
+        if validate_content == ChecksumAlgorithm.CRC64 and self.key_encryption_key:
+            raise ValueError("Using encryption and content validation together is not currently supported.")
         options = _download_blob_options(
             blob_name=self.blob_name,
             container_name=self.container_name,
@@ -778,6 +778,7 @@ def download_blob(
                 'key': self.key_encryption_key,
                 'resolver': self.key_resolver_function
             },
+            validate_content=validate_content,
             config=self._config,
             sdk_moniker=self._sdk_moniker,
             client=self._client,
@@ -2009,15 +2010,11 @@ def stage_block(
         :param int length:
             Size of the block. Optional if the length of data can be determined. For Iterable and IO, if the
             length is not provided and cannot be determined, all data will be read into memory.
-        :keyword bool validate_content:
-            If true, calculates an MD5 hash for each chunk of the blob. The storage
-            service checks the hash of the content that has arrived with the hash
-            that was sent. This is primarily valuable for detecting bitflips on
-            the wire if using http instead of https, as https (the default), will
-            already validate. Note that this MD5 hash is not stored with the
-            blob. Also note that if enabled, the memory-efficient upload algorithm
-            will not be used because computing the MD5 hash requires buffering
-            entire blocks, and doing so defeats the purpose of the memory-efficient algorithm.
+        :keyword validate_content:
+            Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
+            Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
+            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+        :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword lease:
             Required if the blob has an active lease. Value can be a BlobLeaseClient object
             or the lease ID as a string.
@@ -2850,13 +2847,11 @@ def upload_page(
             Required if the blob has an active lease. Value can be a BlobLeaseClient object
             or the lease ID as a string.
         :paramtype lease: ~azure.storage.blob.BlobLeaseClient or str
-        :keyword bool validate_content:
-            If true, calculates an MD5 hash of the page content. The storage
-            service checks the hash of the content that has arrived
-            with the hash that was sent. This is primarily valuable for detecting
-            bitflips on the wire if using http instead of https, as https (the default),
-            will already validate. Note that this MD5 hash is not stored with the
-            blob.
+        :keyword validate_content:
+            Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
+            Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
+            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+        :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword int if_sequence_number_lte:
             If the blob's sequence number is less than or equal to
             the specified value, the request proceeds; otherwise it fails.
@@ -3157,13 +3152,11 @@ def append_block(
         :param int length:
             Size of the block. Optional if the length of data can be determined. For Iterable and IO, if the
             length is not provided and cannot be determined, all data will be read into memory.
-        :keyword bool validate_content:
-            If true, calculates an MD5 hash of the block content. The storage
-            service checks the hash of the content that has arrived
-            with the hash that was sent. This is primarily valuable for detecting
-            bitflips on the wire if using http instead of https, as https (the default),
-            will already validate. Note that this MD5 hash is not stored with the
-            blob.
+        :keyword validate_content:
+            Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
+            Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
+            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+        :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword int maxsize_condition:
             Optional conditional header. The max length in bytes permitted for
             the append blob. If the Append Block operation would cause the blob

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.pyi

@@ -173,7 +173,7 @@ class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin):
         tags: Optional[Dict[str, str]] = None,
         overwrite: bool = False,
         content_settings: Optional[ContentSettings] = None,
-        validate_content: bool = False,
+        validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None,
         lease: Optional[BlobLeaseClient] = None,
         if_modified_since: Optional[datetime] = None,
         if_unmodified_since: Optional[datetime] = None,
@@ -200,7 +200,7 @@ class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin):
         length: Optional[int] = None,
         *,
         version_id: Optional[str] = None,
-        validate_content: bool = False,
+        validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None,
         lease: Optional[Union[BlobLeaseClient, str]] = None,
         if_modified_since: Optional[datetime] = None,
         if_unmodified_since: Optional[datetime] = None,
@@ -222,7 +222,7 @@ class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin):
         length: Optional[int] = None,
         *,
         version_id: Optional[str] = None,
-        validate_content: bool = False,
+        validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None,
         lease: Optional[Union[BlobLeaseClient, str]] = None,
         if_modified_since: Optional[datetime] = None,
         if_unmodified_since: Optional[datetime] = None,
@@ -244,7 +244,7 @@ class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin):
         length: Optional[int] = None,
         *,
         version_id: Optional[str] = None,
-        validate_content: bool = False,
+        validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None,
         lease: Optional[Union[BlobLeaseClient, str]] = None,
         if_modified_since: Optional[datetime] = None,
         if_unmodified_since: Optional[datetime] = None,
@@ -486,7 +486,7 @@ class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin):
         data: Union[bytes, Iterable[bytes], IO[bytes]],
         length: Optional[int] = None,
         *,
-        validate_content: Optional[bool] = None,
+        validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None,
         lease: Optional[Union[BlobLeaseClient, str]] = None,
         encoding: Optional[str] = None,
         cpk: Optional[CustomerProvidedEncryptionKey] = None,
@@ -671,7 +671,7 @@ class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin):
         length: int,
         *,
         lease: Optional[Union[BlobLeaseClient, str]] = None,
-        validate_content: Optional[bool] = None,
+        validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None,
         if_sequence_number_lte: Optional[int] = None,
         if_sequence_number_lt: Optional[int] = None,
         if_sequence_number_eq: Optional[int] = None,
@@ -741,7 +741,7 @@ class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin):
         data: Union[bytes, Iterable[bytes], IO[bytes]],
         length: Optional[int] = None,
         *,
-        validate_content: Optional[bool] = None,
+        validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]] = None,
         maxsize_condition: Optional[int] = None,
         appendpos_condition: Optional[int] = None,
         lease: Optional[Union[BlobLeaseClient, str]] = None,

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py

@@ -8,7 +8,7 @@
 from io import BytesIO
 from typing import (
     Any, AnyStr, AsyncGenerator, AsyncIterable, cast,
-    Dict, IO, Iterable, List, Optional, Tuple, Union,
+    Dict, IO, Iterable, List, Literal, Optional, Tuple, Union,
     TYPE_CHECKING
 )
 from urllib.parse import quote, unquote, urlparse
@@ -58,6 +58,7 @@
 from ._shared.response_handlers import return_headers_and_deserialized, return_response_headers
 from ._shared.uploads import IterStreamer
 from ._shared.uploads_async import AsyncIterStreamer
+from ._shared.validation import parse_validation_option
 from ._upload_helpers import _any_conditions
 
 if TYPE_CHECKING:
@@ -110,6 +111,7 @@ def _upload_blob_options(  # pylint:disable=too-many-statements
     length: Optional[int],
     metadata: Optional[Dict[str, str]],
     encryption_options: Dict[str, Any],
+    validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]],
     config: "StorageConfiguration",
     sdk_moniker: str,
     client: "AzureBlobStorage",
@@ -135,7 +137,6 @@ def _upload_blob_options(  # pylint:disable=too-many-statements
     else:
         raise TypeError(f"Unsupported data type: {type(data)}")
 
-    validate_content = kwargs.pop('validate_content', False)
     content_settings = kwargs.pop('content_settings', None)
     overwrite = kwargs.pop('overwrite', False)
     max_concurrency = kwargs.pop('max_concurrency', None)
@@ -258,6 +259,7 @@ def _download_blob_options(
     length: Optional[int],
     encoding: Optional[str],
     encryption_options: Dict[str, Any],
+    validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]],
     config: "StorageConfiguration",
     sdk_moniker: str,
     client: "AzureBlobStorage",
@@ -279,6 +281,8 @@ def _download_blob_options(
         Encoding to decode the downloaded bytes. Default is None, i.e. no decoding.
     :param Dict[str, Any] encryption_options:
         The options for encryption, if enabled.
+    :param validate_content:
+        Enables checksum validation for the transfer. Already parsed via parse_validation_option.
     :param StorageConfiguration config:
         The Storage configuration options.
     :param str sdk_moniker:
@@ -292,8 +296,6 @@ def _download_blob_options(
         if offset is None:
             raise ValueError("Offset must be provided if length is provided.")
         length = offset + length - 1  # Service actually uses an end-range inclusive index
-
-    validate_content = kwargs.pop('validate_content', False)
     access_conditions = get_access_conditions(kwargs.pop('lease', None))
     mod_conditions = get_modify_conditions(kwargs)
 
@@ -721,7 +723,7 @@ def _stage_block_options(
     if isinstance(data, bytes):
         data = data[:length]
 
-    validate_content = kwargs.pop('validate_content', False)
+    validate_content = parse_validation_option(kwargs.pop('validate_content', None))
     cpk_scope_info = get_cpk_scope_info(kwargs)
     cpk = kwargs.pop('cpk', None)
     cpk_info = None
@@ -1004,7 +1006,7 @@ def _upload_page_options(
     )
     mod_conditions = get_modify_conditions(kwargs)
     cpk_scope_info = get_cpk_scope_info(kwargs)
-    validate_content = kwargs.pop('validate_content', False)
+    validate_content = parse_validation_option(kwargs.pop('validate_content', None))
     cpk = kwargs.pop('cpk', None)
     cpk_info = None
     if cpk:
@@ -1149,7 +1151,7 @@ def _append_block_options(
 
     appendpos_condition = kwargs.pop('appendpos_condition', None)
     maxsize_condition = kwargs.pop('maxsize_condition', None)
-    validate_content = kwargs.pop('validate_content', False)
+    validate_content = parse_validation_option(kwargs.pop('validate_content', None))
     append_conditions = None
     if maxsize_condition or appendpos_condition is not None:
         append_conditions = AppendPositionAccessConditions(

sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py

@@ -1010,15 +1010,11 @@ def upload_blob(
         :keyword ~azure.storage.blob.ContentSettings content_settings:
             ContentSettings object used to set blob properties. Used to set content type, encoding,
             language, disposition, md5, and cache control.
-        :keyword bool validate_content:
-            If true, calculates an MD5 hash for each chunk of the blob. The storage
-            service checks the hash of the content that has arrived with the hash
-            that was sent. This is primarily valuable for detecting bitflips on
-            the wire if using http instead of https, as https (the default), will
-            already validate. Note that this MD5 hash is not stored with the
-            blob. Also note that if enabled, the memory-efficient upload algorithm
-            will not be used, because computing the MD5 hash requires buffering
-            entire blocks, and doing so defeats the purpose of the memory-efficient algorithm.
+        :keyword validate_content:
+            Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
+            Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
+            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+        :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword lease:
             Required if the container has an active lease. Value can be a BlobLeaseClient object
             or the lease ID as a string.
@@ -1255,15 +1251,11 @@ def download_blob(
 
             This keyword argument was introduced in API version '2019-12-12'.
 
-        :keyword bool validate_content:
-            If true, calculates an MD5 hash for each chunk of the blob. The storage
-            service checks the hash of the content that has arrived with the hash
-            that was sent. This is primarily valuable for detecting bitflips on
-            the wire if using http instead of https, as https (the default), will
-            already validate. Note that this MD5 hash is not stored with the
-            blob. Also note that if enabled, the memory-efficient upload algorithm
-            will not be used because computing the MD5 hash requires buffering
-            entire blocks, and doing so defeats the purpose of the memory-efficient algorithm.
+        :keyword validate_content:
+            Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
+            Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
+            NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
+        :paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
         :keyword lease:
             Required if the blob has an active lease. If specified, download_blob only
             succeeds if the blob's lease is active and matches this ID. Value can be a