[Storage][102] CRC64 content validation - part 6 - file-share (#46262)

Author: jalauzon-msft

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py

@@ -65,7 +65,7 @@
 from ._quick_query_helper import BlobQueryReader
 from ._shared.base_client import parse_connection_str, StorageAccountHostsMixin, TransportWrapper
 from ._shared.response_handlers import process_storage_error, return_response_headers
-from ._shared.validation import ChecksumAlgorithm, parse_validation_option
+from ._shared.validation import is_crc64_validation, parse_validation_option
 from ._serialize import (
     get_access_conditions,
     get_api_version,
@@ -614,7 +614,7 @@ def upload_blob(
         if kwargs.get('cpk') and self.scheme.lower() != 'https':
             raise ValueError("Customer provided encryption key must be used over HTTPS.")
         validate_content = parse_validation_option(kwargs.pop('validate_content', None))
-        if validate_content == ChecksumAlgorithm.CRC64 and self.key_encryption_key:
+        if is_crc64_validation(validate_content) and self.key_encryption_key:
             raise ValueError("Using encryption and content validation together is not currently supported.")
         options = _upload_blob_options(
             data=data,
@@ -763,7 +763,7 @@ def download_blob(
         if kwargs.get('cpk') and self.scheme.lower() != 'https':
             raise ValueError("Customer provided encryption key must be used over HTTPS.")
         validate_content = parse_validation_option(kwargs.pop('validate_content', None))
-        if validate_content == ChecksumAlgorithm.CRC64 and self.key_encryption_key:
+        if is_crc64_validation(validate_content) and self.key_encryption_key:
             raise ValueError("Using encryption and content validation together is not currently supported.")
         options = _download_blob_options(
             blob_name=self.blob_name,

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py

@@ -8,7 +8,7 @@
 from io import BytesIO
 from typing import (
     Any, AnyStr, AsyncGenerator, AsyncIterable, cast,
-    Dict, IO, Iterable, List, Literal, Optional, Tuple, Union,
+    Dict, IO, Iterable, List, Optional, Tuple, Union,
     TYPE_CHECKING
 )
 from urllib.parse import quote, unquote, urlparse
@@ -58,7 +58,7 @@
 from ._shared.response_handlers import return_headers_and_deserialized, return_response_headers
 from ._shared.uploads import IterStreamer
 from ._shared.uploads_async import AsyncIterStreamer
-from ._shared.validation import parse_validation_option
+from ._shared.validation import CV_TYPE_PARSED, parse_validation_option
 from ._upload_helpers import _any_conditions
 
 if TYPE_CHECKING:
@@ -111,7 +111,7 @@ def _upload_blob_options(  # pylint:disable=too-many-statements
     length: Optional[int],
     metadata: Optional[Dict[str, str]],
     encryption_options: Dict[str, Any],
-    validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]],
+    validate_content: CV_TYPE_PARSED,
     config: "StorageConfiguration",
     sdk_moniker: str,
     client: "AzureBlobStorage",
@@ -259,7 +259,7 @@ def _download_blob_options(
     length: Optional[int],
     encoding: Optional[str],
     encryption_options: Dict[str, Any],
-    validate_content: Optional[Union[bool, Literal['auto', 'crc64', 'md5']]],
+    validate_content: CV_TYPE_PARSED,
     config: "StorageConfiguration",
     sdk_moniker: str,
     client: "AzureBlobStorage",

sdk/storage/azure-storage-blob/azure/storage/blob/_download.py

@@ -11,7 +11,7 @@
 from io import BytesIO, StringIO
 from typing import (
     Any, Callable, cast, Dict, Generator,
-    Generic, IO, Iterator, List, Literal, Optional,
+    Generic, IO, Iterator, List, Optional,
     overload, Tuple, TypeVar, Union, TYPE_CHECKING
 )
 
@@ -21,7 +21,7 @@
 from ._shared.request_handlers import validate_and_format_range_headers
 from ._shared.response_handlers import parse_length_from_content_range, process_storage_error
 from ._shared.constants import DEFAULT_MAX_CONCURRENCY
-from ._shared.validation import is_md5_validation
+from ._shared.validation import is_md5_validation, CV_TYPE_PARSED
 from ._deserialize import deserialize_blob_properties, get_page_ranges_result
 from ._encryption import (
     adjust_blob_size_for_encryption,
@@ -92,7 +92,7 @@ def __init__(
         current_progress: int,
         start_range: int,
         end_range: int,
-        validate_content: Optional[Union[bool, Literal['crc64', 'md5']]],
+        validate_content: CV_TYPE_PARSED,
         encryption_options: Dict[str, Any],
         encryption_data: Optional["_EncryptionData"] = None,
         stream: Any = None,
@@ -330,7 +330,7 @@ def __init__(
         config: "StorageConfiguration" = None,  # type: ignore [assignment]
         start_range: Optional[int] = None,
         end_range: Optional[int] = None,
-        validate_content: Optional[Union[bool, Literal['crc64', 'md5']]] = None,
+        validate_content: CV_TYPE_PARSED = None,
         encryption_options: Dict[str, Any] = None,  # type: ignore [assignment]
         max_concurrency: Optional[int] = None,
         name: str = None,  # type: ignore [assignment]

sdk/storage/azure-storage-blob/azure/storage/blob/_shared/policies.py

@@ -5,12 +5,11 @@
 # --------------------------------------------------------------------------
 
 import base64
-import hashlib
 import logging
 import random
 import re
 import uuid
-from io import SEEK_SET, UnsupportedOperation
+from io import BytesIO, SEEK_SET, UnsupportedOperation
 from time import time
 from typing import Any, Dict, Optional, TYPE_CHECKING, Union
 from urllib.parse import (
@@ -43,8 +42,8 @@
     CV_TYPE_ERROR_MSG,
     calculate_content_md5,
     calculate_crc64_bytes,
+    is_crc64_validation,
     is_md5_validation,
-    ChecksumAlgorithm,
 )
 
 if TYPE_CHECKING:
@@ -428,7 +427,7 @@ def _prepare_content_validation(request: "PipelineRequest") -> None:
 
     # Download
     if request.http_request.method == "GET":
-        if validate_content == ChecksumAlgorithm.CRC64:
+        if is_crc64_validation(validate_content):
             request.http_request.headers[SM_HEADER] = SM_HEADER_V1_CRC64
 
     # Upload
@@ -441,7 +440,11 @@ def _prepare_content_validation(request: "PipelineRequest") -> None:
             request.http_request.headers[MD5_HEADER] = computed_md5
             request.context["validate_content_md5"] = computed_md5
 
-        elif validate_content == ChecksumAlgorithm.CRC64:
+        elif is_crc64_validation(validate_content):
+            # For crc64-sm, force structured message even for bytes
+            if validate_content == "crc64-sm" and isinstance(data, bytes):
+                data = BytesIO(data)
+
             if isinstance(data, bytes):
                 request.http_request.headers[CRC64_HEADER] = encode_base64(
                     calculate_crc64_bytes(data)
@@ -495,7 +498,7 @@ def _validate_content_response(
                 response=response.http_response,
             )
 
-    elif validate_content == ChecksumAlgorithm.CRC64:
+    elif is_crc64_validation(validate_content):
         # For upload and download verify structured message header present in response if provided in request.
         sm_request = request.http_request.headers.get(SM_HEADER)
         sm_response = response.http_response.headers.get(SM_HEADER)

sdk/storage/azure-storage-blob/azure/storage/blob/_shared/validation.py

@@ -6,24 +6,16 @@
 # pylint: disable=c-extension-no-member
 
 import hashlib
-from enum import Enum
 from io import SEEK_SET
 from typing import IO, Literal, Optional, Union, cast
 
-from azure.core import CaseInsensitiveEnumMeta
-
 CRC64_LENGTH = 8
 CV_TYPE_ERROR_MSG = "Data should be bytes or seekable IO[bytes] for content validation."
 
+_VALID_CV_OPTIONS = ("auto", "crc64", "crc64-sm", "md5")
 
-class ChecksumAlgorithm(str, Enum, metaclass=CaseInsensitiveEnumMeta):
-    AUTO = "auto"
-    MD5 = "md5"
-    CRC64 = "crc64"
-
-    @classmethod
-    def list(cls):
-        return list(map(lambda c: c.value, cls))
+CV_TYPE = Optional[Union[bool, Literal["auto", "crc64", "md5"]]]
+CV_TYPE_PARSED = Optional[Union[bool, Literal["crc64", "crc64-sm", "md5"]]]
 
 
 def _verify_extensions(module: str) -> None:
@@ -37,36 +29,51 @@ def _verify_extensions(module: str) -> None:
 
 
 def parse_validation_option(
-    validate_content: Optional[Union[bool, Literal["auto", "crc64", "md5"]]],
-) -> Optional[Union[bool, Literal["auto", "crc64", "md5"]]]:
+    validate_content: CV_TYPE,
+    *,
+    force_structured_message: bool = False,
+) -> CV_TYPE_PARSED:
     if validate_content is None:
         return None
 
     # Legacy support for bool
     if isinstance(validate_content, bool):
         return validate_content
 
-    if validate_content not in (ChecksumAlgorithm.list()):
+    parsed = validate_content.lower()
+    if parsed not in _VALID_CV_OPTIONS:
         raise ValueError("Invalid value for `validate_content` specified.")
 
     # Resolve auto
-    if validate_content == ChecksumAlgorithm.AUTO:
-        validate_content = ChecksumAlgorithm.CRC64.value
+    if parsed == "auto":
+        parsed = "crc64"
 
-    if validate_content == ChecksumAlgorithm.CRC64:
+    if parsed == "crc64":
         _verify_extensions("crc64")
+        if force_structured_message:
+            parsed = "crc64-sm"
 
-    return validate_content
+    return cast(CV_TYPE_PARSED, parsed)
 
 
 def is_md5_validation(
-    validate_content: Optional[Union[bool, Literal["md5", "crc64"]]],
+    validate_content: CV_TYPE_PARSED,
 ) -> bool:
     if validate_content is None:
         return False
     if isinstance(validate_content, bool):
         return validate_content
-    return validate_content == ChecksumAlgorithm.MD5
+    return validate_content == "md5"
+
+
+def is_crc64_validation(
+    validate_content: CV_TYPE_PARSED,
+) -> bool:
+    if validate_content is None:
+        return False
+    if isinstance(validate_content, bool):
+        return False
+    return validate_content in ("crc64", "crc64-sm")
 
 
 def calculate_content_md5(data: Union[bytes, IO[bytes]]) -> bytes:

sdk/storage/azure-storage-blob/azure/storage/blob/_upload_helpers.py

@@ -5,7 +5,7 @@
 # --------------------------------------------------------------------------
 
 from io import SEEK_SET, UnsupportedOperation
-from typing import Any, cast, Dict, IO, Literal, Optional, TypeVar, Union, TYPE_CHECKING
+from typing import Any, cast, Dict, IO, Optional, TypeVar, TYPE_CHECKING
 
 from azure.core.exceptions import ResourceExistsError, ResourceModifiedError, HttpResponseError
 
@@ -32,6 +32,7 @@
     upload_data_chunks,
     upload_substream_blocks
 )
+from ._shared.validation import CV_TYPE_PARSED
 
 if TYPE_CHECKING:
     from ._generated.operations import AppendBlobOperations, BlockBlobOperations, PageBlobOperations
@@ -71,7 +72,7 @@ def upload_block_blob(  # pylint: disable=too-many-locals, too-many-statements
     encryption_options: Dict[str, Any],
     blob_settings: "StorageConfiguration",
     headers: Dict[str, Any],
-    validate_content: Optional[Union[bool, Literal['crc64', 'md5']]],
+    validate_content: CV_TYPE_PARSED,
     max_concurrency: Optional[int],
     length: Optional[int] = None,
     **kwargs: Any
@@ -213,7 +214,7 @@ def upload_page_blob(
     headers: Dict[str, Any],
     stream: IO,
     length: Optional[int] = None,
-    validate_content: Optional[Union[bool, Literal['crc64', 'md5']]] = None,
+    validate_content: CV_TYPE_PARSED = None,
     max_concurrency: Optional[int] = None,
     **kwargs: Any
 ) -> Dict[str, Any]:
@@ -291,7 +292,7 @@ def upload_append_blob(  # pylint: disable=unused-argument
     headers: Dict[str, Any],
     stream: IO,
     length: Optional[int] = None,
-    validate_content: Optional[Union[bool, Literal['crc64', 'md5']]] = None,
+    validate_content: CV_TYPE_PARSED = None,
     max_concurrency: Optional[int] = None,
     **kwargs: Any
 ) -> Dict[str, Any]:

sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py

@@ -77,7 +77,7 @@
 from .._shared.base_client_async import AsyncStorageAccountHostsMixin, AsyncTransportWrapper, parse_connection_str
 from .._shared.policies_async import ExponentialRetry
 from .._shared.response_handlers import process_storage_error, return_response_headers
-from .._shared.validation import ChecksumAlgorithm, parse_validation_option
+from .._shared.validation import is_crc64_validation, parse_validation_option
 
 if TYPE_CHECKING:
     from azure.core import MatchConditions
@@ -627,7 +627,7 @@ async def upload_blob(
         if kwargs.get('cpk') and self.scheme.lower() != 'https':
             raise ValueError("Customer provided encryption key must be used over HTTPS.")
         validate_content = parse_validation_option(kwargs.pop('validate_content', None))
-        if validate_content == ChecksumAlgorithm.CRC64 and self.key_encryption_key:
+        if is_crc64_validation(validate_content) and self.key_encryption_key:
             raise ValueError("Using encryption and content validation together is not currently supported.")
         options = _upload_blob_options(
             data=data,
@@ -776,7 +776,7 @@ async def download_blob(
         if kwargs.get('cpk') and self.scheme.lower() != 'https':
             raise ValueError("Customer provided encryption key must be used over HTTPS.")
         validate_content = parse_validation_option(kwargs.pop('validate_content', None))
-        if validate_content == ChecksumAlgorithm.CRC64 and self.key_encryption_key:
+        if is_crc64_validation(validate_content) and self.key_encryption_key:
             raise ValueError("Using encryption and content validation together is not currently supported.")
         options = _download_blob_options(
             blob_name=self.blob_name,

sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py

@@ -15,7 +15,7 @@
 from typing import (
     Any, AsyncIterator, Awaitable,
     Generator, Callable, cast, Dict,
-    Generic, IO, Literal, Optional, overload,
+    Generic, IO, Optional, overload,
     Tuple, TypeVar, Union, TYPE_CHECKING
 )
 
@@ -24,7 +24,7 @@
 from .._shared.request_handlers import validate_and_format_range_headers
 from .._shared.response_handlers import parse_length_from_content_range, process_storage_error
 from .._shared.constants import DEFAULT_MAX_CONCURRENCY
-from .._shared.validation import is_md5_validation
+from .._shared.validation import is_md5_validation, CV_TYPE_PARSED
 from .._deserialize import deserialize_blob_properties, get_page_ranges_result
 from .._download import process_range_and_offset, _ChunkDownloader
 from .._encryption import (
@@ -239,7 +239,7 @@ def __init__(
         config: "StorageConfiguration" = None,  # type: ignore [assignment]
         start_range: Optional[int] = None,
         end_range: Optional[int] = None,
-        validate_content: Optional[Union[bool, Literal['crc64', 'md5']]] = None,
+        validate_content: CV_TYPE_PARSED = None,
         encryption_options: Dict[str, Any] = None,  # type: ignore [assignment]
         max_concurrency: Optional[int] = None,
         name: str = None,  # type: ignore [assignment]

sdk/storage/azure-storage-blob/azure/storage/blob/aio/_upload_helpers.py

@@ -6,7 +6,7 @@
 
 import inspect
 from io import SEEK_SET, UnsupportedOperation
-from typing import Any, cast, Dict, IO, Literal, Optional, TypeVar, Union, TYPE_CHECKING
+from typing import Any, cast, Dict, IO, Optional, TypeVar, TYPE_CHECKING
 
 from azure.core.exceptions import HttpResponseError, ResourceModifiedError
 
@@ -32,6 +32,7 @@
     upload_data_chunks,
     upload_substream_blocks
 )
+from .._shared.validation import CV_TYPE_PARSED
 from .._upload_helpers import _any_conditions, _convert_mod_error
 
 if TYPE_CHECKING:
@@ -47,7 +48,7 @@ async def upload_block_blob(  # pylint: disable=too-many-locals, too-many-statem
     encryption_options: Dict[str, Any],
     blob_settings: "StorageConfiguration",
     headers: Dict[str, Any],
-    validate_content: Optional[Union[bool, Literal['crc64', 'md5']]],
+    validate_content: CV_TYPE_PARSED,
     max_concurrency: Optional[int],
     length: Optional[int] = None,
     **kwargs: Any
@@ -193,7 +194,7 @@ async def upload_page_blob(
     headers: Dict[str, Any],
     stream: IO,
     length: Optional[int] = None,
-    validate_content: Optional[Union[bool, Literal['crc64', 'md5']]] = None,
+    validate_content: CV_TYPE_PARSED = None,
     max_concurrency: Optional[int] = None,
     **kwargs: Any
 ) -> Dict[str, Any]:
@@ -271,7 +272,7 @@ async def upload_append_blob(  # pylint: disable=unused-argument
     headers: Dict[str, Any],
     stream: IO,
     length: Optional[int] = None,
-    validate_content: Optional[Union[bool, Literal['crc64', 'md5']]] = None,
+    validate_content: CV_TYPE_PARSED = None,
     max_concurrency: Optional[int] = None,
     **kwargs: Any
 ) -> Dict[str, Any]: