[Storage][102] CRC64 content validation - part 7 - record tests (#46461)

Author: jalauzon-msft

eng/tools/azure-sdk-tools/devtools_testutils/storage/decorators.py

@@ -7,12 +7,12 @@
 class GenericTestProxyParametrize1:
     def __call__(self, fn):
         def _wrapper(test_class, a, **kwargs):
-            fn(test_class, a, **kwargs)
+            return fn(test_class, a, **kwargs)
         return _wrapper
 
 
 class GenericTestProxyParametrize2:
     def __call__(self, fn):
         def _wrapper(test_class, a, b, **kwargs):
-            fn(test_class, a, b, **kwargs)
+            return fn(test_class, a, b, **kwargs)
         return _wrapper

sdk/storage/azure-storage-blob/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/storage/azure-storage-blob",
-  "Tag": "python/storage/azure-storage-blob_339150483d"
+  "Tag": "python/storage/azure-storage-blob_e0a670a6a4"
 }

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py

@@ -265,33 +265,6 @@ def _download_blob_options(
     client: "AzureBlobStorage",
     **kwargs
 ) -> Dict[str, Any]:
-    """Creates a dictionary containing the options for a download blob operation.
-
-    :param str blob_name:
-        The name of the blob.
-    :param str container_name:
-        The name of the container.
-    :param Optional[str] version_id:
-        The version id parameter is a value that, when present, specifies the version of the blob to download.
-    :param Optional[int] offset:
-        Start of byte range to use for downloading a section of the blob. Must be set if length is provided.
-    :param Optional[int] length:
-        Number of bytes to read from the stream. This is optional, but should be supplied for optimal performance.
-    :param Optional[str] encoding:
-        Encoding to decode the downloaded bytes. Default is None, i.e. no decoding.
-    :param Dict[str, Any] encryption_options:
-        The options for encryption, if enabled.
-    :param validate_content:
-        Enables checksum validation for the transfer. Already parsed via parse_validation_option.
-    :param StorageConfiguration config:
-        The Storage configuration options.
-    :param str sdk_moniker:
-        The string representing the SDK package version.
-    :param AzureBlobStorage client:
-        The generated Blob Storage client.
-    :return: A dictionary containing the download blob options.
-    :rtype: Dict[str, Any]
-    """
     if length is not None:
         if offset is None:
             raise ValueError("Offset must be provided if length is provided.")

sdk/storage/azure-storage-blob/azure/storage/blob/_download.py

@@ -386,7 +386,11 @@ def __init__(
         # The service only provides transactional MD5s for chunks under 4MB.
         # If validate_content is using MD5, get only self.MAX_CHUNK_GET_SIZE for the first
         # chunk so a transactional MD5 can be retrieved.
-        first_get_size = self._config.max_single_get_size if not is_md5_validation(self._validate_content) else self._config.max_chunk_get_size
+        first_get_size = (
+            self._config.max_single_get_size
+            if not is_md5_validation(self._validate_content)
+            else self._config.max_chunk_get_size
+        )
 
         initial_request_start = self._download_start
         if self._end_range is not None and self._end_range - initial_request_start < first_get_size:

sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client_async.py

@@ -40,7 +40,11 @@
     StorageHosts,
     StorageRequestHook,
 )
-from .policies_async import AsyncStorageBearerTokenCredentialPolicy, AsyncContentValidationPolicy, AsyncStorageResponseHook
+from .policies_async import (
+    AsyncStorageBearerTokenCredentialPolicy,
+    AsyncContentValidationPolicy,
+    AsyncStorageResponseHook,
+)
 from .response_handlers import PartialBatchErrorException, process_storage_error
 from .._shared_access_signature import _is_credential_sastoken
 

sdk/storage/azure-storage-blob/azure/storage/blob/_shared/policies.py

@@ -45,17 +45,9 @@
 async def retry_hook(settings, **kwargs):
     if settings["hook"]:
         if asyncio.iscoroutine(settings["hook"]):
-            await settings["hook"](
-                retry_count=settings["count"] - 1,
-                location_mode=settings["mode"],
-                **kwargs
-            )
+            await settings["hook"](retry_count=settings["count"] - 1, location_mode=settings["mode"], **kwargs)
         else:
-            settings["hook"](
-                retry_count=settings["count"] - 1,
-                location_mode=settings["mode"],
-                **kwargs
-            )
+            settings["hook"](retry_count=settings["count"] - 1, location_mode=settings["mode"], **kwargs)
 
 
 async def is_checksum_retry(response):
@@ -70,9 +62,9 @@ async def is_checksum_retry(response):
                 await response.http_response.load_body()  # Load the body in memory and close the socket
             except (StreamClosedError, StreamConsumedError):
                 pass
-        computed_md5 = response.http_request.headers.get(
-            "content-md5", None
-        ) or encode_base64(calculate_content_md5(response.http_response.body()))
+        computed_md5 = response.http_request.headers.get("content-md5", None) or encode_base64(
+            calculate_content_md5(response.http_response.body())
+        )
         if response.http_response.headers["content-md5"] != computed_md5:
             return True
     return False
@@ -118,50 +110,36 @@ async def send(self, request: "PipelineRequest") -> "PipelineResponse":
             data_stream_total = request.context.options.pop("data_stream_total", None)
         download_stream_current = request.context.get("download_stream_current")
         if download_stream_current is None:
-            download_stream_current = request.context.options.pop(
-                "download_stream_current", None
-            )
+            download_stream_current = request.context.options.pop("download_stream_current", None)
         upload_stream_current = request.context.get("upload_stream_current")
         if upload_stream_current is None:
-            upload_stream_current = request.context.options.pop(
-                "upload_stream_current", None
-            )
+            upload_stream_current = request.context.options.pop("upload_stream_current", None)
 
-        response_callback = request.context.get(
-            "response_callback"
-        ) or request.context.options.pop("raw_response_hook", self._response_callback)
+        response_callback = request.context.get("response_callback") or request.context.options.pop(
+            "raw_response_hook", self._response_callback
+        )
 
         response = await self.next.send(request)
-        will_retry = is_retry(
-            response, request.context.options.get("mode")
-        ) or await is_checksum_retry(response)
+        will_retry = is_retry(response, request.context.options.get("mode")) or await is_checksum_retry(response)
 
         # Auth error could come from Bearer challenge, in which case this request will be made again
         is_auth_error = response.http_response.status_code == 401
         should_update_counts = not (will_retry or is_auth_error)
 
         if should_update_counts and download_stream_current is not None:
-            download_stream_current += int(
-                response.http_response.headers.get("Content-Length", 0)
-            )
+            download_stream_current += int(response.http_response.headers.get("Content-Length", 0))
             if data_stream_total is None:
                 content_range = response.http_response.headers.get("Content-Range")
                 if content_range:
-                    data_stream_total = int(
-                        content_range.split(" ", 1)[1].split("/", 1)[1]
-                    )
+                    data_stream_total = int(content_range.split(" ", 1)[1].split("/", 1)[1])
                 else:
                     data_stream_total = download_stream_current
         elif should_update_counts and upload_stream_current is not None:
-            upload_stream_current += int(
-                response.http_request.headers.get("Content-Length", 0)
-            )
+            upload_stream_current += int(response.http_request.headers.get("Content-Length", 0))
         for pipeline_obj in [request, response]:
             if hasattr(pipeline_obj, "context"):
                 pipeline_obj.context["data_stream_total"] = data_stream_total
-                pipeline_obj.context["download_stream_current"] = (
-                    download_stream_current
-                )
+                pipeline_obj.context["download_stream_current"] = download_stream_current
                 pipeline_obj.context["upload_stream_current"] = upload_stream_current
         if response_callback:
             if asyncio.iscoroutine(response_callback):
@@ -190,9 +168,7 @@ async def send(self, request):
         while retries_remaining:
             try:
                 response = await self.next.send(request)
-                if is_retry(
-                    response, retry_settings["mode"]
-                ) or await is_checksum_retry(response):
+                if is_retry(response, retry_settings["mode"]) or await is_checksum_retry(response):
                     retries_remaining = self.increment(
                         retry_settings,
                         request=request.http_request,
@@ -211,9 +187,7 @@ async def send(self, request):
             except AzureError as err:
                 if isinstance(err, AzureSigningError):
                     raise
-                retries_remaining = self.increment(
-                    retry_settings, request=request.http_request, error=err
-                )
+                retries_remaining = self.increment(retry_settings, request=request.http_request, error=err)
                 if retries_remaining:
                     await retry_hook(
                         retry_settings,
@@ -275,9 +249,7 @@ def __init__(
         self.initial_backoff = initial_backoff
         self.increment_base = increment_base
         self.random_jitter_range = random_jitter_range
-        super(ExponentialRetry, self).__init__(
-            retry_total=retry_total, retry_to_secondary=retry_to_secondary, **kwargs
-        )
+        super(ExponentialRetry, self).__init__(retry_total=retry_total, retry_to_secondary=retry_to_secondary, **kwargs)
 
     def get_backoff_time(self, settings: Dict[str, Any]) -> float:
         """
@@ -290,14 +262,8 @@ def get_backoff_time(self, settings: Dict[str, Any]) -> float:
         :rtype: int or None
         """
         random_generator = random.Random()
-        backoff = self.initial_backoff + (
-            0 if settings["count"] == 0 else pow(self.increment_base, settings["count"])
-        )
-        random_range_start = (
-            backoff - self.random_jitter_range
-            if backoff > self.random_jitter_range
-            else 0
-        )
+        backoff = self.initial_backoff + (0 if settings["count"] == 0 else pow(self.increment_base, settings["count"]))
+        random_range_start = backoff - self.random_jitter_range if backoff > self.random_jitter_range else 0
         random_range_end = backoff + self.random_jitter_range
         return random_generator.uniform(random_range_start, random_range_end)
 
@@ -335,9 +301,7 @@ def __init__(
         """
         self.backoff = backoff
         self.random_jitter_range = random_jitter_range
-        super(LinearRetry, self).__init__(
-            retry_total=retry_total, retry_to_secondary=retry_to_secondary, **kwargs
-        )
+        super(LinearRetry, self).__init__(retry_total=retry_total, retry_to_secondary=retry_to_secondary, **kwargs)
 
     def get_backoff_time(self, settings: Dict[str, Any]) -> float:
         """
@@ -352,28 +316,18 @@ def get_backoff_time(self, settings: Dict[str, Any]) -> float:
         random_generator = random.Random()
         # the backoff interval normally does not change, however there is the possibility
         # that it was modified by accessing the property directly after initializing the object
-        random_range_start = (
-            self.backoff - self.random_jitter_range
-            if self.backoff > self.random_jitter_range
-            else 0
-        )
+        random_range_start = self.backoff - self.random_jitter_range if self.backoff > self.random_jitter_range else 0
         random_range_end = self.backoff + self.random_jitter_range
         return random_generator.uniform(random_range_start, random_range_end)
 
 
 class AsyncStorageBearerTokenCredentialPolicy(AsyncBearerTokenCredentialPolicy):
     """Custom Bearer token credential policy for following Storage Bearer challenges"""
 
-    def __init__(
-        self, credential: "AsyncTokenCredential", audience: str, **kwargs: Any
-    ) -> None:
-        super(AsyncStorageBearerTokenCredentialPolicy, self).__init__(
-            credential, audience, **kwargs
-        )
+    def __init__(self, credential: "AsyncTokenCredential", audience: str, **kwargs: Any) -> None:
+        super(AsyncStorageBearerTokenCredentialPolicy, self).__init__(credential, audience, **kwargs)
 
-    async def on_challenge(
-        self, request: "PipelineRequest", response: "PipelineResponse"
-    ) -> bool:
+    async def on_challenge(self, request: "PipelineRequest", response: "PipelineResponse") -> bool:
         try:
             auth_header = response.http_response.headers.get("WWW-Authenticate")
             challenge = StorageHttpChallenge(auth_header)