Skip to content

Commit 3171871

Browse files
jalauzon-msftjalauzon-msft
authored
[Storage] Disable decompression on download when using crc64 (#47553)
1 parent 15d9064 commit 3171871

13 files changed

Lines changed: 148 additions & 11 deletions

File tree

‎sdk/storage/azure-storage-blob/assets.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"AssetsRepo": "Azure/azure-sdk-assets",
33
"AssetsRepoPrefixPath": "python",
44
"TagPrefix": "python/storage/azure-storage-blob",
5-
"Tag": "python/storage/azure-storage-blob_b09e37b521"
5+
"Tag": "python/storage/azure-storage-blob_2d1cd89589"
66
}

‎sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -678,7 +678,13 @@ def download_blob(
678678
:keyword validate_content:
679679
Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
680680
Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
681-
NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
681+
682+
.. note:: When using CRC64 validation (including when "auto" resolves to CRC64):
683+
684+
- The ``ext-checksums`` extra must be installed.
685+
- Automatic decompression is not supported. If ``decompress=True`` is explicitly
686+
set, a :class:`ValueError` will be raised. If ``decompress`` is not specified,
687+
it will be set to ``False`` automatically.
682688
:paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
683689
:keyword lease:
684690
Required if the blob has an active lease. If specified, download_blob only

‎sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client_helpers.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@
6363
from ._shared.response_handlers import return_headers_and_deserialized, return_response_headers
6464
from ._shared.uploads import IterStreamer
6565
from ._shared.uploads_async import AsyncIterStreamer
66-
from ._shared.validation import CV_TYPE_PARSED, parse_validation_option
66+
from ._shared.validation import CV_TYPE_PARSED, is_crc64_validation, parse_validation_option
6767
from ._upload_helpers import _any_conditions
6868

6969
if TYPE_CHECKING:
@@ -294,6 +294,13 @@ def _download_blob_options(
294294
config.user_agent_policy.user_agent, sdk_moniker, encryption_options["version"], kwargs
295295
)
296296

297+
# Decompression is not supported with CRC64 content validation
298+
if is_crc64_validation(validate_content):
299+
decompress = kwargs.get("decompress")
300+
if decompress is True:
301+
raise ValueError("Decompression is not supported when using CRC64 content validation.")
302+
kwargs["decompress"] = False
303+
297304
options = {
298305
"clients": client,
299306
"config": config,

‎sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -707,7 +707,13 @@ async def download_blob(
707707
:keyword validate_content:
708708
Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
709709
Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
710-
NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
710+
711+
.. note:: When using CRC64 validation (including when "auto" resolves to CRC64):
712+
713+
- The ``ext-checksums`` extra must be installed.
714+
- Automatic decompression is not supported. If ``decompress=True`` is explicitly
715+
set, a :class:`ValueError` will be raised. If ``decompress`` is not specified,
716+
it will be set to ``False`` automatically.
711717
:paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
712718
:keyword lease:
713719
Required if the blob has an active lease. If specified, download_blob only

‎sdk/storage/azure-storage-blob/tests/test_content_validation.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -645,3 +645,23 @@ def hook_fail_once(response):
645645
blob.commit_block_list([BlobBlock("1")])
646646
result = blob.download_blob()
647647
assert result.read() == content
648+
649+
@BlobPreparer()
650+
@pytest.mark.parametrize("a", ["auto", "crc64"]) # a: validate_content
651+
@GenericTestProxyParametrize1()
652+
@recorded_by_proxy
653+
def test_download_decompress_with_crc64(self, a, **kwargs):
654+
storage_account_name = kwargs.pop("storage_account_name")
655+
656+
self._setup(storage_account_name)
657+
blob = self.container.get_blob_client(self._get_blob_reference())
658+
data = b"abc" * 512
659+
blob.upload_blob(data, overwrite=True)
660+
661+
# decompress=True should raise ValueError
662+
with pytest.raises(ValueError, match="Decompression is not supported when using CRC64 content validation."):
663+
blob.download_blob(validate_content=a, decompress=True)
664+
665+
# decompress=False should work fine
666+
downloader = blob.download_blob(validate_content=a, decompress=False)
667+
assert downloader.read() == data

‎sdk/storage/azure-storage-blob/tests/test_content_validation_async.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -628,3 +628,23 @@ def hook_fail_once(response):
628628
await blob.commit_block_list([BlobBlock("1")])
629629
result = await blob.download_blob()
630630
assert await result.read() == content
631+
632+
@BlobPreparer()
633+
@pytest.mark.parametrize("a", ["auto", "crc64"]) # a: validate_content
634+
@GenericTestProxyParametrize1()
635+
@recorded_by_proxy_async
636+
async def test_download_decompress_with_crc64(self, a, **kwargs):
637+
storage_account_name = kwargs.pop("storage_account_name")
638+
639+
await self._setup(storage_account_name)
640+
blob = self.container.get_blob_client(self._get_blob_reference())
641+
data = b"abc" * 512
642+
await blob.upload_blob(data, overwrite=True)
643+
644+
# decompress=True should raise ValueError
645+
with pytest.raises(ValueError, match="Decompression is not supported when using CRC64 content validation."):
646+
await blob.download_blob(validate_content=a, decompress=True)
647+
648+
# decompress=False should work fine
649+
downloader = await blob.download_blob(validate_content=a, decompress=False)
650+
assert await downloader.read() == data

‎sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -688,7 +688,13 @@ def download_file(
688688
:keyword validate_content:
689689
Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
690690
Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
691-
NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
691+
692+
.. note:: When using CRC64 validation (including when "auto" resolves to CRC64):
693+
694+
- The ``ext-checksums`` extra must be installed.
695+
- Automatic decompression is not supported. If ``decompress=True`` is explicitly
696+
set, a :class:`ValueError` will be raised. If ``decompress`` is not specified,
697+
it will be set to ``False`` automatically.
692698
:paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
693699
:keyword int timeout:
694700
Sets the server-side timeout for the operation in seconds. For more details see

‎sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -710,7 +710,13 @@ async def download_file(
710710
:keyword validate_content:
711711
Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the blob.
712712
Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
713-
NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
713+
714+
.. note:: When using CRC64 validation (including when "auto" resolves to CRC64):
715+
716+
- The ``ext-checksums`` extra must be installed.
717+
- Automatic decompression is not supported. If ``decompress=True`` is explicitly
718+
set, a :class:`ValueError` will be raised. If ``decompress`` is not specified,
719+
it will be set to ``False`` automatically.
714720
:paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
715721
:keyword int timeout:
716722
Sets the server-side timeout for the operation in seconds. For more details see

‎sdk/storage/azure-storage-file-share/assets.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"AssetsRepo": "Azure/azure-sdk-assets",
33
"AssetsRepoPrefixPath": "python",
44
"TagPrefix": "python/storage/azure-storage-file-share",
5-
"Tag": "python/storage/azure-storage-file-share_d5f376c42f"
5+
"Tag": "python/storage/azure-storage-file-share_bcf00830c4"
66
}

‎sdk/storage/azure-storage-file-share/azure/storage/fileshare/_file_client.py‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
from ._shared.request_handlers import add_metadata_headers, get_length
4343
from ._shared.response_handlers import return_response_headers, process_storage_error
4444
from ._shared.uploads import IterStreamer, FileChunkUploader, upload_data_chunks
45-
from ._shared.validation import CV_TYPE_PARSED, parse_validation_option
45+
from ._shared.validation import CV_TYPE_PARSED, is_crc64_validation, parse_validation_option
4646

4747
if TYPE_CHECKING:
4848
from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential, TokenCredential
@@ -902,7 +902,13 @@ def download_file(
902902
:keyword validate_content:
903903
Enables checksum validation for the transfer. Any checksum calculated is NOT stored with the file.
904904
Choose "auto" (let the SDK choose the best algorithm), "crc64", or "md5". The use of bool is deprecated.
905-
NOTE: The use of "auto" or "crc64" requires the `azure-storage-extensions` package to be installed.
905+
906+
.. note:: When using CRC64 validation (including when "auto" resolves to CRC64):
907+
908+
- The ``ext-checksums`` extra must be installed.
909+
- Automatic decompression is not supported. If ``decompress=True`` is explicitly
910+
set, a :class:`ValueError` will be raised. If ``decompress`` is not specified,
911+
it will be set to ``False`` automatically.
906912
:paramtype validate_content: Union[bool, Literal['auto', 'crc64', 'md5']]
907913
:keyword lease:
908914
Required if the file has an active lease. Value can be a ShareLeaseClient object
@@ -945,6 +951,13 @@ def download_file(
945951
access_conditions = get_access_conditions(kwargs.pop("lease", None))
946952
validate_content = parse_validation_option(kwargs.pop("validate_content", None))
947953

954+
# Decompression is not supported with CRC64 content validation
955+
if is_crc64_validation(validate_content):
956+
decompress = kwargs.get("decompress")
957+
if decompress is True:
958+
raise ValueError("Decompression is not supported when using CRC64 content validation.")
959+
kwargs["decompress"] = False
960+
948961
return StorageStreamDownloader(
949962
client=self._client.file,
950963
config=self._config,

0 commit comments

Comments
 (0)