Skip to content

Commit d09810f

Browse files
jalauzon-msftjalauzon-msft
authored
[Storage] Refactor extensions packaging + other misc changes (#47092)
1 parent 017da14 commit d09810f

19 files changed

Lines changed: 141 additions & 87 deletions

File tree

‎sdk/storage/azure-storage-blob/azure/storage/blob/_shared/validation.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,11 @@
2020

2121
def _verify_extensions(module: str) -> None:
2222
try:
23-
import azure.storage.extensions # pylint: disable=unused-import
23+
import azure.storage.extensions.checksums # pylint: disable=unused-import
2424
except ImportError as exc:
2525
raise ValueError(
26-
f"The use of {module} requires the azure-storage-extensions package to be installed. "
27-
f"Please install this package and try again."
26+
f"The use of {module} requires the extra [ext-checksums] to be installed. "
27+
f"Please install this extra and try again."
2828
) from exc
2929

3030

@@ -100,13 +100,13 @@ def calculate_content_md5(data: Union[bytes, IO[bytes]]) -> bytes:
100100

101101
def calculate_crc64(data: bytes, initial_crc: int) -> int:
102102
# Locally import to avoid error if not installed.
103-
from azure.storage.extensions import crc64
103+
from azure.storage.extensions import checksums
104104

105-
return cast(int, crc64.compute(data, initial_crc))
105+
return checksums.crc64.compute(data, initial_crc)
106106

107107

108108
def calculate_crc64_bytes(data: bytes) -> bytes:
109109
# Locally import to avoid error if not installed.
110-
from azure.storage.extensions import crc64
110+
from azure.storage.extensions import checksums
111111

112-
return cast(bytes, crc64.compute(data, 0).to_bytes(CRC64_LENGTH, "little"))
112+
return checksums.crc64.compute(data, 0).to_bytes(CRC64_LENGTH, "little")

‎sdk/storage/azure-storage-blob/setup.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,5 +88,8 @@
8888
"aio": [
8989
"azure-core[aio]>=1.37.0",
9090
],
91+
"ext-checksums": [
92+
"azure-storage-extensions>=0.1.0,<1.0.0",
93+
],
9194
},
9295
)

‎sdk/storage/azure-storage-blob/tests/test_streams.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
StructuredMessageEncodeStream,
2020
StructuredMessageProperties,
2121
)
22-
from azure.storage.extensions import crc64
22+
from azure.storage.extensions import checksums
2323

2424

2525
def _iter_bytes(data: bytes, chunk_size: int = 1024) -> Iterator[bytes]:
@@ -83,12 +83,12 @@ def _build_structured_message(
8383

8484
segment_crc = None
8585
if StructuredMessageProperties.CRC64 in flags:
86-
segment_crc = crc64.compute(segment_data, 0) # pylint: disable=I1101
86+
segment_crc = checksums.crc64.compute(segment_data, 0)
8787
if i == invalidate_crc_segment:
8888
segment_crc += 5
8989
_write_segment(i, segment_data, segment_crc, message)
9090

91-
message_crc = crc64.compute(segment_data, message_crc) # pylint: disable=I1101
91+
message_crc = checksums.crc64.compute(segment_data, message_crc)
9292

9393
# Message footer
9494
if StructuredMessageProperties.CRC64 in flags:

‎sdk/storage/azure-storage-blob/tests/test_streams_async.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
StructuredMessageProperties,
1717
)
1818
from azure.storage.blob._shared.streams_async import AsyncStructuredMessageDecoder
19-
from azure.storage.extensions import crc64
19+
from azure.storage.extensions import checksums
2020

2121

2222
async def _async_iter_bytes(data: bytes, chunk_size: int = 1024) -> AsyncIterator[bytes]:
@@ -80,12 +80,12 @@ def _build_structured_message(
8080

8181
segment_crc = None
8282
if StructuredMessageProperties.CRC64 in flags:
83-
segment_crc = crc64.compute(segment_data, 0) # pylint: disable=I1101
83+
segment_crc = checksums.crc64.compute(segment_data, 0)
8484
if i == invalidate_crc_segment:
8585
segment_crc += 5
8686
_write_segment(i, segment_data, segment_crc, message)
8787

88-
message_crc = crc64.compute(segment_data, message_crc) # pylint: disable=I1101
88+
message_crc = checksums.crc64.compute(segment_data, message_crc)
8989

9090
# Message footer
9191
if StructuredMessageProperties.CRC64 in flags:

‎sdk/storage/azure-storage-extensions/MANIFEST.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
recursive-include azure *.h *.c *.pyi py.typed

‎sdk/storage/azure-storage-extensions/README.md‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,22 +12,18 @@ This package contains native extension modules that provide performance-critical
1212

1313
## Installation
1414

15-
**Recommended**: Install this package via extras when installing Azure Storage libraries:
15+
Install this package via extras when installing Azure Storage libraries:
1616

1717
```bash
18-
pip install azure-storage-blob[extensions]
18+
pip install azure-storage-blob[ext-checksums]
1919
```
2020

2121
This ensures you get compatible versions of both the SDK and the extensions package.
2222

23-
You can also install it directly if needed:
24-
25-
```bash
26-
pip install azure-storage-extensions
27-
```
23+
> ⚠️ Installing `azure-storage-extensions` directly is not recommended. Use the extras syntax above to ensure compatibility.
2824
2925
### Prerequisites
30-
* Python 3.9 or later is required to use this package. For more details, please read our page on [Azure SDK for Python version support policy](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/python_version_support_policy.md).
26+
* Python 3.10 or later is required to use this package. For more details, please read our page on [Azure SDK for Python version support policy](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/python_version_support_policy.md).
3127

3228
### Troubleshooting Installation Issues
3329

‎sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/__init__.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# -------------------------------------------------------------------------
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# Licensed under the MIT License. See License.txt in the project root for
4+
# license information.
5+
# --------------------------------------------------------------------------
6+
7+
from . import crc64
8+
9+
__all__ = ["crc64"]

‎sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/crc64.pyi‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# -------------------------------------------------------------------------
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# Licensed under the MIT License. See License.txt in the project root for
4+
# license information.
5+
# --------------------------------------------------------------------------
6+
7+
def compute(data: bytes, crc: int, /) -> int:
8+
"""Compute Storage CRC64 over given data with given initial CRC64 value.
9+
10+
:param data: The bytes data to compute the CRC64 over.
11+
:param crc: The initial CRC64 value.
12+
:returns: The computed CRC64 value.
13+
"""
14+
...
15+
16+
def concat(
17+
initial_crc_ab: int,
18+
initial_crc_a: int,
19+
final_crc_a: int,
20+
size_a: int,
21+
initial_crc_b: int,
22+
final_crc_b: int,
23+
size_b: int,
24+
/,
25+
) -> int:
26+
"""Concatenate two Storage CRC64s together.
27+
28+
:param initial_crc_ab: The initial CRC64 of the combined data.
29+
:param initial_crc_a: The initial CRC64 of the first data segment.
30+
:param final_crc_a: The final CRC64 of the first data segment.
31+
:param size_a: The size of the first data segment in bytes.
32+
:param initial_crc_b: The initial CRC64 of the second data segment.
33+
:param final_crc_b: The final CRC64 of the second data segment.
34+
:param size_b: The size of the second data segment in bytes.
35+
:returns: The concatenated CRC64 value.
36+
"""
37+
...

sdk/storage/azure-storage-extensions/azure/storage/extensions/crc64/crc64module.c renamed to sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/crc64/crc64module.c

File renamed without changes.

sdk/storage/azure-storage-extensions/azure/storage/extensions/crc64/helpers.h renamed to sdk/storage/azure-storage-extensions/azure/storage/extensions/checksums/crc64/helpers.h

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33

44
#include <stdint.h>
55

6-
const uint64_t poly = 0x9A6C9329AC4BC9B5ULL;
6+
static const uint64_t poly = 0x9A6C9329AC4BC9B5ULL;
77

8-
const uint64_t m_uComplement = ~0ULL;
9-
const uint64_t m_u1[] =
8+
static const uint64_t m_uComplement = ~0ULL;
9+
static const uint64_t m_u1[] =
1010
{
1111
0x0000000000000000ULL, 0x7f6ef0c830358979ULL, 0xfedde190606b12f2ULL, 0x81b31158505e9b8bULL,
1212
0xc962e5739841b68fULL, 0xb60c15bba8743ff6ULL, 0x37bf04e3f82aa47dULL, 0x48d1f42bc81f2d04ULL,
@@ -74,7 +74,7 @@ const uint64_t m_u1[] =
7474
0xab69411fbfb21ca3ULL, 0xd407b1d78f8795daULL, 0x55b4a08fdfd90e51ULL, 0x2ada5047efec8728ULL,
7575
};
7676

77-
const uint64_t m_u32[] =
77+
static const uint64_t m_u32[] =
7878
{
7979
0x0000000000000000ULL, 0xb8c533c1177eb231ULL, 0x455341d1766af709ULL, 0xfd96721061144538ULL,
8080
0x8aa683a2ecd5ee12ULL, 0x3263b063fbab5c23ULL, 0xcff5c2739abf191bULL, 0x7730f1b28dc1ab2aULL,
@@ -597,7 +597,7 @@ const uint64_t m_u32[] =
597597
0x609abef3367d2567ULL, 0xd02690aba479d067ULL, 0x353bc4114ae35c0cULL, 0x8587ea49d8e7a90cULL,
598598
};
599599

600-
const uint64_t m_uX2N[] =
600+
static const uint64_t m_uX2N[] =
601601
{
602602
0x0080000000000000ULL, 0x0000800000000000ULL, 0x0000000080000000ULL, 0x9a6c9329ac4bc9b5ULL,
603603
0x10f4bb0f129310d6ULL, 0x70f05dcea2ebd226ULL, 0x311211205672822dULL, 0x2fc297db0f46c96eULL,
@@ -622,7 +622,7 @@ const uint64_t m_uX2N[] =
622622
// "a" and "b" are represented in "reversed" order -- LSB is x**(XX-1) coefficient, MSB is x^0 coefficient.
623623
// "POLY" is represented in the same manner except for omitted x**XX coefficient
624624
//
625-
uint64_t MulPolyUnrolled(uint64_t a, uint64_t b)
625+
static uint64_t MulPolyUnrolled(uint64_t a, uint64_t b)
626626
{
627627
const uint64_t p = poly;
628628
const uint64_t p2 = (p >> 1) ^ (p * (p & 1));
@@ -647,7 +647,7 @@ uint64_t MulPolyUnrolled(uint64_t a, uint64_t b)
647647
return vr[0] ^ vr[1];
648648
}
649649

650-
uint64_t MulX_N(uint64_t a, uint64_t uSize)
650+
static uint64_t MulX_N(uint64_t a, uint64_t uSize)
651651
{
652652
uint64_t i = 0;
653653
uint64_t r = a;

0 commit comments

Comments
 (0)