Merge branch 'main' of github.com:zarr-developers/zarr-python into refactor/metadata-package

Author: d-v-b

changes/3679.feature.md

@@ -0,0 +1,3 @@
+Adds a new in-memory storage backend called `ManagedMemoryStore`. Instances of `ManagedMemoryStore`
+function similarly to `MemoryStore`, but instances of `ManagedMemoryStore` can be constructed from
+a URL like `memory://store`.

changes/3781.feature.md

@@ -0,0 +1 @@
+Added `Struct` class (subclass of `Structured`) implementing the zarr-extensions `struct` dtype spec. Uses object-style field format and dict fill values. Legacy `Structured` remains available for backward compatibility.

docs/user-guide/arrays.md

@@ -14,15 +14,14 @@ np.random.seed(0)
 
 ```python exec="true" session="arrays" source="above" result="ansi"
 import zarr
-store = zarr.storage.MemoryStore()
-z = zarr.create_array(store=store, shape=(10000, 10000), chunks=(1000, 1000), dtype='int32')
+z = zarr.create_array(store="memory://arrays-demo", shape=(10000, 10000), chunks=(1000, 1000), dtype='int32')
 print(z)
 ```
 
 The code above creates a 2-dimensional array of 32-bit integers with 10000 rows
 and 10000 columns, divided into chunks where each chunk has 1000 rows and 1000
-columns (and so there will be 100 chunks in total). The data is written to a
-[`zarr.storage.MemoryStore`][] (e.g. an in-memory dict). See
+columns (and so there will be 100 chunks in total). The data is written to an
+in-memory store (see [`zarr.storage.MemoryStore`][] for more details). See
 [Persistent arrays](#persistent-arrays) for details on storing arrays in other stores,
 and see [Data types](data_types.md) for an in-depth look at the data types supported
 by Zarr.

docs/user-guide/attributes.md

@@ -3,33 +3,32 @@
 Zarr arrays and groups support custom key/value attributes, which can be useful for
 storing application-specific metadata. For example:
 
-```python exec="true" session="arrays" source="above" result="ansi"
+```python exec="true" session="attributes" source="above" result="ansi"
 import zarr
-store = zarr.storage.MemoryStore()
-root = zarr.create_group(store=store)
+root = zarr.create_group(store="memory://attributes-demo")
 root.attrs['foo'] = 'bar'
 z = root.create_array(name='zzz', shape=(10000, 10000), dtype='int32')
 z.attrs['baz'] = 42
 z.attrs['qux'] = [1, 4, 7, 12]
 print(sorted(root.attrs))
 ```
 
-```python exec="true" session="arrays" source="above" result="ansi"
+```python exec="true" session="attributes" source="above" result="ansi"
 print('foo' in root.attrs)
 ```
 
-```python exec="true" session="arrays" source="above" result="ansi"
+```python exec="true" session="attributes" source="above" result="ansi"
 print(root.attrs['foo'])
 ```
-```python exec="true" session="arrays" source="above" result="ansi"
+```python exec="true" session="attributes" source="above" result="ansi"
 print(sorted(z.attrs))
 ```
 
-```python exec="true" session="arrays" source="above" result="ansi"
+```python exec="true" session="attributes" source="above" result="ansi"
 print(z.attrs['baz'])
 ```
 
-```python exec="true" session="arrays" source="above" result="ansi"
+```python exec="true" session="attributes" source="above" result="ansi"
 print(z.attrs['qux'])
 ```
 

docs/user-guide/consolidated_metadata.md

@@ -27,8 +27,7 @@ import zarr
 import warnings
 
 warnings.filterwarnings("ignore", category=UserWarning)
-store = zarr.storage.MemoryStore()
-group = zarr.create_group(store=store)
+group = zarr.create_group(store="memory://consolidated-metadata-demo")
 print(group)
 array = group.create_array(shape=(1,), name='a', dtype='float64')
 print(array)
@@ -45,7 +44,7 @@ print(array)
 ```
 
 ```python exec="true" session="consolidated_metadata" source="above" result="ansi"
-result = zarr.consolidate_metadata(store)
+result = zarr.consolidate_metadata("memory://consolidated-metadata-demo")
 print(result)
 ```
 
@@ -56,7 +55,7 @@ that can be used.:
 from pprint import pprint
 import io
 
-consolidated = zarr.open_group(store=store)
+consolidated = zarr.open_group(store="memory://consolidated-metadata-demo")
 consolidated_metadata = consolidated.metadata.consolidated_metadata.metadata
 
 # Note: pprint can be users without capturing the output regularly
@@ -76,7 +75,7 @@ With nested groups, the consolidated metadata is available on the children, recu
 ```python exec="true" session="consolidated_metadata" source="above" result="ansi"
 child = group.create_group('child', attributes={'kind': 'child'})
 grandchild = child.create_group('child', attributes={'kind': 'grandchild'})
-consolidated = zarr.consolidate_metadata(store)
+consolidated = zarr.consolidate_metadata("memory://consolidated-metadata-demo")
 
 output = io.StringIO()
 pprint(consolidated['child'].metadata.consolidated_metadata, stream=output, width=60)

docs/user-guide/data_types.md

@@ -229,6 +229,37 @@ here, it's possible to create it yourself: see [Adding New Data Types](#adding-n
 #### Struct-like
 - [Structured][zarr.dtype.Structured]
 
+!!! note "Zarr V3 Structured Data Types"
+
+    In Zarr V3, structured data types are specified using the `struct` extension defined in the
+    [zarr-extensions repository](https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/struct).
+    The JSON representation uses an object format for fields:
+
+    ```json
+    {
+        "name": "struct",
+        "configuration": {
+            "fields": [
+                {"name": "x", "data_type": "float32"},
+                {"name": "y", "data_type": "int64"}
+            ]
+        }
+    }
+    ```
+
+    For backward compatibility, Zarr Python also accepts the legacy `structured` name with
+    tuple-format fields when reading existing data.
+
+    Fill values for structured types are represented as JSON objects mapping field names to values:
+
+    ```json
+    {"x": 1.5, "y": 42}
+    ```
+
+    When using structured types with multi-byte fields, the `bytes` codec must specify an
+    explicit `endian` parameter. If omitted, Zarr Python assumes little-endian for legacy
+    compatibility but emits a warning.
+
 ### Example Usage
 
 This section will demonstrates the basic usage of Zarr data types.

docs/user-guide/gpu.md

@@ -20,9 +20,8 @@ buffers used internally by Zarr via `enable_gpu()`.
 import zarr
 import cupy as cp
 zarr.config.enable_gpu()
-store = zarr.storage.MemoryStore()
 z = zarr.create_array(
-    store=store, shape=(100, 100), chunks=(10, 10), dtype="float32",
+    store="memory://gpu-demo", shape=(100, 100), chunks=(10, 10), dtype="float32",
 )
 type(z[:10, :10])
 # cupy.ndarray

docs/user-guide/groups.md

@@ -8,8 +8,7 @@ To create a group, use the [`zarr.group`][] function:
 
 ```python exec="true" session="groups" source="above" result="ansi"
 import zarr
-store = zarr.storage.MemoryStore()
-root = zarr.create_group(store=store)
+root = zarr.create_group(store="memory://groups-demo")
 print(root)
 ```
 
@@ -105,8 +104,7 @@ Diagnostic information about arrays and groups is available via the `info`
 property. E.g.:
 
 ```python exec="true" session="groups" source="above" result="ansi"
-store = zarr.storage.MemoryStore()
-root = zarr.group(store=store)
+root = zarr.group(store="memory://diagnostics-demo")
 foo = root.create_group('foo')
 bar = foo.create_array(name='bar', shape=1000000, chunks=100000, dtype='int64')
 bar[:] = 42

src/zarr/codecs/bytes.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import sys
+import warnings
 from dataclasses import dataclass, replace
 from enum import Enum
 from typing import TYPE_CHECKING
@@ -9,6 +10,7 @@
 from zarr.core.buffer import Buffer, NDBuffer
 from zarr.core.common import JSON, parse_enum, parse_named_configuration
 from zarr.core.dtype.common import HasEndianness
+from zarr.core.dtype.npy.structured import Struct
 
 if TYPE_CHECKING:
     from typing import Self
@@ -56,7 +58,20 @@ def to_dict(self) -> dict[str, JSON]:
             return {"name": "bytes", "configuration": {"endian": self.endian.value}}
 
     def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
-        if not isinstance(array_spec.dtype, HasEndianness):
+        if isinstance(array_spec.dtype, Struct):
+            if array_spec.dtype.has_multi_byte_fields():
+                if self.endian is None:
+                    warnings.warn(
+                        "Missing 'endian' for structured dtype with multi-byte fields. "
+                        "Assuming little-endian for legacy compatibility.",
+                        UserWarning,
+                        stacklevel=2,
+                    )
+                    return replace(self, endian=Endian.little)
+            else:
+                if self.endian is not None:
+                    return replace(self, endian=None)
+        elif not isinstance(array_spec.dtype, HasEndianness):
             if self.endian is not None:
                 return replace(self, endian=None)
         elif self.endian is None:

src/zarr/core/array.py

@@ -69,6 +69,7 @@
 )
 from zarr.core.config import config as zarr_config
 from zarr.core.dtype import (
+    Structured,
     VariableLengthBytes,
     VariableLengthUTF8,
     ZDType,
@@ -4879,10 +4880,13 @@ def default_serializer_v3(dtype: ZDType[Any, Any]) -> ArrayBytesCodec:
     length strings and variable length bytes have hard-coded serializers -- ``VLenUTF8Codec`` and
     ``VLenBytesCodec``, respectively.
 
+    Structured data types with multi-byte fields use ``BytesCodec`` with little-endian encoding.
     """
     serializer: ArrayBytesCodec = BytesCodec(endian=None)
 
-    if isinstance(dtype, HasEndianness):
+    if isinstance(dtype, HasEndianness) or (
+        isinstance(dtype, Structured) and dtype.has_multi_byte_fields()
+    ):
         serializer = BytesCodec(endian="little")
     elif isinstance(dtype, HasObjectCodec):
         if dtype.object_codec_id == "vlen-bytes":