Merge branch 'main' into store-move

Author: brokkoli71

changes/2950.bufgix.rst

@@ -0,0 +1 @@
+Specifying the memory order of Zarr format 2 arrays using the ``order`` keyword argument has been fixed.

changes/2998.bugfix.md

@@ -0,0 +1 @@
+Fix structured `dtype` fill value serialization for consolidated metadata

changes/3027.misc.rst

@@ -0,0 +1 @@
+Simplified scalar indexing of size-1 arrays.

src/zarr/api/asynchronous.py

@@ -1040,15 +1040,13 @@ async def create(
             )
             warnings.warn(UserWarning(msg), stacklevel=1)
         config_dict["write_empty_chunks"] = write_empty_chunks
-    if order is not None:
-        if config is not None:
-            msg = (
-                "Both order and config keyword arguments are set. "
-                "This is redundant. When both are set, order will be ignored and "
-                "config will be used."
-            )
-            warnings.warn(UserWarning(msg), stacklevel=1)
-        config_dict["order"] = order
+    if order is not None and config is not None:
+        msg = (
+            "Both order and config keyword arguments are set. "
+            "This is redundant. When both are set, order will be ignored and "
+            "config will be used."
+        )
+        warnings.warn(UserWarning(msg), stacklevel=1)
 
     config_parsed = ArrayConfig.from_dict(config_dict)
 
@@ -1062,6 +1060,7 @@ async def create(
         overwrite=overwrite,
         filters=filters,
         dimension_separator=dimension_separator,
+        order=order,
         zarr_format=zarr_format,
         chunk_shape=chunk_shape,
         chunk_key_encoding=chunk_key_encoding,

src/zarr/core/array.py

@@ -609,6 +609,7 @@ async def _create(
 
             if order is not None:
                 _warn_order_kwarg()
+                config_parsed = replace(config_parsed, order=order)
 
             result = await cls._create_v3(
                 store_path,
@@ -1044,7 +1045,10 @@ def order(self) -> MemoryOrder:
         bool
             Memory order of the array
         """
-        return self._config.order
+        if self.metadata.zarr_format == 2:
+            return self.metadata.order
+        else:
+            return self._config.order
 
     @property
     def attrs(self) -> dict[str, JSON]:
@@ -1276,14 +1280,14 @@ async def _get_selection(
             out_buffer = prototype.nd_buffer.create(
                 shape=indexer.shape,
                 dtype=out_dtype,
-                order=self._config.order,
+                order=self.order,
                 fill_value=self.metadata.fill_value,
             )
         if product(indexer.shape) > 0:
             # need to use the order from the metadata for v2
             _config = self._config
             if self.metadata.zarr_format == 2:
-                _config = replace(_config, order=self.metadata.order)
+                _config = replace(_config, order=self.order)
 
             # reading chunks and decoding them
             await self.codec_pipeline.read(
@@ -4256,6 +4260,11 @@ async def init_array(
             chunks_out = chunk_shape_parsed
             codecs_out = sub_codecs
 
+        if config is None:
+            config = {}
+        if order is not None and isinstance(config, dict):
+            config["order"] = config.get("order", order)
+
         meta = AsyncArray._create_metadata_v3(
             shape=shape_parsed,
             dtype=dtype_parsed,

src/zarr/core/buffer/core.py

@@ -427,16 +427,7 @@ def as_scalar(self) -> ScalarType:
         """Returns the buffer as a scalar value"""
         if self._data.size != 1:
             raise ValueError("Buffer does not contain a single scalar value")
-        item = self.as_numpy_array().item()
-        scalar: ScalarType
-
-        if np.issubdtype(self.dtype, np.datetime64):
-            unit: str = np.datetime_data(self.dtype)[0]  # Extract the unit (e.g., 'Y', 'D', etc.)
-            scalar = np.datetime64(item, unit)
-        else:
-            scalar = self.dtype.type(item)  # Regular conversion for non-datetime types
-
-        return scalar
+        return cast(ScalarType, self.as_numpy_array()[()])
 
     @property
     def dtype(self) -> np.dtype[Any]:

src/zarr/core/group.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import base64
 import itertools
 import json
 import logging
@@ -358,7 +359,13 @@ def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]:
                     d[f"{k}/{ZATTRS_JSON}"] = _replace_special_floats(attrs)
                     if "shape" in v:
                         # it's an array
-                        d[f"{k}/{ZARRAY_JSON}"] = _replace_special_floats(v)
+                        if isinstance(v.get("fill_value", None), np.void):
+                            v["fill_value"] = base64.standard_b64encode(
+                                cast(bytes, v["fill_value"])
+                            ).decode("ascii")
+                        else:
+                            v = _replace_special_floats(v)
+                        d[f"{k}/{ZARRAY_JSON}"] = v
                     else:
                         d[f"{k}/{ZGROUP_JSON}"] = {
                             "zarr_format": self.zarr_format,

tests/test_api.py

@@ -326,13 +326,12 @@ def test_array_order(zarr_format: ZarrFormat) -> None:
 def test_array_order_warns(order: MemoryOrder | None, zarr_format: ZarrFormat) -> None:
     with pytest.warns(RuntimeWarning, match="The `order` keyword argument .*"):
         arr = zarr.ones(shape=(2, 2), order=order, zarr_format=zarr_format)
-    expected = order or zarr.config.get("array.order")
-    assert arr.order == expected
+    assert arr.order == order
 
     vals = np.asarray(arr)
-    if expected == "C":
+    if order == "C":
         assert vals.flags.c_contiguous
-    elif expected == "F":
+    elif order == "F":
         assert vals.flags.f_contiguous
     else:
         raise AssertionError

tests/test_array.py

@@ -1262,39 +1262,43 @@ async def test_data_ignored_params(store: Store) -> None:
             await create_array(store, data=data, shape=None, dtype=data.dtype, overwrite=True)
 
     @staticmethod
-    @pytest.mark.parametrize("order_config", ["C", "F", None])
+    @pytest.mark.parametrize("order", ["C", "F", None])
+    @pytest.mark.parametrize("with_config", [True, False])
     def test_order(
-        order_config: MemoryOrder | None,
+        order: MemoryOrder | None,
+        with_config: bool,
         zarr_format: ZarrFormat,
         store: MemoryStore,
     ) -> None:
         """
         Test that the arrays generated by array indexing have a memory order defined by the config order
         value, and that for zarr v2 arrays, the ``order`` field in the array metadata is set correctly.
         """
-        config: ArrayConfigLike = {}
-        if order_config is None:
+        config: ArrayConfigLike | None = {}
+        if order is None:
             config = {}
             expected = zarr.config.get("array.order")
         else:
-            config = {"order": order_config}
-            expected = order_config
+            config = {"order": order}
+            expected = order
+
+        if not with_config:
+            # Test without passing config parameter
+            config = None
+
+        arr = zarr.create_array(
+            store=store,
+            shape=(2, 2),
+            zarr_format=zarr_format,
+            dtype="i4",
+            order=order,
+            config=config,
+        )
+        assert arr.order == expected
         if zarr_format == 2:
-            arr = zarr.create_array(
-                store=store,
-                shape=(2, 2),
-                zarr_format=zarr_format,
-                dtype="i4",
-                order=expected,
-                config=config,
-            )
-            # guard for type checking
             assert arr.metadata.zarr_format == 2
             assert arr.metadata.order == expected
-        else:
-            arr = zarr.create_array(
-                store=store, shape=(2, 2), zarr_format=zarr_format, dtype="i4", config=config
-            )
+
         vals = np.asarray(arr)
         if expected == "C":
             assert vals.flags.c_contiguous

tests/test_buffer.py

@@ -155,3 +155,9 @@ def test_numpy_buffer_prototype() -> None:
     assert isinstance(ndbuffer.as_ndarray_like(), np.ndarray)
     with pytest.raises(ValueError, match="Buffer does not contain a single scalar value"):
         ndbuffer.as_scalar()
+
+
+# TODO: the same test for other buffer classes
+def test_cpu_buffer_as_scalar() -> None:
+    buf = cpu.buffer_prototype.nd_buffer.create(shape=(), dtype="int64")
+    assert buf.as_scalar() == buf.as_ndarray_like()[()]  # type: ignore[index]