Allow Stores to opt out of consolidated metadata.

Some Stores don't benefit from Zarr's consolidated metadata mechanism.
These Stores usually implement their own consolidation mechanism, or
provide good performance for metadata retrieval out of the box.

These Stores can now implement the `supports_consolidated_metadata`
property returning `False`. In this situation, Zarr will silently ignore
any requests to consolidate the metadata.

Author: paraseba

docs/user-guide/consolidated_metadata.rst

@@ -114,3 +114,19 @@ removed, or modified, consolidated metadata may not be desirable.
    metadata.
 
 .. _Consolidated Metadata: https://github.com/zarr-developers/zarr-specs/pull/309
+
+Stores Without Support for Consolidated Metadata
+------------------------------------------------
+
+Some stores may want to opt out of the conolidated metadata mechanism. This
+may be for several reasons like:
+
+* They want to maintain read-write consistency, which is challenging with
+  consolidated metadata.
+* They have their own consolidated metadata mechanism.
+* They offer good enough performance without need for consolidation.
+
+This type of store can declare it doesn't want consolidation by implementing
+`Store.supports_consolidated_metadata`. For stores that don't support
+consolidation, Zarr will silently ignore any `consolidate_metadata` calls,
+maintainting the store in its unconsolidated state.

src/zarr/abc/store.py

@@ -264,6 +264,18 @@ async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None:
         """
         await gather(*starmap(self.set, values))
 
+    @property
+    def supports_consolidated_metadata(self) -> bool:
+        """
+        Does the store support and benefit from consolidated metadata?.
+
+        If it doesn't Zarr will ignore requests to consolidate the metadata.
+        Stores that would return `True` are the ones that implement their own
+        consolidation mechanism, that allows fast querying of metadata keys.
+        """
+
+        return True
+
     @property
     @abstractmethod
     def supports_deletes(self) -> bool:

src/zarr/api/asynchronous.py

@@ -174,7 +174,8 @@ async def consolidate_metadata(
     Consolidate the metadata of all nodes in a hierarchy.
 
     Upon completion, the metadata of the root node in the Zarr hierarchy will be
-    updated to include all the metadata of child nodes.
+    updated to include all the metadata of child nodes. For Stores that prefer
+    not to use consolidated metadata, this operation does nothing.
 
     Parameters
     ----------
@@ -194,11 +195,16 @@ async def consolidate_metadata(
     -------
     group: AsyncGroup
         The group, with the ``consolidated_metadata`` field set to include
-        the metadata of each child node.
+        the metadata of each child node. If the Store doesn't prefer
+        consolidated metadata, this is function does nothing and returns
+        the group without modifications. See ``Store.supports_consolidated_metadata``.
     """
     store_path = await make_store_path(store, path=path)
 
     group = await AsyncGroup.open(store_path, zarr_format=zarr_format, use_consolidated=False)
+    if not group.store_path.store.supports_consolidated_metadata:
+        return group
+
     group.store_path.store._check_writable()
 
     members_metadata = {

src/zarr/api/synchronous.py

@@ -81,7 +81,8 @@ def consolidate_metadata(
     Consolidate the metadata of all nodes in a hierarchy.
 
     Upon completion, the metadata of the root node in the Zarr hierarchy will be
-    updated to include all the metadata of child nodes.
+    updated to include all the metadata of child nodes. For Stores that prefer
+    not to use consolidated metadata, this operation does nothing.
 
     Parameters
     ----------
@@ -101,7 +102,10 @@ def consolidate_metadata(
     -------
     group: Group
         The group, with the ``consolidated_metadata`` field set to include
-        the metadata of each child node.
+        the metadata of each child node. If the Store doesn't prefer
+        consolidated metadata, this is function does nothing and returns
+        the group without modifications. See ``Store.supports_consolidated_metadata``.
+
     """
     return Group(sync(async_api.consolidate_metadata(store, path=path, zarr_format=zarr_format)))
 

src/zarr/core/group.py

@@ -492,8 +492,11 @@ async def open(
             store (in the ``zarr.json`` for Zarr format 3 and in the ``.zmetadata`` file
             for Zarr format 2).
 
-            To explicitly require consolidated metadata, set ``use_consolidated=True``,
-            which will raise an exception if consolidated metadata is not found.
+            To explicitly require consolidated metadata, set ``use_consolidated=True``.
+            If the Store supports consolidated metadata, this will raise an
+            exception if consolidated metadata is not found. If the Store doesn't want
+            to use consolidated metadata, we assume it implements its own consolidation,
+            so this is equivalent to use_consolidated=False.
 
             To explicitly *not* use consolidated metadata, set ``use_consolidated=False``,
             which will fall back to using the regular, non consolidated metadata.
@@ -503,6 +506,8 @@ async def open(
             to load consolidated metadata from a non-default key.
         """
         store_path = await make_store_path(store)
+        if not store_path.store.supports_consolidated_metadata:
+            use_consolidated = False
 
         consolidated_key = ZMETADATA_V2_JSON
 

tests/test_metadata/test_consolidated.py

@@ -17,14 +17,14 @@
     open,
     open_consolidated,
 )
-from zarr.core.buffer import cpu, default_buffer_prototype
+from zarr.core.buffer import Buffer, cpu, default_buffer_prototype
 from zarr.core.group import ConsolidatedMetadata, GroupMetadata
 from zarr.core.metadata import ArrayV3Metadata
 from zarr.core.metadata.v2 import ArrayV2Metadata
 from zarr.storage import StorePath
 
 if TYPE_CHECKING:
-    from zarr.abc.store import Store
+    from zarr.abc.store import ByteRequest, Store
     from zarr.core.common import ZarrFormat
 
 
@@ -651,3 +651,27 @@ async def test_consolidated_metadata_encodes_special_chars(
     elif zarr_format == 3:
         assert root_metadata["child"]["attributes"]["test"] == expected_fill_value
         assert root_metadata["time"]["fill_value"] == expected_fill_value
+
+
+async def test_consolidate_metadata_is_noop_for_self_consolidating_stores():
+    """Verify calling consolidate_metadata on a non supporting stores does nothing"""
+
+    # We create a store that doesn't support consolidated metadata
+    class Store(zarr.storage.MemoryStore):
+        @property
+        def supports_consolidated_metadata(self) -> bool:
+            return False
+
+    memory_store = Store()
+    root = await zarr.api.asynchronous.create_group(store=memory_store)
+    await root.create_group("a/b")
+
+    # now we monkey patch the store so it raises if `Store.set` is called
+    async def set_raises(self, value: Buffer, byte_range: ByteRequest | None = None) -> None:
+        raise ValueError("consolidated metadata called")
+
+    memory_store.set = set_raises
+
+    # consolidate_metadata would call `set` if the store supported consolidated metadata
+    # if this doesn't raise, it means consolidate_metadata is NOOP
+    await zarr.api.asynchronous.consolidate_metadata(memory_store)