Move config from codec_pipeline -> array. Update docs, tests.

Author: williamsnell

changes/3748.feature.md

@@ -1 +1 @@
-Added `codec_pipeline.fill_missing_chunks` configuration option. When set to `False`, reading missing chunks raises a `MissingChunkError` instead of filling them with the fill value.
+Added `array.fill_missing_chunks` configuration option. When set to `False`, reading missing chunks raises a `MissingChunkError` instead of filling them with the array's fill value.

docs/user-guide/arrays.md

@@ -158,13 +158,18 @@ print(f"Shape after second append: {z.shape}")
 
 Zarr arrays are parametrized with a configuration that determines certain aspects of array behavior.
 
-We currently support two configuration options for arrays: `write_empty_chunks` and `order`.
+We currently support three configuration options for arrays: `write_empty_chunks`, `fill_missing_chunks`, and `order`.
 
 | field | type | default | description |
 | - |     - | - | - |
 | `write_empty_chunks` | `bool` | `False` | Controls whether empty chunks are written to storage. See [Empty chunks](performance.md#empty-chunks).
+| `fill_missing_chunks` | `bool` | `True` | Controls whether missing chunks are filled with the array's fill value on read. If `False`, reading missing chunks raises a `MissingChunkError`.
 | `order` | `Literal["C", "F"]` | `"C"` | The memory layout of arrays returned when reading data from the store.
 
+!!! note
+    `write_empty_chunks=False` skips writing chunks that are entirely the array's fill value.
+    If `fill_missing_chunks=False`, attempting to read these missing chunks will raise an error.
+
 You can specify the configuration when you create an array with the `config` keyword argument.
 `config` can be passed as either a `dict` or an `ArrayConfig` object.
 

docs/user-guide/config.md

@@ -30,7 +30,7 @@ Configuration options include the following:
 - Default Zarr format `default_zarr_version`
 - Default array order in memory `array.order`
 - Whether empty chunks are written to storage `array.write_empty_chunks`
-- Whether missing chunks are filled with the fill value on read `codec_pipeline.fill_missing_chunks` (default `True`). Set to `False` to raise a `MissingChunkError` instead.
+- Whether missing chunks are filled with the array's fill value on read `array.fill_missing_chunks` (default `True`). Set to `False` to raise a `MissingChunkError` instead.
 - Async and threading options, e.g. `async.concurrency` and `threading.max_workers`
 - Selections of implementations of codecs, codec pipelines and buffers
 - Enabling GPU support with `zarr.config.enable_gpu()`. See GPU support for more.

src/zarr/codecs/sharding.py

@@ -711,7 +711,7 @@ def _get_index_chunk_spec(self, chunks_per_shard: tuple[int, ...]) -> ArraySpec:
             dtype=UInt64(endianness="little"),
             fill_value=MAX_UINT_64,
             config=ArrayConfig(
-                order="C", write_empty_chunks=False
+                order="C", write_empty_chunks=False, fill_missing_chunks=True
             ),  # Note: this is hard-coded for simplicity -- it is not surfaced into user code,
             prototype=default_buffer_prototype(),
         )

src/zarr/core/array_spec.py

@@ -28,6 +28,7 @@ class ArrayConfigParams(TypedDict):
 
     order: NotRequired[MemoryOrder]
     write_empty_chunks: NotRequired[bool]
+    fill_missing_chunks: NotRequired[bool]
 
 
 @dataclass(frozen=True)
@@ -41,17 +42,25 @@ class ArrayConfig:
         The memory layout of the arrays returned when reading data from the store.
     write_empty_chunks : bool
         If True, empty chunks will be written to the store.
+    fill_missing_chunks : bool
+        If True, missing chunks will be filled with the array's fill value on read.
+        If False, reading missing chunks will raise a ``MissingChunkError``.
     """
 
     order: MemoryOrder
     write_empty_chunks: bool
+    fill_missing_chunks: bool
 
-    def __init__(self, order: MemoryOrder, write_empty_chunks: bool) -> None:
+    def __init__(
+        self, order: MemoryOrder, write_empty_chunks: bool, fill_missing_chunks: bool
+    ) -> None:
         order_parsed = parse_order(order)
         write_empty_chunks_parsed = parse_bool(write_empty_chunks)
+        fill_missing_chunks_parsed = parse_bool(fill_missing_chunks)
 
         object.__setattr__(self, "order", order_parsed)
         object.__setattr__(self, "write_empty_chunks", write_empty_chunks_parsed)
+        object.__setattr__(self, "fill_missing_chunks", fill_missing_chunks_parsed)
 
     @classmethod
     def from_dict(cls, data: ArrayConfigParams) -> Self:
@@ -62,7 +71,9 @@ def from_dict(cls, data: ArrayConfigParams) -> Self:
         """
         kwargs_out: ArrayConfigParams = {}
         for f in fields(ArrayConfig):
-            field_name = cast("Literal['order', 'write_empty_chunks']", f.name)
+            field_name = cast(
+                "Literal['order', 'write_empty_chunks', 'fill_missing_chunks']", f.name
+            )
             if field_name not in data:
                 kwargs_out[field_name] = zarr_config.get(f"array.{field_name}")
             else:
@@ -73,7 +84,11 @@ def to_dict(self) -> ArrayConfigParams:
         """
         Serialize an instance of this class to a dict.
         """
-        return {"order": self.order, "write_empty_chunks": self.write_empty_chunks}
+        return {
+            "order": self.order,
+            "write_empty_chunks": self.write_empty_chunks,
+            "fill_missing_chunks": self.fill_missing_chunks,
+        }
 
 
 ArrayConfigLike = ArrayConfig | ArrayConfigParams

src/zarr/core/codec_pipeline.py

@@ -265,7 +265,7 @@ async def read_batch(
                 if chunk_array is not None:
                     out[out_selection] = chunk_array
                 else:
-                    if config.get("codec_pipeline.fill_missing_chunks", True):
+                    if chunk_spec.config.fill_missing_chunks:
                         out[out_selection] = fill_value_or_default(chunk_spec)
                     else:
                         raise MissingChunkError
@@ -292,7 +292,7 @@ async def read_batch(
                         tmp = tmp.squeeze(axis=drop_axes)
                     out[out_selection] = tmp
                 else:
-                    if config.get("codec_pipeline.fill_missing_chunks", True):
+                    if chunk_spec.config.fill_missing_chunks:
                         out[out_selection] = fill_value_or_default(chunk_spec)
                     else:
                         raise MissingChunkError

src/zarr/core/config.py

@@ -96,6 +96,7 @@ def enable_gpu(self) -> ConfigSet:
             "array": {
                 "order": "C",
                 "write_empty_chunks": False,
+                "fill_missing_chunks": True,
                 "target_shard_size_bytes": None,
             },
             "async": {"concurrency": 10, "timeout": None},
@@ -104,7 +105,6 @@ def enable_gpu(self) -> ConfigSet:
             "codec_pipeline": {
                 "path": "zarr.core.codec_pipeline.BatchedCodecPipeline",
                 "batch_size": 1,
-                "fill_missing_chunks": True,
             },
             "codecs": {
                 "blosc": "zarr.codecs.blosc.BloscCodec",

tests/test_config.py

@@ -53,6 +53,7 @@ def test_config_defaults_set() -> None:
                 "array": {
                     "order": "C",
                     "write_empty_chunks": False,
+                    "fill_missing_chunks": True,
                     "target_shard_size_bytes": None,
                 },
                 "async": {"concurrency": 10, "timeout": None},
@@ -61,7 +62,6 @@ def test_config_defaults_set() -> None:
                 "codec_pipeline": {
                     "path": "zarr.core.codec_pipeline.BatchedCodecPipeline",
                     "batch_size": 1,
-                    "fill_missing_chunks": True,
                 },
                 "codecs": {
                     "blosc": "zarr.codecs.blosc.BloscCodec",
@@ -344,13 +344,13 @@ def test_config_fill_missing_chunks(store: Store, kwargs: dict[str, Any]) -> Non
     assert np.array_equal(result, np.full((4, 4), 42, dtype="int32"))
 
     # with fill_missing_chunks=False, reading missing chunks raises an error
-    with config.set({"codec_pipeline.fill_missing_chunks": False}):
+    with config.set({"array.fill_missing_chunks": False}):
         with pytest.raises(MissingChunkError):
             zarr.open_array(store)[:]
 
     # after writing data, all chunks exist and no error is raised
     arr[:] = np.arange(16, dtype="int32").reshape(4, 4)
-    with config.set({"codec_pipeline.fill_missing_chunks": False}):
+    with config.set({"array.fill_missing_chunks": False}):
         result = zarr.open_array(store)[:]
         assert np.array_equal(result, np.arange(16, dtype="int32").reshape(4, 4))