Delegate missing-shard detection away from _get_chunk_spec. Codify

expected behaviour of fill_missing_chunks for both sharding and
write_empty_chunks via tests. Use elif to make control flow slightly
clearer.

Author: williamsnell

src/zarr/codecs/sharding.py

@@ -717,11 +717,16 @@ def _get_index_chunk_spec(self, chunks_per_shard: tuple[int, ...]) -> ArraySpec:
         )
 
     def _get_chunk_spec(self, shard_spec: ArraySpec) -> ArraySpec:
+        # Because the shard index and inner chunks should be stored
+        # together, we detect missing data via the shard index.
+        # The inner chunks defined here are thus allowed to return
+        # None, even if fill_missing_chunks=False at the array level.
+        config = replace(shard_spec.config, fill_missing_chunks=True)
         return ArraySpec(
             shape=self.chunk_shape,
             dtype=shard_spec.dtype,
             fill_value=shard_spec.fill_value,
-            config=shard_spec.config,
+            config=config,
             prototype=shard_spec.prototype,
         )
 

src/zarr/core/codec_pipeline.py

@@ -264,11 +264,10 @@ async def read_batch(
             ):
                 if chunk_array is not None:
                     out[out_selection] = chunk_array
+                elif chunk_spec.config.fill_missing_chunks:
+                    out[out_selection] = fill_value_or_default(chunk_spec)
                 else:
-                    if chunk_spec.config.fill_missing_chunks:
-                        out[out_selection] = fill_value_or_default(chunk_spec)
-                    else:
-                        raise MissingChunkError
+                    raise MissingChunkError
         else:
             chunk_bytes_batch = await concurrent_map(
                 [(byte_getter, array_spec.prototype) for byte_getter, array_spec, *_ in batch_info],
@@ -291,11 +290,10 @@ async def read_batch(
                     if drop_axes != ():
                         tmp = tmp.squeeze(axis=drop_axes)
                     out[out_selection] = tmp
+                elif chunk_spec.config.fill_missing_chunks:
+                    out[out_selection] = fill_value_or_default(chunk_spec)
                 else:
-                    if chunk_spec.config.fill_missing_chunks:
-                        out[out_selection] = fill_value_or_default(chunk_spec)
-                    else:
-                        raise MissingChunkError
+                    raise MissingChunkError
 
     def _merge_chunk_array(
         self,

tests/test_config.py

@@ -355,6 +355,65 @@ def test_config_fill_missing_chunks(store: Store, kwargs: dict[str, Any]) -> Non
         assert np.array_equal(result, np.arange(16, dtype="int32").reshape(4, 4))
 
 
+@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"])
+def test_config_fill_missing_chunks_sharded_inner(store: Store) -> None:
+    """Missing inner chunks within a shard are always filled with the array's
+    fill value, even when fill_missing_chunks=False."""
+    arr = zarr.create_array(
+        store=store,
+        shape=(8, 4),
+        chunks=(2, 2),
+        shards=(4, 4),
+        dtype="int32",
+        fill_value=42,
+    )
+
+    # write only one inner chunk in the first shard, leaving the second shard empty
+    arr[0:2, 0:2] = np.ones((2, 2), dtype="int32")
+
+    with config.set({"array.fill_missing_chunks": False}):
+        a = zarr.open_array(store)
+
+        # first shard exists: missing inner chunks are filled, no error
+        result = a[:4]
+        expected = np.full((4, 4), 42, dtype="int32")
+        expected[0:2, 0:2] = 1
+        assert np.array_equal(result, expected)
+
+        # second shard is entirely missing: raises an error
+        with pytest.raises(MissingChunkError):
+            a[4:]
+
+
+@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"])
+def test_config_fill_missing_chunks_write_empty_chunks(store: Store) -> None:
+    """write_empty_chunks=False drops chunks equal to fill_value, which then
+    appear missing to fill_missing_chunks=False."""
+    arr = zarr.create_array(
+        store=store,
+        shape=(4,),
+        chunks=(2,),
+        dtype="int32",
+        fill_value=0,
+        config={"write_empty_chunks": False, "fill_missing_chunks": False},
+    )
+
+    # write non-fill-value data: chunks are stored
+    arr[:] = [1, 2, 3, 4]
+    assert np.array_equal(arr[:], [1, 2, 3, 4])
+
+    # overwrite with fill_value: chunks are dropped by write_empty_chunks=False
+    arr[:] = 0
+    with pytest.raises(MissingChunkError):
+        arr[:]
+
+    # with write_empty_chunks=True, chunks are kept and no error is raised
+    with config.set({"array.write_empty_chunks": True}):
+        arr = zarr.open_array(store)
+        arr[:] = 0
+        assert np.array_equal(arr[:], [0, 0, 0, 0])
+
+
 @pytest.mark.parametrize(
     "key",
     [