fix(convert): 🐛 warn when fine chunks mean a million files

Capping the output chunk by the source's granularity is what stops the
conversion OOMing, but it has a cost worth naming: for the stitched-TIFF case
level 0 goes from ~60k chunks to ~950k, and without sharding a chunk is a
file. That lands on a shared filesystem at write time and on every read
afterwards.

Sharding fixes it without touching the chunking -- and safely, because
_auto_shard grows the two largest axes, which are y and x, so a shard still
lives inside one source plane and needs no more memory than a chunk did. For
the real dataset that is 4,536 files instead of 952,560.

So the write now says so, with the setting to change, rather than leaving it
to be discovered later.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: lguerard

src/patchworks/plugins/ome_zarr.py

@@ -466,6 +466,37 @@ def _one(idx: tuple[int, ...]) -> None:
             pass
 
 
+# One chunk is one file without sharding. A shared cluster filesystem starts
+# to hurt well before a million of them, and the cost lands on every later
+# read too, not just the write.
+_CHUNK_COUNT_WARN = 200_000
+
+
+def _warn_if_many_chunks(
+    shape: tuple[int, ...], chunks: tuple[int, ...], shard: ShardSpec
+) -> None:
+    """Warn when a level will be written as a very large number of files.
+
+    A coarse source forces fine output chunks (see :func:`_default_chunks`),
+    and fine chunks mean many files. Sharding packs them into far fewer
+    without changing the chunking, so the fix is cheap -- but only if someone
+    notices before the store is written.
+    """
+    if shard:
+        return
+    count = int(np.prod([math.ceil(s / c) for s, c in zip(shape, chunks)]))
+    if count < _CHUNK_COUNT_WARN:
+        return
+    logger.warning(
+        "level 0 will be %s chunks, i.e. that many files: the source's own "
+        "granularity forced chunks of %s. Set shard=True (config: "
+        "shard: true) to pack them into far fewer files -- same chunking, "
+        "same memory, and kinder to a shared filesystem.",
+        f"{count:,}",
+        chunks,
+    )
+
+
 def _bounded_scheduler(arr: da.Array):
     """Cap dask's threads by the memory one **source** chunk costs.
 
@@ -706,6 +737,7 @@ def _write_pyramid(
         base_chunks = chunks or _default_chunks(
             arr.shape, axes, source_chunks=arr.chunksize
         )
+        _warn_if_many_chunks(arr.shape, base_chunks, shard)
         _to_zarr_level(
             arr.rechunk(base_chunks), group_path, base_name, shard, progress
         )

tests/test_ome_zarr.py

@@ -550,3 +550,29 @@ def test_uncalibrated_tiff_reports_nothing(tmp_path):
     path = tmp_path / "bare.tif"
     tifffile.imwrite(path, np.zeros((8, 8), "uint16"))
     assert _tiff_pixel_size(str(path)) == {}
+
+
+def test_many_chunks_suggests_sharding(caplog):
+    """A coarse source forces fine chunks; fine chunks mean many files.
+
+    That trade-off is invisible until a shared filesystem is carrying a
+    million of them, so it has to be said at write time -- with the remedy.
+    """
+    import logging
+
+    from patchworks.plugins.ome_zarr import _warn_if_many_chunks
+
+    real = (4, 126, 45961, 42072)  # ~950k chunks at (1, 1, 1024, 1024)
+    with caplog.at_level(logging.WARNING, logger="patchworks.plugins.ome_zarr"):
+        _warn_if_many_chunks(real, (1, 1, 1024, 1024), shard=False)
+    assert "shard: true" in caplog.text
+    assert "952,560" in caplog.text
+
+    # Sharding already asked for, or a modest chunk count: nothing to say.
+    caplog.clear()
+    with caplog.at_level(logging.WARNING, logger="patchworks.plugins.ome_zarr"):
+        _warn_if_many_chunks(real, (1, 1, 1024, 1024), shard=True)
+        _warn_if_many_chunks(
+            (4, 126, 1024, 1024), (1, 16, 1024, 1024), shard=False
+        )
+    assert caplog.text == ""