Skip to content

Commit cc08076

Browse files
committed
Update to avoid sharding structured dtypes
1 parent c728bf5 commit cc08076

2 files changed

Lines changed: 14 additions & 4 deletions

File tree

src/mdio/builder/xarray_builder.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
"""Convert MDIO v1 schema Dataset to Xarray DataSet and write it in Zarr."""
22

3+
import logging
4+
35
import numcodecs
46
import numpy as np
57
import zarr
@@ -25,6 +27,8 @@
2527
from mdio.converters.type_converter import to_numpy_dtype
2628
from mdio.core.zarr_io import zarr_warnings_suppress_unstable_numcodecs_v3
2729

30+
logger = logging.getLogger(__name__)
31+
2832

2933
def _import_numcodecs_zfpy() -> "type[numcodecs.ZFPY]":
3034
"""Helper to import the optional dependency at runtime."""
@@ -250,6 +254,14 @@ def to_xarray_dataset(mdio_ds: Dataset) -> xr_Dataset: # noqa: PLR0912, PLR0915
250254
# When sharding is configured for Zarr v3 but variable has non-shardable dtype,
251255
# use shard shape as chunk shape to maintain consistent I/O patterns
252256
if shard_shape is not None and zarr_format == ZarrFormat.V3 and is_non_shardable:
257+
dtype_desc = "structured" if isinstance(v.data_type, StructuredType) else "void/bytes"
258+
logger.warning(
259+
"Sharding is not supported for %s dtypes. Variable '%s' will use regular "
260+
"chunking with chunk shape %s (shard shape) instead of sharding.",
261+
dtype_desc,
262+
v.name,
263+
shard_shape,
264+
)
253265
chunks_to_use = shard_shape
254266
else:
255267
chunks_to_use = original_chunks

src/mdio/converters/segy.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -607,11 +607,9 @@ def enhanced_add_variables() -> None:
607607
# Call the original method first
608608
original_add_variables()
609609

610-
# Now add the raw headers variable
611-
chunk_shape = mdio_template.full_chunk_shape[:-1]
612-
613610
# Create chunk grid metadata using template's method to respect sharding config
614-
chunk_grid = mdio_template._create_chunk_grid(chunk_shape)
611+
# exclude_vertical=True because raw_headers don't have the sample dimension
612+
chunk_grid = mdio_template._create_chunk_grid(exclude_vertical=True)
615613

616614
# Add the raw headers variable using the builder's add_variable method
617615
mdio_template._builder.add_variable(

0 commit comments

Comments
 (0)