Skip to content
Open
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
e3ee33b
add copy_store convenience method
melonora Dec 3, 2025
83550a3
add synchronous call
melonora Dec 3, 2025
d473d6e
change comment, test not async
melonora Dec 3, 2025
bb1405e
fix test
melonora Dec 3, 2025
c62543a
support zarr v2
melonora Dec 3, 2025
cdbc2f7
remove use of _iter_chunk_regions
melonora Dec 3, 2025
e6e10df
change method name
melonora Dec 3, 2025
9c42567
remove consolidate_metadata argument
melonora Dec 3, 2025
63c652e
consolidate if consolidated
melonora Dec 4, 2025
d4924f5
add consolidated_metadata argument
melonora Dec 5, 2025
e83dda5
add docstring and argument
melonora Dec 5, 2025
59b18ea
add support subgroup consolidated metadata
melonora Dec 5, 2025
b65d257
add argument to docstring
melonora Dec 5, 2025
1056b9e
add example to docs groups
melonora Dec 5, 2025
eadb647
adjust docs
melonora Dec 5, 2025
8c3471c
partial fix pre-commit
melonora Dec 5, 2025
2cbb9b9
add to changes
melonora Dec 5, 2025
128b924
change to call using self
melonora Dec 10, 2025
fa95e9c
obtain consolidated metadata from self
melonora Dec 10, 2025
848811f
add test for matching parameters
melonora Dec 10, 2025
3398325
implement path and add test
melonora Dec 11, 2025
ae205b5
Merge branch 'main' into copy_store
d-v-b Dec 16, 2025
06ad2d7
add type hints
melonora Dec 17, 2025
6b81a07
merge upstream changes
melonora Dec 17, 2025
3e92ee9
remove unnecessary type ignore
melonora Dec 17, 2025
0a4b5b8
Merge branch 'main' into copy_store
d-v-b Jan 8, 2026
418c670
Merge branch 'main' into copy_store
d-v-b Jan 8, 2026
f3e4f87
Merge branch 'main' into copy_store
d-v-b Jan 8, 2026
c37e69d
Merge branch 'main' into copy_store
d-v-b Feb 11, 2026
1abaab6
add failing tests
d-v-b Feb 11, 2026
8a1e9e9
use relative path provided by members iterator
d-v-b Feb 11, 2026
29c9dbb
copy raw bytes instead of decoding array chunks
d-v-b Feb 11, 2026
db7c120
Merge branch 'main' into copy_store
d-v-b Feb 12, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions src/zarr/core/group.py
Original file line number Diff line number Diff line change
Expand Up @@ -697,6 +697,69 @@ def from_dict(
store_path=store_path,
)

async def copy_to(
self,
store: StoreLike,
*,
overwrite: bool = False,
) -> AsyncGroup:
target_zarr_format = self.metadata.zarr_format

new_group = await AsyncGroup.from_store(
store,
overwrite=overwrite,
attributes=self.metadata.attributes,
zarr_format=target_zarr_format,
)

async for _, member in self.members(max_depth=None):
child_path = member.store_path.path
target_path = StorePath(store=new_group.store, path=child_path)

if isinstance(member, AsyncGroup):
await async_api.group(
store=target_path,
overwrite=overwrite,
attributes=member.metadata.attributes,
zarr_format=target_zarr_format,
)
else:
kwargs = {}
if target_zarr_format == 3:
kwargs["chunk_key_encoding"] = member.metadata.chunk_key_encoding
kwargs["dimension_names"] = member.metadata.dimension_names
else:
kwargs["chunk_key_encoding"] = {
"name": "v2",
"separator": member.metadata.dimension_separator,
}
# Serializer done this way in case of having zarr_format 2, otherwise mypy complains.
new_array = await new_group.create_array(
name=child_path,
shape=member.shape,
dtype=member.dtype,
chunks=member.chunks,
shards=member.shards,
filters=member.filters,
compressors=member.compressors,
serializer=member.serializer if member.serializer is not None else "auto",
fill_value=member.metadata.fill_value,
attributes=member.attrs,
overwrite=overwrite,
config={"order": member.order},
**kwargs,
)

for region in member._iter_shard_regions():
data = await member.getitem(selection=region)
await new_array.setitem(selection=region, value=data)

group = await self.open(self.store, zarr_format=target_zarr_format)
if group.metadata.consolidated_metadata:
await async_api.consolidate_metadata(new_group.store)
Comment thread
melonora marked this conversation as resolved.
Outdated

return new_group

async def setitem(self, key: str, value: Any) -> None:
"""
Fastpath for creating a new array
Expand Down Expand Up @@ -1874,6 +1937,14 @@ def open(
obj = sync(AsyncGroup.open(store, zarr_format=zarr_format))
return cls(obj)

def copy_to(
self,
store: StoreLike,
*,
overwrite: bool = False,
) -> Group:
return Group(sync(self._async_group.copy_to(store=store, overwrite=overwrite)))

def __getitem__(self, path: str) -> AnyArray | Group:
"""Obtain a group member.

Expand Down
53 changes: 53 additions & 0 deletions tests/test_group.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,59 @@ def test_group_members(store: Store, zarr_format: ZarrFormat, consolidated_metad
members_observed = group.members(max_depth=-1)


@pytest.mark.parametrize(
("zarr_format", "shards", "consolidate_metadata"),
[
(2, None, False),
(2, None, True),
(3, (50,), False),
(3, (50,), True),
],
)
def test_copy_to(zarr_format: int, shards: tuple[int, ...], consolidate_metadata: bool) -> None:
src_store = MemoryStore()
src = Group.from_store(src_store, attributes={"root": True}, zarr_format=zarr_format)

src.create_group("subgroup")

arr_data = np.arange(100)
src.create_array(
"dataset",
shape=(100,),
chunks=(10,),
shards=shards,
dtype=arr_data.dtype,
)
src["dataset"] = arr_data
if consolidate_metadata:
if zarr_format == 3:
with pytest.warns(ZarrUserWarning, match="Consolidated metadata is currently"):
zarr.consolidate_metadata(src_store)
else:
zarr.consolidate_metadata(src_store)

dst_store = MemoryStore()
if zarr_format == 3 and consolidate_metadata:
with pytest.warns(ZarrUserWarning, match="Consolidated metadata is currently"):
dst = src.copy_to(dst_store, overwrite=True)
else:
dst = src.copy_to(dst_store, overwrite=True)

assert dst.attrs.get("root") is True

subgroup = dst["subgroup"]
assert isinstance(subgroup, Group)

copied_arr = dst["dataset"]
copied_data = copied_arr[:]
assert np.array_equal(copied_data, arr_data)

if consolidate_metadata:
assert zarr.open_group(dst_store).metadata.consolidated_metadata
else:
assert not zarr.open_group(dst_store).metadata.consolidated_metadata


def test_group(store: Store, zarr_format: ZarrFormat) -> None:
"""
Test basic Group routines.
Expand Down