Skip to content

Commit b11fd9c

Browse files
d-v-bd-v-b
committed
update to use new multiscales metadata
1 parent f2fcfb7 commit b11fd9c

12 files changed

Lines changed: 275 additions & 217 deletions

‎src/eopf_geozarr/data_api/geozarr/multiscales/__init__.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
"""Zarr multiscales convention support."""
2+
3+
from .geozarr import MultiscaleGroupAttrs, MultiscaleMeta
4+
from .zcm import MULTISCALE_CONVENTION_METADATA, Multiscales, ScaleLevel, ScaleLevelJSON
5+
6+
__all__ = [
7+
"MultiscaleGroupAttrs",
8+
"MultiscaleMeta",
9+
"MULTISCALE_CONVENTION_METADATA",
10+
"Multiscales",
11+
"ScaleLevel",
12+
"ScaleLevelJSON",
13+
]

‎src/eopf_geozarr/data_api/geozarr/multiscales/geozarr.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
from typing import Self
44

5-
from pydantic import BaseModel, computed_field, model_validator
5+
from pydantic import BaseModel, model_validator
66
from pydantic.experimental.missing_sentinel import MISSING
77
from typing_extensions import NotRequired, TypedDict
88

@@ -85,7 +85,6 @@ def valid_zcm_and_tms(self) -> Self:
8585
)
8686
return self
8787

88-
@computed_field # type: ignore[prop-decorator]
8988
@property
9089
def multiscale_meta(self) -> MultiscaleMetaDict:
9190
out: MultiscaleMetaDict = {}

‎src/eopf_geozarr/data_api/geozarr/v2.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,10 @@
1212
BaseDataArrayAttrs,
1313
DatasetAttrs,
1414
GridMappingAttrs,
15-
MultiscaleGroupAttrs,
1615
check_grid_mapping,
1716
check_valid_coordinates,
1817
)
18+
from eopf_geozarr.data_api.geozarr.multiscales import MultiscaleGroupAttrs
1919
from eopf_geozarr.data_api.geozarr.types import XARRAY_DIMS_KEY
2020

2121

‎src/eopf_geozarr/data_api/geozarr/v3.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@
1010
from eopf_geozarr.data_api.geozarr.common import (
1111
BaseDataArrayAttrs,
1212
DatasetAttrs,
13-
MultiscaleGroupAttrs,
1413
check_grid_mapping,
1514
check_valid_coordinates,
1615
)
16+
from eopf_geozarr.data_api.geozarr.multiscales import MultiscaleGroupAttrs
1717

1818

1919
class DataArray(ArraySpec[BaseDataArrayAttrs]):

‎src/eopf_geozarr/s2_optimization/s2_multiscale.py‎

Lines changed: 29 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
Uses lazy evaluation to minimize memory usage during dataset preparation.
44
"""
55

6+
from __future__ import annotations
7+
68
from collections.abc import Hashable, Mapping
79
from typing import Any, Literal
810

@@ -11,16 +13,18 @@
1113
import xarray as xr
1214
from dask import delayed
1315
from dask.array import from_delayed
16+
from pydantic.experimental.missing_sentinel import MISSING
1417
from pyproj import CRS
1518

1619
from eopf_geozarr.conversion import fs_utils
1720
from eopf_geozarr.conversion.geozarr import (
1821
_create_tile_matrix_limits,
1922
create_native_crs_tile_matrix_set,
2023
)
21-
from eopf_geozarr.data_api.geozarr.multiscales import (
22-
MULTISCALE_CONVENTION,
23-
ScaleLevelJSON,
24+
from eopf_geozarr.data_api.geozarr.multiscales import tms, zcm
25+
from eopf_geozarr.data_api.geozarr.multiscales.geozarr import (
26+
MultiscaleGroupAttrs,
27+
MultiscaleMeta,
2428
)
2529
from eopf_geozarr.data_api.geozarr.types import (
2630
XARRAY_ENCODING_KEYS,
@@ -491,6 +495,9 @@ def add_multiscales_metadata_to_parent(
491495
return
492496

493497
multiscales: dict[str, Any] = {"multiscales": {}}
498+
layout: list[zcm.ScaleLevel] | MISSING = MISSING
499+
tile_matrix_set: tms.TileMatrixSet | MISSING = MISSING
500+
tile_matrix_limits: dict[str, tms.TileMatrixLimit] | MISSING = MISSING
494501

495502
if "ogc_tms" in multiscales_flavor:
496503
# Create tile matrix set using geozarr function
@@ -514,34 +521,34 @@ def add_multiscales_metadata_to_parent(
514521
}
515522
)
516523
if "experimental_multiscales_convention" in multiscales_flavor:
517-
scale_levels: list[ScaleLevelJSON] = []
524+
layout = []
518525
for overview_level in overview_levels:
519-
scale_levels.append(
520-
{
521-
"asset": str(overview_level["level"]),
522-
"scale": (overview_level["scale_relative"],),
523-
"translation": (
524-
overview_level["translation_relative"],
525-
overview_level["translation_relative"],
526+
layout.append(
527+
zcm.ScaleLevel(
528+
asset=str(overview_level["level"]),
529+
derived_from=str(all_resolutions[0]),
530+
transform=zcm.Transform(
531+
scale=(overview_level["scale_relative"],) * 2,
532+
translation=(overview_level["translation_relative"],) * 2,
526533
),
527-
}
534+
)
528535
)
529-
multiscales["zarr_conventions_version"] = "0.1.0"
530-
multiscales["zarr_conventions"] = MULTISCALE_CONVENTION
531-
multiscales["multiscales"].update(
532-
{
533-
"layout": tuple(scale_levels),
534-
"resampling_method": "average",
535-
}
536-
)
536+
multiscale_attrs = MultiscaleGroupAttrs(
537+
zarr_conventions=(zcm.MultiscaleConventionMetadata(),),
538+
multiscales=MultiscaleMeta(
539+
layout=layout,
540+
resampling_method="average",
541+
tile_matrix_set=tile_matrix_set,
542+
tile_matrix_limits=tile_matrix_limits,
543+
),
544+
)
537545

538546
# Create parent group path
539547
parent_group_path = f"{output_path}{base_path}"
540548
dt_multiscale = xr.DataTree()
541549
for res in all_resolutions:
542550
dt_multiscale[res] = xr.DataTree()
543-
dt_multiscale.attrs.update(multiscales)
544-
551+
dt_multiscale.attrs.update(multiscale_attrs.model_dump())
545552
dt_multiscale.to_zarr(
546553
parent_group_path,
547554
mode="a",

‎tests/test_cli_e2e.py‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,13 +47,18 @@ def test_convert_s2_optimized(s2_group_example: Path, tmp_path: Path) -> None:
4747
# don't care about that here, so we convert all lists to tuples before creating the GroupSpec
4848
observed_structure = GroupSpec(**tuplify_json(observed_structure_json))
4949
observed_structure_flat = observed_structure.to_flat()
50+
expected_structure_path = Path(
51+
"tests/test_data_api/optimized_geozarr_examples/"
52+
) / (s2_group_example.stem + ".json")
53+
54+
# Uncomment this section to write out the expected structure from the observed structure
55+
# This is useful when the expected structure needs to be updated
56+
# expected_structure_path.write_text(
57+
# json.dumps(observed_structure_json, indent=2, sort_keys=True)
58+
# )
59+
5060
expected_structure_json = tuplify_json(
51-
json.loads(
52-
(
53-
Path("tests/test_data_api/optimized_geozarr_examples/")
54-
/ (s2_group_example.stem + ".json")
55-
).read_text()
56-
)
61+
json.loads(expected_structure_path.read_text())
5762
)
5863
expected_structure = GroupSpec(**expected_structure_json)
5964
expected_structure_flat = expected_structure.to_flat()

‎tests/test_data_api/conftest.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -202,9 +202,6 @@ def _load_multiscales_examples() -> dict[str, dict[str, object]]:
202202
}
203203

204204

205-
MULTISCALES_EXAMPLES = _load_multiscales_examples()
206-
207-
208205
def _load_json_examples(
209206
*, prefix: Path, glob_str: str = "*.json"
210207
) -> dict[str, dict[str, object]]:

‎tests/test_data_api/optimized_geozarr_examples/S2A_MSIL2A_20251008T100041_N0511_R122_T32TQM_20251008T122613.json‎

Lines changed: 72 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -4698,63 +4698,87 @@
46984698
"layout": [
46994699
{
47004700
"asset": "r10m",
4701-
"scale": [
4702-
0.0
4703-
],
4704-
"translation": [
4705-
-5.0,
4706-
-5.0
4707-
]
4701+
"derived_from": "r10m",
4702+
"transform": {
4703+
"scale": [
4704+
0.0,
4705+
0.0
4706+
],
4707+
"translation": [
4708+
-5.0,
4709+
-5.0
4710+
]
4711+
}
47084712
},
47094713
{
47104714
"asset": "r20m",
4711-
"scale": [
4712-
0.0
4713-
],
4714-
"translation": [
4715-
-5.0,
4716-
-5.0
4717-
]
4715+
"derived_from": "r10m",
4716+
"transform": {
4717+
"scale": [
4718+
0.0,
4719+
0.0
4720+
],
4721+
"translation": [
4722+
-5.0,
4723+
-5.0
4724+
]
4725+
}
47184726
},
47194727
{
47204728
"asset": "r60m",
4721-
"scale": [
4722-
0.0
4723-
],
4724-
"translation": [
4725-
-5.0,
4726-
-5.0
4727-
]
4729+
"derived_from": "r10m",
4730+
"transform": {
4731+
"scale": [
4732+
0.0,
4733+
0.0
4734+
],
4735+
"translation": [
4736+
-5.0,
4737+
-5.0
4738+
]
4739+
}
47284740
},
47294741
{
47304742
"asset": "r120m",
4731-
"scale": [
4732-
0.0
4733-
],
4734-
"translation": [
4735-
-5.0,
4736-
-5.0
4737-
]
4743+
"derived_from": "r10m",
4744+
"transform": {
4745+
"scale": [
4746+
0.0,
4747+
0.0
4748+
],
4749+
"translation": [
4750+
-5.0,
4751+
-5.0
4752+
]
4753+
}
47384754
},
47394755
{
47404756
"asset": "r360m",
4741-
"scale": [
4742-
0.0
4743-
],
4744-
"translation": [
4745-
-5.0,
4746-
-5.0
4747-
]
4757+
"derived_from": "r10m",
4758+
"transform": {
4759+
"scale": [
4760+
0.0,
4761+
0.0
4762+
],
4763+
"translation": [
4764+
-5.0,
4765+
-5.0
4766+
]
4767+
}
47484768
},
47494769
{
47504770
"asset": "r720m",
4751-
"scale": [
4752-
0.0
4753-
],
4754-
"translation": [
4755-
-5.0,
4756-
-5.0
4757-
]
4771+
"derived_from": "r10m",
4772+
"transform": {
4773+
"scale": [
4774+
0.0,
4775+
0.0
4776+
],
4777+
"translation": [
4778+
-5.0,
4779+
-5.0
4780+
]
4781+
}
47584782
}
47594783
],
47604784
"resampling_method": "average",
@@ -4893,16 +4917,15 @@
48934917
"title": "Native CRS Tile Matrix Set (EPSG:32632)"
48944918
}
48954919
},
4896-
"zarr_conventions": {
4897-
"d35379db-88df-4056-af3a-620245f8e347": {
4920+
"zarr_conventions": [
4921+
{
48984922
"description": "Multiscale layout of zarr datasets",
48994923
"name": "multiscales",
4900-
"schema": "https://raw.githubusercontent.com/zarr-conventions/multiscales/refs/tags/v0.1.0/schema.json",
4901-
"spec": "https://github.com/zarr-conventions/multiscales/blob/v0.1.0/README.md",
4902-
"version": "0.1.0"
4924+
"schema_url": "https://raw.githubusercontent.com/zarr-conventions/multiscales/refs/tags/v1/schema.json",
4925+
"spec_url": "https://github.com/zarr-conventions/multiscales/blob/v1/README.md",
4926+
"uuid": "d35379db-88df-4056-af3a-620245f8e347"
49034927
}
4904-
},
4905-
"zarr_conventions_version": "0.1.0"
4928+
]
49064929
},
49074930
"members": {
49084931
"r10m": {

0 commit comments

Comments
 (0)