Support latest protobuf version

Author: lukasbindreiter

tilebox-datasets/tilebox/datasets/protobuf_conversion/protobuf_xarray.py

@@ -374,19 +374,16 @@ def _create_field_converter(field: FieldDescriptor) -> _FieldConverter:
     """
     # special handling for enums:
     if field.type == FieldDescriptor.TYPE_ENUM:
-        if field.label == FieldDescriptor.LABEL_REPEATED:
+        if field.is_repeated:
             raise NotImplementedError("Repeated enum fields are not supported")
 
         return _EnumFieldConverter(field.name, enum_mapping_from_field_descriptor(field))
 
     field_type = infer_field_type(field)
-    if field.label == FieldDescriptor.LABEL_OPTIONAL:  # simple fields (in proto3 every simple field is optional)
-        return _SimpleFieldConverter(field.name, field_type)
-
-    if field.label == FieldDescriptor.LABEL_REPEATED:
+    if field.is_repeated:
         return _ArrayFieldConverter(field.name, field_type)
 
-    raise ValueError(f"Unsupported field type with label {field.label} and type {field.type}")
+    return _SimpleFieldConverter(field.name, field_type)
 
 
 def _combine_dimension_names(array_dimensions: dict[str, int]) -> dict[str, tuple[str, int]]:

tilebox-datasets/tilebox/datasets/protobuf_conversion/to_protobuf.py

@@ -6,7 +6,6 @@
 import numpy as np
 import pandas as pd
 import xarray as xr
-from google.protobuf.descriptor import FieldDescriptor
 from google.protobuf.message import Message
 
 from tilebox.datasets.protobuf_conversion.field_types import (
@@ -80,7 +79,7 @@ def to_messages(  # noqa: C901, PLR0912
         descriptor = field_descriptors_by_name[field_name]
         field_type = infer_field_type(descriptor)
 
-        if descriptor.label == FieldDescriptor.LABEL_REPEATED:
+        if descriptor.is_repeated:
             values = convert_repeated_values_to_proto(values, field_type)
         else:
             values = convert_values_to_proto(values, field_type, filter_none=False)

tilebox-storage/tests/test_storage_client.py

@@ -9,9 +9,20 @@
 from obstore.store import LocalStore
 from pytest_httpx import HTTPXMock, IteratorStream
 
-from tests.storage_data import ers_granules, s5p_granules, umbra_granules
-from tilebox.storage.aio import ASFStorageClient, CopernicusStorageClient, UmbraStorageClient, _HttpClient
-from tilebox.storage.granule import ASFStorageGranule, CopernicusStorageGranule, UmbraStorageGranule
+from tests.storage_data import ers_granules, landsat_granules, s5p_granules, umbra_granules
+from tilebox.storage.aio import (
+    ASFStorageClient,
+    CopernicusStorageClient,
+    UmbraStorageClient,
+    USGSLandsatStorageClient,
+    _HttpClient,
+)
+from tilebox.storage.granule import (
+    ASFStorageGranule,
+    CopernicusStorageGranule,
+    UmbraStorageGranule,
+    USGSLandsatStorageGranule,
+)
 
 
 @pytest.mark.asyncio
@@ -277,3 +288,74 @@ async def test_copernicus_storage_client_download_objects(granule: CopernicusSto
         assert folder.exists()
         assert (folder / granule.granule_name).read_bytes() == b"content1"
         assert not (folder / f"other_product_{granule.granule_name}").exists()
+
+
+@pytest.mark.asyncio
+@given(landsat_granules())
+@settings(max_examples=1, deadline=timedelta(milliseconds=100))
+async def test_landsat_storage_client_download(granule: USGSLandsatStorageGranule) -> None:
+    with TemporaryDirectory(delete=True) as tmp_path:
+        store_path = Path(tmp_path) / "store"
+        store_path.mkdir(exist_ok=True, parents=True)
+        store = LocalStore(store_path)
+
+        await store.put_async(
+            f"{granule.location.removeprefix('s3://usgs-landsat/')}/{granule.granule_name}", b"content1"
+        )
+        with patch("tilebox.storage.aio.S3Store") as store_mock:
+            store_mock.return_value = store
+            landsat = USGSLandsatStorageClient(cache_directory=Path(tmp_path))
+
+        folder = await landsat.download(granule, show_progress=False)
+        assert folder.exists()
+        assert (folder / granule.granule_name).read_bytes() == b"content1"
+
+
+@pytest.mark.asyncio
+@given(landsat_granules())
+@settings(max_examples=1, deadline=timedelta(milliseconds=100))
+async def test_landsat_storage_client_list_objects(granule: USGSLandsatStorageGranule) -> None:
+    with TemporaryDirectory(delete=True) as tmp_path:
+        store_path = Path(tmp_path) / "store"
+        store_path.mkdir(exist_ok=True, parents=True)
+        store = LocalStore(store_path)
+
+        await store.put_async(
+            f"{granule.location.removeprefix('s3://usgs-landsat/')}/{granule.granule_name}", b"content1"
+        )
+        await store.put_async(
+            f"{granule.location.removeprefix('s3://usgs-landsat/')}/{granule.granule_name}_thumb_small.jpeg",
+            b"content2",
+        )
+        with patch("tilebox.storage.aio.S3Store") as store_mock:
+            store_mock.return_value = store
+            landsat = USGSLandsatStorageClient(cache_directory=Path(tmp_path))
+
+        objects = await landsat.list_objects(granule)
+        assert len(objects) == 2
+        assert sorted(objects) == sorted([granule.granule_name, f"{granule.granule_name}_thumb_small.jpeg"])
+
+
+@pytest.mark.asyncio
+@given(landsat_granules())
+@settings(max_examples=1, deadline=timedelta(milliseconds=100))
+async def test_landsat_storage_client_download_objects(granule: USGSLandsatStorageGranule) -> None:
+    with TemporaryDirectory(delete=True) as tmp_path:
+        store_path = Path(tmp_path) / "store"
+        store_path.mkdir(exist_ok=True, parents=True)
+        store = LocalStore(store_path)
+
+        await store.put_async(
+            f"{granule.location.removeprefix('s3://usgs-landsat/')}/{granule.granule_name}", b"content1"
+        )
+        await store.put_async(
+            f"{granule.location.removeprefix('s3://usgs-landsat/')}/other_product_{granule.granule_name}", b"content2"
+        )
+        with patch("tilebox.storage.aio.S3Store") as store_mock:
+            store_mock.return_value = store
+            landsat = USGSLandsatStorageClient(cache_directory=Path(tmp_path))
+
+        folder = await landsat.download_objects(granule, [granule.granule_name], show_progress=False)
+        assert folder.exists()
+        assert (folder / granule.granule_name).read_bytes() == b"content1"
+        assert not (folder / f"other_product_{granule.granule_name}").exists()