Refactor storage clients to use obstore

Author: lukasbindreiter

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
       - id: end-of-file-fixer
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # keep the version here in sync with the version in uv.lock
-    rev: "v0.12.7"
+    rev: "v0.12.9"
     hooks:
       - id: ruff-check
         args: [--fix, --exit-non-zero-on-fix]

tilebox-storage/pyproject.toml

@@ -26,8 +26,7 @@ dependencies = [
     "aiofile>=3.8",
     "folium>=0.15",
     "shapely>=2",
-    "boto3>=1.33",
-    "boto3-stubs[essential]>=1.33",
+    "obstore>=0.8.0",
 ]
 
 [dependency-groups]
@@ -37,7 +36,6 @@ dev = [
     "pytest-asyncio>=0.24.0",
     "pytest-cov>=5.0.0",
     "pytest>=8.3.2",
-    "moto>=5",
 ]
 
 [project.urls]

tilebox-storage/tests/storage_data.py

@@ -11,7 +11,12 @@
 
 from hypothesis.strategies import DrawFn, booleans, composite, datetimes, integers, just, one_of, text, uuids
 
-from tilebox.storage.granule import ASFStorageGranule, CopernicusStorageGranule, UmbraStorageGranule
+from tilebox.storage.granule import (
+    ASFStorageGranule,
+    CopernicusStorageGranule,
+    UmbraStorageGranule,
+    USGSLandsatStorageGranule,
+)
 from tilebox.storage.providers import _ASF_URL, StorageURLs
 
 
@@ -82,3 +87,23 @@ def s5p_granules(draw: DrawFn) -> CopernicusStorageGranule:
 
     file_size = draw(integers(min_value=10_000, max_value=999_999_999))
     return CopernicusStorageGranule(start, granule_name, location, file_size)
+
+
+@composite
+def landsat_granules(draw: DrawFn) -> USGSLandsatStorageGranule:
+    """Generate a realistic-looking random USGS Landsat granule."""
+    time = draw(datetimes(min_value=datetime(1990, 1, 1), max_value=datetime(2025, 1, 1), timezones=just(None)))
+    landsat_mission = draw(integers(min_value=1, max_value=9))
+
+    path = draw(integers(min_value=1, max_value=999))
+    row = draw(integers(min_value=1, max_value=999))
+
+    granule_name = f"LC{landsat_mission:02d}_L1GT_{path:03d}{row:03d}_{time:%Y%m%d}_{time:%Y%m%d}_02_T1"
+    location = f"s3://usgs-landsat/collection02/level-1/standard/oli-tirs/{time:%Y}/{path:03d}/{row:03d}/{granule_name}"
+    thumbnail = draw(one_of(just(f"{granule_name}_thumb_small.jpeg"), just(None)))
+    return USGSLandsatStorageGranule(
+        time,
+        granule_name,
+        location,
+        thumbnail,
+    )

tilebox-storage/tests/test_granule.py

@@ -4,8 +4,14 @@
 from hypothesis import given
 from hypothesis.strategies import lists
 
-from tests.storage_data import ers_granules, s5p_granules, umbra_granules
-from tilebox.storage.granule import ASFStorageGranule, CopernicusStorageGranule, UmbraStorageGranule, _asf_download_urls
+from tests.storage_data import ers_granules, landsat_granules, s5p_granules, umbra_granules
+from tilebox.storage.granule import (
+    ASFStorageGranule,
+    CopernicusStorageGranule,
+    UmbraStorageGranule,
+    USGSLandsatStorageGranule,
+    _asf_download_urls,
+)
 
 
 def _asf_granule_to_datapoint(granule: ASFStorageGranule) -> xr.Dataset:
@@ -101,3 +107,31 @@ def test_granule_from_copernicus_datapoints(granules: list[CopernicusStorageGran
 
     for i in range(len(granules)):  # converting a dataset with a time dimension of 1 should still work though
         assert CopernicusStorageGranule.from_data(dataset.isel(time=i)) == granules[i]
+
+
+def _landsat_granule_to_datapoint(granule: USGSLandsatStorageGranule) -> xr.Dataset:
+    datapoint = xr.Dataset()
+    datapoint.coords["time"] = np.array(granule.time).astype("datetime64[ns]")
+    datapoint["granule_name"] = granule.granule_name
+    datapoint["location"] = granule.location
+    if granule.thumbnail is not None:
+        datapoint["thumbnail"] = f"{granule.location}/{granule.thumbnail}"
+    return datapoint
+
+
+@given(landsat_granules())
+def test_granule_from_landsat_datapoint(granule: USGSLandsatStorageGranule) -> None:
+    datapoint = _landsat_granule_to_datapoint(granule)
+    assert USGSLandsatStorageGranule.from_data(datapoint) == granule
+    assert USGSLandsatStorageGranule.from_data(USGSLandsatStorageGranule.from_data(datapoint)) == granule
+
+
+@given(lists(landsat_granules(), min_size=2, max_size=5))
+def test_granule_from_landsat_datapoints(granules: list[USGSLandsatStorageGranule]) -> None:
+    datapoints = [_landsat_granule_to_datapoint(granule) for granule in granules]
+    dataset = xr.concat(datapoints, dim="time")
+    with pytest.raises(ValueError, match=".*more than one granule.*"):
+        USGSLandsatStorageGranule.from_data(dataset)
+
+    for i in range(len(granules)):  # converting a dataset with a time dimension of 1 should still work though
+        assert USGSLandsatStorageGranule.from_data(dataset.isel(time=i)) == granules[i]

tilebox-storage/tests/test_providers.py

@@ -5,7 +5,7 @@
 from tilebox.storage.providers import _asf_login
 
 
-@pytest.mark.anyio
+@pytest.mark.asyncio
 async def test_asf_login(httpx_mock: HTTPXMock) -> None:
     httpx_mock.add_response(headers={"Set-Cookie": "logged_in=yes"})
 
@@ -15,8 +15,10 @@ async def test_asf_login(httpx_mock: HTTPXMock) -> None:
     assert isinstance(client.auth, BasicAuth)
     assert client.cookies["logged_in"] == "yes"
 
+    await client.aclose()
 
-@pytest.mark.anyio
+
+@pytest.mark.asyncio
 async def test_asf_login_invalid_auth(httpx_mock: HTTPXMock) -> None:
     httpx_mock.add_response(401)
     with pytest.raises(ValueError, match="Invalid username or password."):