Remove legacy datasets from python clients (#192)

* Remove geobuf

* Remove legacy datasets

Author: lukasbindreiter

.pre-commit-config.yaml

@@ -6,13 +6,13 @@ repos:
       - id: end-of-file-fixer
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # keep the version here in sync with the version in uv.lock
-    rev: "v0.11.5"
+    rev: "v0.11.12"
     hooks:
-      - id: ruff
+      - id: ruff-check
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/RobertCraigie/pyright-python
     # keep the version here in sync with the version in uv.lock
-    rev: v1.1.399
+    rev: v1.1.400
     hooks:
       - id: pyright

pyproject.toml

@@ -23,8 +23,9 @@ dev = [
     # DeprecationWarning: Pyarrow will become a required dependency of pandas in the next major release of pandas (pandas 3.0)
     "pyarrow>=17.0.0",
     # some dev tooling
-    "ruff>=0.6.4",
-    "pyright>=1.1.379",
+    "ruff>=0.11.10",
+    # pyright 1.1.401 reports many wrong false positives, let's wait until that is fixed before upgrading
+    "pyright>=1.1.379,<1.1.401",
     "pre-commit>=3.8.0",
     "types-protobuf>=5.27.0.20240907",
     "junitparser>=3.2.0",

tilebox-datasets/tests/data/datapoint.py

@@ -33,10 +33,8 @@
 from tests.example_dataset.example_dataset_pb2 import ExampleDatapoint
 from tilebox.datasets.data.datapoint import (
     AnyMessage,
-    Datapoint,
     DatapointInterval,
     DatapointIntervalLike,
-    DatapointPage,
     IngestResponse,
     QueryResultPage,
     RepeatedAny,
@@ -203,37 +201,6 @@ def datapoint_metadata_messages(draw: DrawFn) -> core_pb2.DatapointMetadata:
     return core_pb2.DatapointMetadata(event_time=event_time, ingestion_time=ingestion_time, id=data_point_id)
 
 
-@composite
-def datapoints(draw: DrawFn, generated_fields: bool = False, missing_fields: bool = False) -> Datapoint:
-    """A hypothesis strategy for generating random datapoints"""
-    meta = draw(datapoint_metadata_messages())
-    data = draw(anys(generated_fields, missing_fields))
-    return Datapoint(meta, data)
-
-
-@composite
-def datapoint_pages(
-    draw: DrawFn, empty_next_page: bool | None = None, generated_fields: bool = True, missing_fields: bool = False
-) -> DatapointPage:
-    """
-    A hypothesis strategy for generating random datapoint pages
-
-    Args:
-        empty_next_page: Whether the next page should be empty or not. If None, it will randomly be either an
-            empty or non-empty next page.
-        generated_fields: Whether to generate datapoints with all generated fields (id and ingestion_time) set as well.
-            If True, datapoints will have all meta fields set. If False, those fields will be set to None, similar
-            to how a datapoint for ingestion would look like.
-        missing_fields: Whether to generate datapoints with missing custom fields. If True, datapoints
-            will randomly have some fields missing. If False, all fields will be set with data.
-    """
-    meta = draw(lists(datapoint_metadata_messages(), min_size=1, max_size=5))
-    data = draw(repeated_anys(generated_fields, missing_fields, fixed_length=len(meta)))
-    next_page = draw(paginations(empty_next_page))
-    byte_size = sum(len(m) for m in data.value)
-    return DatapointPage(meta, data, next_page, byte_size)
-
-
 @composite
 def query_result_pages(
     draw: DrawFn, empty_next_page: bool | None = None, generated_fields: bool = True, missing_fields: bool = False
@@ -257,11 +224,11 @@ def query_result_pages(
 
 
 @composite
-def paginated_datapoint_for_interval_responses(draw: DrawFn) -> list[DatapointPage]:
+def paginated_query_results(draw: DrawFn) -> list[QueryResultPage]:
     """A hypothesis strategy for generating random datapoint pages for a time interval"""
     # let's generate a couple of pages, that each have a next page set, indicating that there are more pages
-    first_pages = draw(lists(datapoint_pages(empty_next_page=False), min_size=0, max_size=5))
-    last_page = draw(datapoint_pages(empty_next_page=True))
+    first_pages = draw(lists(query_result_pages(empty_next_page=False), min_size=0, max_size=5))
+    last_page = draw(query_result_pages(empty_next_page=True))
     return [*first_pages, last_page]
 
 

tilebox-datasets/tests/data/test_datapoint.py

@@ -4,18 +4,14 @@
     anys,
     datapoint_intervals,
     datapoint_intervals_like,
-    datapoint_pages,
-    datapoints,
     ingest_datapoints_responses,
     query_result_pages,
     repeated_anys,
 )
 from tilebox.datasets.data.datapoint import (
     AnyMessage,
-    Datapoint,
     DatapointInterval,
     DatapointIntervalLike,
-    DatapointPage,
     IngestResponse,
     QueryResultPage,
     RepeatedAny,
@@ -50,16 +46,6 @@ def test_repeated_anys_to_message_and_back(repeated_any: RepeatedAny) -> None:
     assert RepeatedAny.from_message(repeated_any.to_message()) == repeated_any
 
 
-@given(datapoints())
-def test_datapoints_to_message_and_back(datapoint: Datapoint) -> None:
-    assert Datapoint.from_message(datapoint.to_message()) == datapoint
-
-
-@given(datapoint_pages())
-def test_datapoint_pages_to_message_and_back(page: DatapointPage) -> None:
-    assert DatapointPage.from_message(page.to_message()) == page
-
-
 @given(query_result_pages())
 def test_query_result_pages_to_message_and_back(page: QueryResultPage) -> None:
     assert QueryResultPage.from_message(page.to_message()) == page

tilebox-datasets/tests/data/well_known_types.py

@@ -69,18 +69,6 @@ def latlonalt_messages(draw: DrawFn) -> well_known_types_pb2.LatLonAlt:
     )
 
 
-@composite
-def geobuf_messages(draw: DrawFn) -> well_known_types_pb2.GeobufData:
-    encoded_geobufs = [
-        # POLYGON ((-24.7917 80.1388, -20.6726 80.9065, -42.9612 82.8902, -45.6082 81.9369, -24.7917 80.1388))
-        b"\x10\x02\x18\x042\x1e\x08\x05\x12\x01\x04\x1a\x17\xd9\xa1\x1e\xd8\xe9a\xce\x83\x05\xfaw\xcb\x9a\x1b\xfa\xb5\x02\xcb\x9d\x03\xf9\x94\x01",
-        # POLYGON ((37.1165 82.5691, 60.97982545 85.05115, 3.218114686 85.05115, 3.0578 83.8312, 37.1165 82.5691))
-        b"\x10\x02\x18\t2/\x08\x05\x12\x01\x04\x1a(\xc0\xe0\x86\xc5\x94\x02\xc0\xeb\x83\x98\xe7\x04\x94\xcc\xeb\xe5\xb1\x01\xa0\xcf\x88\xbf\x12\xd7\x8a\xee\xad\xae\x03\x00\xfb\xd4\xf1\x98\x01\xdf\xd6\xb7\x8b\t",
-    ]
-    data = draw(sampled_from(encoded_geobufs))
-    return well_known_types_pb2.GeobufData.FromString(data)
-
-
 @composite
 def shapely_polygons(draw: DrawFn) -> Polygon:
     xmin = draw(floats(min_value=-180, max_value=160))

tilebox-datasets/tests/protobuf_conversion/test_geobuf.py

@@ -8,14 +8,10 @@
 from shapely import MultiPolygon, Polygon, from_wkb
 from xarray.testing import assert_equal
 
-from tests.data.datapoint import datapoint_pages, datapoints, example_datapoints
+from tests.data.datapoint import example_datapoints
 from tests.example_dataset.example_dataset_pb2 import ExampleDatapoint
-from tilebox.datasets.data.datapoint import Datapoint, DatapointPage
 from tilebox.datasets.data.time_interval import timestamp_to_datetime, us_to_datetime
-from tilebox.datasets.protobuf_conversion.protobuf_xarray import (
-    MessageToXarrayConverter,
-    TimeseriesToXarrayConverter,
-)
+from tilebox.datasets.protobuf_conversion.protobuf_xarray import MessageToXarrayConverter
 
 
 @given(example_datapoints(generated_fields=True, missing_fields=False))
@@ -102,20 +98,6 @@ def test_convert_datapoint(datapoint: ExampleDatapoint) -> None:  # noqa: PLR091
         assert isinstance(dataset.some_repeated_geometry[i].item(), Polygon | MultiPolygon)
 
 
-@given(datapoints(generated_fields=False, missing_fields=True))
-def test_convert_timeseries_datapoint(datapoint: Datapoint) -> None:
-    converter = TimeseriesToXarrayConverter()
-    converter.convert(datapoint)
-    dataset = converter.finalize()
-
-    assert dataset.sizes["time"] == 1
-    assert dataset.id.item() == datapoint.meta.id
-    event_time = dataset.time.item() // 1000
-    assert us_to_datetime(event_time) == timestamp_to_datetime(datapoint.meta.event_time)
-    ingestion_time = dataset.ingestion_time.item() // 1000
-    assert us_to_datetime(ingestion_time) == timestamp_to_datetime(datapoint.meta.ingestion_time)
-
-
 @given(lists(example_datapoints(generated_fields=True, missing_fields=True), min_size=5, max_size=30))
 def test_convert_datapoints(datapoints: list[ExampleDatapoint]) -> None:  # noqa: C901, PLR0912
     converter = MessageToXarrayConverter()
@@ -187,23 +169,6 @@ def test_convert_datapoints(datapoints: list[ExampleDatapoint]) -> None:  # noqa
             assert bytes_ is None or isinstance(bytes_, bytes)
 
 
-@given(datapoint_pages(empty_next_page=True, missing_fields=True))
-def test_convert_timeseries_datapoints(page: DatapointPage) -> None:
-    converter = TimeseriesToXarrayConverter()
-    converter.convert_all(page)
-    dataset = converter.finalize()
-
-    assert dataset.sizes["time"] == len(page.meta)
-    for i in range(len(page.meta)):
-        datapoint = dataset.isel(time=i)
-        meta = page.meta[i]
-        assert datapoint.id.item() == meta.id
-        event_time = datapoint.time.item() // 1000
-        assert us_to_datetime(event_time) == timestamp_to_datetime(meta.event_time)
-        ingestion_time = datapoint.ingestion_time.item() // 1000
-        assert us_to_datetime(ingestion_time) == timestamp_to_datetime(meta.ingestion_time)
-
-
 @given(lists(example_datapoints(missing_fields=True), min_size=1, max_size=10))
 @settings(max_examples=10)
 def test_convert_datapoints_all_at_once_or_one_by_one_same_result(

tilebox-datasets/tests/protobuf_conversion/test_protobuf_xarray.py

@@ -10,7 +10,7 @@
 from _tilebox.grpc.error import NotFoundError
 from _tilebox.grpc.replay import open_recording_channel, open_replay_channel
 from tilebox.datasets import Client, DatasetClient
-from tilebox.datasets.data.datapoint import DatapointPage
+from tilebox.datasets.data.datapoint import QueryResultPage
 from tilebox.datasets.data.time_interval import us_to_datetime
 
 
@@ -35,7 +35,7 @@ def record_client(recording_file: str) -> Client:
     # this will open a channel to api.tilebox.com, which will send real requests to the server, and record them
     # for later offline replay
     recording_channel = open_recording_channel(
-        "http://localhost:8083", os.environ["TILEBOX_OPENDATA_ONLY_API_KEY"], recording
+        "https://api.tilebox.com", os.environ["TILEBOX_OPENDATA_ONLY_API_KEY"], recording
     )
 
     with patch("tilebox.datasets.sync.client.open_channel") as open_channel_mock:
@@ -101,12 +101,11 @@ def test_find_datapoint() -> None:
     collection = s2_dataset.collection("S2A_S2MSI1C")
 
     for skip_data in (False, True):
-        datapoint = collection.find("0181f4ef-2040-1613-3eed-1c970dde6d2b", skip_data=skip_data)
+        datapoint = collection.find("0181f4ef-2040-13e7-ba1f-d5575e2a32a4", skip_data=skip_data)
         assert isinstance(datapoint, xr.Dataset)
 
-        assert datapoint.id.item() == "0181f4ef-2040-1613-3eed-1c970dde6d2b"
+        assert datapoint.id.item() == "0181f4ef-2040-13e7-ba1f-d5575e2a32a4"
         assert _dt(datapoint.time.item()) == datetime(2022, 7, 13, 0, 22, 1, 24000, tzinfo=timezone.utc)
-        assert _dt(datapoint.ingestion_time.item()) == datetime(2024, 7, 17, 4, 14, 29, 349263, tzinfo=timezone.utc)
 
         if not skip_data:
             assert datapoint.granule_name.item() == "S2A_MSIL1C_20220713T002201_N0400_R102_T08XNS_20220713T015332.SAFE"
@@ -128,47 +127,47 @@ def test_datapoint_not_found() -> None:
     collection = s2_dataset.collection("S2A_S2MSI1C")
 
     with pytest.raises(NotFoundError, match="No such datapoint.*"):
-        collection.find("0181f4dc-53c0-4912-acda-e35a368994fc")  # is in another collection
+        collection.find("0181f4ef-2040-101a-1423-d818e4d1895e")  # is in another collection
 
 
-def test_load_data() -> None:
-    client = replay_client("load_s2_data_interval.rpcs.bin")
+def test_query() -> None:
+    client = replay_client("query_sentinel2.rpcs.bin")
 
     s2_dataset = client.dataset("open_data.copernicus.sentinel2_msi")
     collection = s2_dataset.collection("S2A_S2MSI1C")
 
     for skip_data in (False, True):
-        data = collection.load(("2022-07-13", "2022-07-13T02:00"), skip_data=skip_data)
+        data = collection.query(temporal_extent=("2022-07-13", "2022-07-13T02:00"), skip_data=skip_data)
         assert isinstance(data, xr.Dataset)
 
         assert data.sizes["time"] == 756
-        assert data.id[0] == "0181f4ef-2040-0566-def5-50246aabcabc"
-        assert data.id[-1] == "0181f506-51c0-ffb7-20cb-a2ab4f5057cf"
+        assert data.id[0] == "0181f4ef-2040-1004-5540-5bca22067ac8"
+        assert data.id[-1] == "0181f506-51c0-a351-f295-6502e81f8ecf"
 
         if not skip_data:
-            assert data.granule_name[0] == "S2A_MSIL1C_20220713T002201_N0400_R102_T09XWK_20220713T015332.SAFE"
-            assert data.granule_name[-1] == "S2A_MSIL1C_20220713T004721_N0400_R102_T53HNA_20220713T021615.SAFE"
+            assert data.granule_name[0] == "S2A_MSIL1C_20220713T002201_N0400_R102_T09XWL_20220713T015332.SAFE"
+            assert data.granule_name[-1] == "S2A_MSIL1C_20220713T004721_N0400_R102_T53HPV_20220713T021615.SAFE"
         else:
             assert "granule_name" not in data
 
 
-def test_load_data_pagination() -> None:
-    client = replay_client("load_s2_data_interval_paging.rpcs.bin")
+def test_query_pagination() -> None:
+    client = replay_client("query_sentinel2_paging.rpcs.bin")
 
     s2_dataset = client.dataset("open_data.copernicus.sentinel2_msi")
     collection = s2_dataset.collection("S2A_S2MSI1C")
 
-    pages = list(collection._iter_pages_legacy(("2022-07-13", "2022-07-13T02:00"), page_size=10))
+    pages = list(collection._iter_pages(("2022-07-13", "2022-07-13T02:00"), page_size=10))
 
     assert len(pages) == 76  # we have 756 datapoints, so 76 pages, and the last page has only 6 datapoints
 
     for i, page in enumerate(pages):
-        assert isinstance(page, DatapointPage)
-        assert page.meta[0].id >= "0181f4ef-2040-0566-def5-50246aabcabc"
-        assert page.meta[-1].id <= "0181f506-51c0-ffb7-20cb-a2ab4f5057cf"
+        assert isinstance(page, QueryResultPage)
+        assert str(page.min_id) >= "0181f4ef-2040-1004-5540-5bca22067ac8"
+        assert str(page.max_id) <= "0181f506-51c0-a351-f295-6502e81f8ecf"
         is_last_page = i == len(pages) - 1
         expected_len = 6 if is_last_page else 10
-        assert len(page.meta) == expected_len
+        assert page.n_datapoints == expected_len
 
 
 def _dt(timestamp_nano: int) -> datetime: