refactor Slice methods to avoid fetching all items (#180)

* deprecate client.slice_info

* add slice.summary for shortform info (+test)

* add name and items, refactor info, fix tests

- add name property
- add items property
- refactor old info as _fetch_all
- refactor summary as new info
- refactor tests relying on Slice.info

* lint

* fix broken Slice.info calls

* pylint

* mypy...

* isort...

* update version and changelog

Author: drakejwong

CHANGELOG.md

@@ -4,6 +4,29 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.2](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.4.2) - 2021-12-16
+
+### Added
+- `Slice.name` property that fetches the Slice's user-defined name.
+  - The Slice's items are no longer fetched unnecessarily; this used to cause considerable latency.
+- `Slice.items` property that fetches all items contained in the Slice.
+
+### Changed
+- `Slice.info()` now only retrieves the Slice's `name`, `slice_id`, and `dataset_id`.
+  - The Slice's items are no longer fetched unnecessarily; this used to cause considerable latency.
+  - This method issues a warning to use `Slice.items` when attempting to `items`.
+
+[###](###) Deprecated
+- `NucleusClient.slice_info(..)` is deprecated in favor of `Slice.info()`.
+
+## [0.4.1](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.4.1) - 2021-12-13
+
+### Changed
+- Datasets in Nucleus now fall under two categories: scene or item.
+  - Scene Datasets can only have scenes uploaded to them.
+  - Item Datasets can only have items uploaded to them.
+- `NucleusClient.create_dataset` now requires a boolean parameter `is_scene` to immutably set whether the Dataset is a scene or item Dataset.
+
 ## [0.4.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.4.0) - 2021-08-12
 
 ### Added

nucleus/__init__.py

@@ -741,8 +741,9 @@ def get_slice(self, slice_id: str) -> Slice:
         """
         return Slice(slice_id, self)
 
+    @deprecated("Prefer calling Slice.info instead.")
     def slice_info(self, slice_id: str) -> dict:
-        # TODO: migrate to Slice method and deprecate
+        # TODO: deprecate in favor of Slice.info
         response = self.make_request(
             {},
             f"slice/{slice_id}",

nucleus/slice.py

@@ -7,7 +7,11 @@
 from nucleus.constants import EXPORTED_ROWS
 from nucleus.dataset_item import DatasetItem
 from nucleus.job import AsyncJob
-from nucleus.utils import convert_export_payload, format_dataset_item_response
+from nucleus.utils import (
+    KeyErrorDict,
+    convert_export_payload,
+    format_dataset_item_response,
+)
 
 
 class Slice:
@@ -41,6 +45,7 @@ def __init__(self, slice_id: str, client):
         self.id = slice_id
         self._slice_id = slice_id
         self._client = client
+        self._name = None
         self._dataset_id = None
 
     def __repr__(self):
@@ -52,6 +57,30 @@ def __eq__(self, other):
                 return True
         return False
 
+    def _fetch_all(self) -> dict:
+        """Retrieves info and all items of the Slice.
+
+        Returns:
+            A dict mapping keys to the corresponding info retrieved.
+            ::
+
+                {
+                    "name": Union[str, int],
+                    "slice_id": str,
+                    "dataset_id": str,
+                    "dataset_items": List[{
+                        "id": str,
+                        "metadata": Dict[str, Union[str, int, float]],
+                        "ref_id": str,
+                        "original_image_url": str
+                    }]
+                }
+        """
+        response = self._client.make_request(
+            {}, f"slice/{self.id}", requests_command=requests.get
+        )
+        return response
+
     @property
     def slice_id(self):
         warnings.warn(
@@ -60,33 +89,45 @@ def slice_id(self):
         )
         return self._slice_id
 
+    @property
+    def name(self):
+        """The name of the Slice."""
+        if self._name is None:
+            self._name = self.info()["name"]
+        return self._name
+
     @property
     def dataset_id(self):
         """The ID of the Dataset to which the Slice belongs."""
         if self._dataset_id is None:
-            self.info()
+            self._dataset_id = self.info()["dataset_id"]
         return self._dataset_id
 
+    @property
+    def items(self):
+        """All DatasetItems contained in the Slice."""
+        return self._fetch_all()["dataset_items"]
+
     def info(self) -> dict:
-        """Retrieves info and items of the Slice.
+        """Retrieves the name, slice_id, and dataset_id of the Slice.
 
         Returns:
             A dict mapping keys to the corresponding info retrieved.
             ::
 
                 {
                     "name": Union[str, int],
+                    "slice_id": str,
                     "dataset_id": str,
-                    "dataset_items": List[{
-                        "id": str,
-                        "metadata": Dict[str, Union[str, int, float]],
-                        "ref_id": str,
-                        "original_image_url": str
-                    }]
                 }
         """
-        info = self._client.slice_info(self.id)
-        self._dataset_id = info["dataset_id"]
+        info = KeyErrorDict(
+            items="The 'items' key is now deprecated for Slice.info. Use Slice.items instead."
+        )
+        res = self._client.make_request(
+            {}, f"slice/{self.id}/info", requests_command=requests.get
+        )
+        info.update(res)
         return info
 
     def append(
@@ -137,11 +178,10 @@ def items_and_annotation_generator(
                     }
                 }]
         """
-        info = self.info()
-        for item_metadata in info["dataset_items"]:
+        for item_metadata in self.items:
             yield format_dataset_item_response(
                 self._client.dataitem_loc(
-                    dataset_id=info["dataset_id"],
+                    dataset_id=self.dataset_id,
                     dataset_item_id=item_metadata["id"],
                 )
             )
@@ -259,14 +299,12 @@ def check_annotations_are_in_slice(
         1. True if all Annotations are in the Slice, False otherwise;
         2. List of reference IDs not in the Slice.
     """
-    info = slice_to_check.info()
-
     reference_ids_not_found_in_slice = {
         annotation.reference_id
         for annotation in annotations
         if annotation.reference_id is not None
     }.difference(
-        {item_metadata["ref_id"] for item_metadata in info["dataset_items"]}
+        {item_metadata["ref_id"] for item_metadata in slice_to_check.items}
     )
     if reference_ids_not_found_in_slice:
         annotations_are_in_slice = False

nucleus/utils.py

@@ -48,6 +48,40 @@
 }
 
 
+class KeyErrorDict(dict):
+    """Wrapper for response dicts with deprecated keys.
+
+    Parameters:
+        **kwargs: Mapping from the deprecated key to a warning message.
+    """
+
+    def __init__(self, **kwargs):
+        self._deprecated = {}
+
+        for key, msg in kwargs.items():
+            if not isinstance(key, str):
+                raise TypeError(
+                    f"All keys must be strings! Received non-string '{key}'"
+                )
+            if not isinstance(msg, str):
+                raise TypeError(
+                    f"All warning messages must be strings! Received non-string '{msg}'"
+                )
+
+            self._deprecated[key] = msg
+
+        super().__init__()
+
+    def __missing__(self, key):
+        """Raises KeyError for deprecated keys, otherwise uses base dict logic."""
+        if key in self._deprecated:
+            raise KeyError(self._deprecated[key])
+        try:
+            super().__missing__(key)
+        except AttributeError as e:
+            raise KeyError(key) from e
+
+
 def format_prediction_response(
     response: dict,
 ) -> Union[

pyproject.toml

@@ -21,7 +21,7 @@ exclude = '''
 
 [tool.poetry]
 name = "scale-nucleus"
-version = "0.4.1"
+version = "0.4.2"
 description = "The official Python client library for Nucleus, the Data Platform for AI"
 license =  "MIT"
 authors = ["Scale AI Nucleus Team <nucleusapi@scaleapi.com>"]

tests/test_slice.py

@@ -64,16 +64,22 @@ def test_slice_create_and_delete_and_list(dataset):
     assert len(dataset_slices) == 1
     assert slc.id == dataset_slices[0]
 
-    response = slc.info()
-    assert response["name"] == TEST_SLICE_NAME
-    assert response["dataset_id"] == dataset.id
-    assert len(response["dataset_items"]) == 2
+    assert slc.name == TEST_SLICE_NAME
+    assert slc.dataset_id == dataset.id
+
+    items = slc.items
+    assert len(items) == 2
     for item in ds_items[:2]:
         assert (
-            item.reference_id == response["dataset_items"][0]["ref_id"]
-            or item.reference_id == response["dataset_items"][1]["ref_id"]
+            item.reference_id == items[0]["ref_id"]
+            or item.reference_id == items[1]["ref_id"]
         )
 
+    response = slc.info()
+    assert response["name"] == TEST_SLICE_NAME
+    assert response["slice_id"] == slc.slice_id
+    assert response["dataset_id"] == dataset.id
+
 
 def test_slice_create_and_export(dataset):
     # Dataset upload
@@ -132,13 +138,13 @@ def test_slice_append(dataset):
     # Insert duplicate first item
     slc.append(reference_ids=[item.reference_id for item in ds_items[:3]])
 
-    response = slc.info()
-    assert len(response["dataset_items"]) == 3
+    items = slc.items
+    assert len(items) == 3
     for item in ds_items[:3]:
         assert (
-            item.reference_id == response["dataset_items"][0]["ref_id"]
-            or item.reference_id == response["dataset_items"][1]["ref_id"]
-            or item.reference_id == response["dataset_items"][2]["ref_id"]
+            item.reference_id == items[0]["ref_id"]
+            or item.reference_id == items[1]["ref_id"]
+            or item.reference_id == items[2]["ref_id"]
         )
 
     all_stored_items = [_[ITEM_KEY] for _ in slc.items_and_annotations()]
@@ -178,8 +184,8 @@ def test_slice_send_to_labeling(dataset):
         reference_ids=[ds_items[0].reference_id, ds_items[1].reference_id],
     )
 
-    response = slc.info()
-    assert len(response["dataset_items"]) == 2
+    items = slc.items
+    assert len(items) == 2
 
     response = slc.send_to_labeling(TEST_PROJECT_ID)
     assert isinstance(response, AsyncJob)