enh: get_collections and get_circles now return dicts, other minor improvements

Author: paulmueller

CHANGELOG

@@ -19,10 +19,13 @@
  - enh: remove unused "Sharing" page
  - enh: add icon for "Find Data"
  - enh: cache CKANAPI instance in GUI
+ - enh: `APIInterrogator.search_dataset_via_api` optionally returns list
+ - enh: `DBExtract.__iadd__` supports lists
  - ref: remove `mode` from `DBInterrogator`
  - ref: replace `user_list` with `user_autocomplete`
  - ref: replace `search_dataset` with `search_dataset_via_api`
    in `APIInterrogator`; `search_dataset` is now only free text search
+ - ref: `get_collections` and `get_circles` now return list of dictionaries
  - ref: rename gui submodules for better overview
  - ref: move "My Data" panel into its own submodule
  - ref: implement `is_dc_resource_dict`

dcoraid/dbmodel/db_api.py

@@ -30,12 +30,11 @@ def __init__(self, api):
     def get_circles(self):
         """Return the list of DCOR Circle names
         """
-        data = self.api.get("organization_list")
-        return data
+        return self.api.get("organization_list", all_fields=True)
 
     def get_collections(self):
         """Return the list of DCOR Collection names"""
-        data = self.api.get("group_list")
+        data = self.api.get("group_list", all_fields=True, limit=1000)
         if len(data) == 1000:
             raise NotImplementedError(
                 "Reached hard limit of 1000 results! "
@@ -72,14 +71,14 @@ def get_datasets_user_shared(self):
 
             for circles_batch in batched(self.get_circles(), 20):
                 dbe += self.search_dataset_via_api(
-                    circles=list(circles_batch),
+                    circles=[c["name"] for c in circles_batch],
                     filter_queries=[f"-creator_user_id:{self.api.user_id}"],
                     limit=0,
                     )
 
             for collections_batch in batched(self.get_collections(), 20):
                 dbe += self.search_dataset_via_api(
-                    collections=list(collections_batch),
+                    collections=[c["name"] for c in collections_batch],
                     filter_queries=[f"-creator_user_id:{self.api.user_id}"],
                     limit=0,
                     )
@@ -134,7 +133,9 @@ def search_dataset_via_api(self,
                                since_time: float = None,
                                sort_solr: str = "metadata_created desc",
                                start: int = 0,
-                               limit: int = 100):
+                               limit: int = 100,
+                               ret_db_extract: bool = True,
+                               ):
         """Search datasets via the CKAN API
 
         Parameters
@@ -167,6 +168,10 @@ def search_dataset_via_api(self,
             returned datasets should begin.
         limit: int
             limit number of search results; Set to 0 to get all results
+        ret_db_extract: bool
+            whether to return an instance of :class:`DBExtract`; if set to
+            `False`, then a list of datasets is returned instead which is
+            faster.
         """
         if filter_queries is None:
             filter_queries = []
@@ -230,7 +235,10 @@ def search_dataset_via_api(self,
 
         num_total = np.inf  # just the initial value
         num_retrieved = 0
-        dbe = DBExtract()
+        if ret_db_extract:
+            dbe = DBExtract()
+        else:
+            dbe = []
         while start + num_retrieved < min(start + limit, num_total) and rows:
             data = self.api.get(
                 "package_search",
@@ -249,7 +257,7 @@ def search_dataset_via_api(self,
                 # in the next iteration, only get the final
                 # few results.
                 rows = num_total - num_retrieved
-            dbe.add_datasets(data["results"])
+            dbe += data["results"]
 
         return dbe
 

dcoraid/dbmodel/db_api_cached.py

@@ -142,15 +142,19 @@ def update(self, reset=False, abort_event=None):
 
         new_timestamp = time.time()
 
-        dbe = self.ai.search_dataset_via_api(
-            since_time=self.local_timestamp,
-            limit=0,
-        )
-
-        for ds_dict in dbe:
+        for cdict in self.get_circles():
+            logger.info(f"Fetching dataset from circle {cdict['name']}")
+            dbe = self.ai.search_dataset_via_api(
+                since_time=self.local_timestamp,
+                circles=[cdict["name"]],
+                limit=0,
+                ret_db_extract=False,
+            )
             if abort_event and abort_event.is_set():
                 break
-            self._mc.upsert_dataset(ds_dict)
+            self._mc.upsert_many(dbe, org_id=cdict["id"])
+            logger.info(
+                f"Loaded {len(dbe)} datasets from circle {cdict['name']}")
         else:
             # Only update the local timestamp if we actually did
             # update the local database.

dcoraid/dbmodel/db_core.py

@@ -28,12 +28,12 @@ def close(self):
 
     @abc.abstractmethod
     def get_circles(self):
-        """Return the list of DCOR Circles"""
+        """Return the list of DCOR Circle dictionaries"""
         pass
 
     @abc.abstractmethod
     def get_collections(self):
-        """Return the list of DCOR Collections"""
+        """Return the list of DCOR Collection dictionaries"""
         pass
 
     def get_datasets_user(self):

dcoraid/gui/panel_my_data/widget_my_data.py

@@ -38,18 +38,21 @@ def __init__(self, *args, **kwargs):
         self.checkBox_user_shared.clicked.connect(self.on_update_view)
 
     @QtCore.pyqtSlot(dict, list)
-    def on_added_datasets_to_collection(self, collection, dataset_ids):
+    def on_added_datasets_to_collection(self,
+                                        collection: dict,
+                                        dataset_ids: list[str],
+                                        ):
         """User manually added a bunch of datasets to a collection"""
         # Get the collection
         for col in self.database.get_collections():
-            if col == collection["name"]:
+            if col["id"] == collection["id"]:
                 cid = collection["id"]
                 break
         else:
             # we have to reset the database and try again
             self.database.update(reset=True)
             for col in self.database.get_collections():
-                if col == collection["name"]:
+                if col["id"] == collection["id"]:
                     cid = collection["id"]
                     break
             else:

tests/test_dbmodel_api.py

@@ -19,7 +19,7 @@
 def test_get_circles():
     api = common.get_api()
     db = db_api.APIInterrogator(api=api)
-    circles = db.get_circles()
+    circles = [c["name"] for c in db.get_circles()]
     defaults = common.get_test_defaults()
     assert defaults["circle"] in circles
     # requires that the "dcoraid" user is in the figshare-import circle
@@ -31,7 +31,7 @@ def test_get_circles():
 def test_get_collections():
     api = common.get_api()
     db = db_api.APIInterrogator(api=api)
-    collections = db.get_collections()
+    collections = [c["name"] for c in db.get_collections()]
     defaults = common.get_test_defaults()
     assert defaults["collection"] in collections
     # requires that the "dcoraid" user is in figshare-collection collection
@@ -57,8 +57,10 @@ def test_public_api_interrogator():
     api = common.get_api()
     db = db_api.APIInterrogator(api=api)
     defaults = common.get_test_defaults()
-    assert defaults["circle"] in db.get_circles()
-    assert defaults["collection"] in db.get_collections()
+    circles = [c["name"] for c in db.get_circles()]
+    collections = [c["name"] for c in db.get_collections()]
+    assert defaults["circle"] in circles
+    assert defaults["collection"] in collections
     assert defaults["user"] in db.get_users()
 
 

tests/test_dbmodel_api_cache.py

@@ -17,7 +17,8 @@
 def test_get_circles(tmp_path):
     api = common.get_api()
     db = CachedAPIInterrogator(cache_location=tmp_path, api=api)
-    circles = db.get_circles()
+    circles_dicts = db.get_circles()
+    circles = [c["name"] for c in circles_dicts]
     defaults = common.get_test_defaults()
     assert defaults["circle"] in circles
     # requires that the "dcoraid" user is in the figshare-import circle
@@ -29,7 +30,7 @@ def test_get_circles(tmp_path):
 def test_get_collections(tmp_path):
     api = common.get_api()
     db = CachedAPIInterrogator(cache_location=tmp_path, api=api)
-    collections = db.get_collections()
+    collections = [c["name"] for c in db.get_collections()]
     defaults = common.get_test_defaults()
     assert defaults["collection"] in collections
     # requires that the "dcoraid" user is in figshare-collection collection
@@ -55,9 +56,11 @@ def test_get_users_anonymous(tmp_path):
 def test_public_api_interrogator(tmp_path):
     api = common.get_api()
     db = CachedAPIInterrogator(cache_location=tmp_path, api=api)
+    circles = [c["name"] for c in db.get_circles()]
     defaults = common.get_test_defaults()
-    assert defaults["circle"] in db.get_circles()
-    assert defaults["collection"] in db.get_collections()
+    collections = [c["name"] for c in db.get_collections()]
+    assert defaults["circle"] in circles
+    assert defaults["collection"] in collections
     assert defaults["user"] in db.get_users()