enh: think more about caching in the new design

Author: paulmueller

dcoraid/dbmodel/db_api_cached.py

@@ -57,19 +57,21 @@ def local_version_score(self):
     def close(self):
         self._mc.close()
 
-    def get_circles(self, refresh=False):
+    def get_circles(self):
         """Return the list of DCOR Circle names
         """
-        clist = self._cache_fleeting["circles"]
-        if not clist or refresh:
+        # TODO: Use self._mc to extract circles
+        clist = self._cache_fleeting.setdefault("circles", [])
+        if not clist:
             clist.clear()
             clist += self.ai.get_circles()
         return clist
 
-    def get_collections(self, refresh=False):
+    def get_collections(self):
         """Return the list of DCOR Collection names"""
-        clist = self._cache_fleeting["collections"]
-        if not clist or refresh:
+        # TODO: Use self._mc to extract collections
+        clist = self._cache_fleeting.setdefault("collections", [])
+        if not clist:
             clist.clear()
             clist += self.ai.get_collections()
         return clist
@@ -80,7 +82,10 @@ def get_dataset_dict(self, ds_id):
     def get_datasets_user_following(self) -> DBExtract:
         """Return datasets the user is following"""
         # TODO: Use datasets in self._mc
-        return self.ai.get_datasets_user_following()
+        ckey = "get_datasets_user_following"
+        if self._cache_fleeting.get(ckey) is None:
+            self._cache_fleeting[ckey] = self.ai.get_datasets_user_following()
+        return self._cache_fleeting[ckey]
 
     def get_datasets_user_owned(self) -> DBExtract:
         """Return datasets the user created"""
@@ -92,7 +97,10 @@ def get_datasets_user_owned(self) -> DBExtract:
     def get_datasets_user_shared(self) -> DBExtract:
         """Return datasets shared with the user"""
         # TODO: Use datasets in self._mc
-        return self.ai.get_datasets_user_shared()
+        ckey = "get_datasets_user_shared"
+        if self._cache_fleeting.get(ckey) is None:
+            self._cache_fleeting[ckey] = self.ai.get_datasets_user_shared()
+        return self._cache_fleeting[ckey]
 
     def get_users(self):
         """Return the list of DCOR users"""
@@ -119,15 +127,18 @@ def search_dataset(self, query="", limit=100):
 
     def update(self, reset=False, abort_event=None):
         """Update the local metadata cache based on the last local timestamp"""
+        # Clear the fleeting cache.
+        self._cache_fleeting.clear()
         if self.remote_version_score != self.local_version_score:
             reset = True
 
         if reset:
             self.local_timestamp = 0
             self._mc.reset()
 
-        self.get_circles(refresh=True)
-        self.get_collections(refresh=True)
+        # Call these methods now so they reflect the current database state.
+        self.get_circles()
+        self.get_collections()
 
         new_timestamp = time.time()
 
@@ -147,4 +158,5 @@ def update(self, reset=False, abort_event=None):
 
     def update_dataset(self, ds_dict):
         """Update a single dataset in the database without calling `update`"""
+        self._cache_fleeting.clear()
         self._mc.upsert_dataset(ds_dict)

dcoraid/gui/panel_my_data/widget_my_data.py

@@ -41,12 +41,21 @@ def __init__(self, *args, **kwargs):
     def on_added_datasets_to_collection(self, collection, dataset_ids):
         """User manually added a bunch of datasets to a collection"""
         # Get the collection
-        for col in self.database.get_collections(refresh=True):
+        for col in self.database.get_collections():
             if col == collection["name"]:
                 cid = collection["id"]
                 break
         else:
-            raise KeyError(f"Collection {collection['id']} not found!")
+            # we have to reset the database and try again
+            self.database.update(reset=True)
+            for col in self.database.get_collections():
+                if col == collection["name"]:
+                    cid = collection["id"]
+                    break
+            else:
+                raise ValueError(
+                    f"Could not, with the best will in the world, "
+                    f"find this collection: '{collection['name']}'")
         # Append it to each dataset
         for ds_id in dataset_ids:
             ds_dict = self.database.get_dataset_dict(ds_id)
@@ -57,6 +66,7 @@ def on_added_datasets_to_collection(self, collection, dataset_ids):
             else:
                 ds_dict["groups"].append(collection)
             self.database.update_dataset(ds_dict)
+        self.on_update_view()
 
     @QtCore.pyqtSlot()
     def on_update_db(self):

tests/test_gui.py

@@ -140,7 +140,7 @@ def test_gui_mydata_dataset_add_to_collection(mw, qtbot):
     with mock.patch.object(
             QInputDialog, "getItem",
             return_value=(f"{ii}: {item['display_name']}", True)):
-        qtbot.mouseClick(fw_datasets.toolButton_custom,
+        qtbot.mouseClick(fw_datasets.pushButton_custom,
                          QtCore.Qt.MouseButton.LeftButton)
 
     # Check whether dataset is in collection in database