ref: move logic for downloading collections/datasets/resources to download widget

Author: paulmueller

dcoraid/gui/dbview/filter_base.py

@@ -7,7 +7,11 @@
 class FilterBase(QtWidgets.QWidget):
     #: The user selection has changed
     selection_changed = QtCore.pyqtSignal(list)
-    download_resource = QtCore.pyqtSignal(str, bool)
+    #: Download something (which, id_or_name, condensed)
+    #: - `which` is e.g. "resource", "dataset", "collection", circle
+    #: - `id_or_name` is the identifier or name of the thing
+    #: - `condensed` determines whether to download condensed resources only
+    download_item = QtCore.pyqtSignal(str, str, bool)
 
     def __init__(self, *args, **kwargs):
         """Filter view widget with title, edit, checkbox, and table

dcoraid/gui/dbview/filter_chain.py

@@ -11,7 +11,7 @@
 
 
 class FilterChain(QtWidgets.QWidget):
-    download_resource = QtCore.pyqtSignal(str, bool)
+    download_item = QtCore.pyqtSignal(str, str, bool)
 
     def __init__(self, *args, **kwargs):
         """Filter chain widget with multiple filter views
@@ -35,10 +35,10 @@ def __init__(self, *args, **kwargs):
         # resource filtered by .rtdc
         self.fw_resources.checkBox.stateChanged.connect(self.update_resources)
         # request resource downloads
-        self.fw_circles.download_resource.connect(self.download_resource)
-        self.fw_collections.download_resource.connect(self.download_resource)
-        self.fw_datasets.download_resource.connect(self.download_resource)
-        self.fw_resources.download_resource.connect(self.download_resource)
+        self.fw_circles.download_item.connect(self.download_item)
+        self.fw_collections.download_item.connect(self.download_item)
+        self.fw_datasets.download_item.connect(self.download_item)
+        self.fw_resources.download_item.connect(self.download_item)
 
     @property
     def selected_circles(self):

dcoraid/gui/dbview/filter_views.py

@@ -1,13 +1,12 @@
 from functools import partial
 import webbrowser
 
-from PyQt6 import QtCore, QtWidgets
+from PyQt6 import QtWidgets
 from PyQt6.QtCore import Qt
 
 from ...common import is_dc_resource_dict
 
 from ..api import get_ckan_api
-from ..tools import ShowWaitCursor
 
 from . import filter_base
 
@@ -39,47 +38,18 @@ def __init__(self, *args, **kwargs):
         self.checkBox.setVisible(False)
         self.label_info.setVisible(False)
 
-    def download_collection(self, collection_name, condensed=False):
-        """Emit signals to download all resources of a collection
-
-        Parameters
-        ----------
-        collection_name: str
-            Name of the collection
-        condensed: bool
-            Whether to download condensed resources instead
-        """
-        with ShowWaitCursor():
-            api = get_ckan_api()
-            search_dict = api.get("package_search",
-                                  fq=f"+groups:{collection_name}",
-                                  include_private=True,
-                                  rows=1000)
-            num_datasets = search_dict["count"]
-            if num_datasets >= 1000:
-                raise NotImplementedError(
-                    # We have to increase ckan.search.rows_max = 1000
-                    # or (better) use the `start` parameter until we
-                    # hit a number < 1000.
-                    f"There are too many datasets in '{collection_name}'!")
-            for ii, ds_dict in enumerate(search_dict["results"]):
-                for res_dict in ds_dict.get("resources", []):
-                    self.download_resource.emit(res_dict["id"], condensed)
-                    QtWidgets.QApplication.processEvents(
-                        QtCore.QEventLoop.ProcessEventsFlag.AllEvents,
-                        300)
-
     def get_entry_actions(self, row, entry):
         api = get_ckan_api()
         url = f"{api.server}/group/{entry['name']}"
         actions = [
             {"icon": "angle-down",
              "tooltip": f"download collection {entry['name']}",
-             "function": partial(self.download_collection, entry["name"])},
+             "function": partial(self.download_item.emit,
+                                 "collection", entry["name"], False)},
             {"icon": "angles-down",
              "tooltip": f"download condensed collection {entry['name']}",
-             "function": partial(self.download_collection, entry["name"],
-                                 condensed=True)},
+             "function": partial(self.download_item.emit,
+                                 "collection", entry["name"], True)},
             {"icon": "eye",
              "tooltip": f"view collection {entry['name']} online",
              "function": partial(webbrowser.open, url)}
@@ -95,35 +65,18 @@ def __init__(self, *args, **kwargs):
         self.checkBox.setVisible(False)
         self.label_info.setVisible(False)
 
-    def download_dataset(self, dataset_id, condensed=False):
-        """Emit signals to download all resources of a dataset
-
-        Parameters
-        ----------
-        dataset_id: str
-            dataset ID
-        condensed: bool
-            Whether to download condensed resources instead
-        """
-        api = get_ckan_api()
-        ds_dict = api.get("package_show", id=dataset_id)
-        for res_dict in ds_dict.get("resources", []):
-            self.download_resource.emit(res_dict["id"], condensed)
-            QtWidgets.QApplication.processEvents(
-                QtCore.QEventLoop.ProcessEventsFlag.AllEvents,
-                300)
-
     def get_entry_actions(self, row, entry):
         api = get_ckan_api()
         url = f"{api.server}/dataset/{entry['name']}"
         actions = [
             {"icon": "angle-down",
              "tooltip": f"download dataset {entry['name']}",
-             "function": partial(self.download_dataset, entry["id"])},
+             "function": partial(self.download_item.emit,
+                                 "dataset", entry["id"], False)},
             {"icon": "angles-down",
              "tooltip": f"download condensed dataset {entry['name']}",
-             "function": partial(self.download_dataset, entry["id"],
-                                 condensed=True)},
+             "function": partial(self.download_item.emit,
+                                 "dataset", entry["id"], True)},
             {"icon": "eye",
              "tooltip": f"view dataset {entry['name']} online",
              "function": partial(webbrowser.open, url)},
@@ -153,8 +106,8 @@ def get_entry_actions(self, row, entry):
         actions = [
             {"icon": "angle-down",
              "tooltip": f"download resource {entry['name']}",
-             "function": partial(self.download_resource.emit,
-                                 entry["id"], False)},
+             "function": partial(self.download_item.emit,
+                                 "resource", entry["id"], False)},
             {"icon": "eye",
              "tooltip": f"view resource {entry['name']} online",
              "function": partial(webbrowser.open, url)},
@@ -165,8 +118,9 @@ def get_entry_actions(self, row, entry):
                 1,
                 {"icon": "angles-down",
                  "tooltip": f"download condensed resource {entry['name']}",
-                 "function": partial(self.download_resource.emit,
-                                     entry["id"], True)},)
+                 "function": partial(self.download_item.emit,
+                                     "resource", entry["id"], True)},
+            )
         return actions
 
     def set_entry_label(self, row, entry):

dcoraid/gui/main.py

@@ -104,10 +104,10 @@ def __init__(self, *args, **kwargs):
         self.status_widget.clicked.connect(self.dlg_pref.on_show_server)
 
         # Signal for requesting resource download
-        self.panel_find_data.request_download.connect(
-            self.panel_download.download_resource)
-        self.panel_my_data.request_download.connect(
-            self.panel_download.download_resource)
+        self.panel_find_data.download_item.connect(
+            self.panel_download.download_an_item)
+        self.panel_my_data.download_item.connect(
+            self.panel_download.download_an_item)
 
         # Signal for dataset upload or modification
         self.panel_upload.upload_finished.connect(self.on_dataset_changed)
@@ -151,6 +151,7 @@ def __init__(self, *args, **kwargs):
         else:
             self.panel_find_data.set_database(self.database)
             self.panel_my_data.set_database(self.database)
+            self.panel_download.set_database(self.database)
 
         self.status_widget.request_status_update()
 

dcoraid/gui/panel_downloads/widget_download.py

@@ -12,6 +12,7 @@
 from PyQt6 import uic, QtCore, QtGui, QtWidgets
 from PyQt6.QtCore import QStandardPaths
 
+from ...common import is_dc_resource_dict
 from ...download import DownloadQueue
 
 from ..api import get_ckan_api
@@ -33,6 +34,8 @@ def __init__(self, *args, **kwargs):
         with resources.as_file(ref_ui) as path_ui:
             uic.loadUi(path_ui, self)
 
+        self.database = None
+
         self.settings = QtCore.QSettings()
 
         #: path to persistent shelf to be able to resume uploads on startup
@@ -83,11 +86,49 @@ def initialize(self):
             self.init_timer.setInterval(1000)
             self.init_timer.start()
 
+    @QtCore.pyqtSlot(str, str, bool)
+    def download_an_item(self, which, identifier, condensed=False):
+        if which == "collection":
+            self.download_collection(identifier, condensed)
+        elif which == "dataset":
+            self.download_dataset(identifier, condensed)
+        elif which == "resource":
+            self.download_resource(identifier, condensed)
+        else:
+            raise ValueError(f"Invalid download item specified: {which}")
+
+    @QtCore.pyqtSlot(str, bool)
+    def download_collection(self, collection_id, condensed=False):
+        """Download all resources from all datasets in a collection"""
+        # Get all datasets in that collection
+        api = get_ckan_api()
+        ds_list = api.get("package_search",
+                          fq=f"+groups:{collection_id}",
+                          include_private=True,
+                          rows=1000)["results"]
+        for ds_dict in ds_list:
+            # use this opportunity to update our database
+            self.database.update_dataset(ds_dict)
+            # send this dataset on for downloading
+            self.download_dataset(ds_dict["id"], condensed)
+
+    @QtCore.pyqtSlot(str, bool)
+    def download_dataset(self, dataset_id, condensed=False):
+        """Download all resources in a dataset"""
+        dl_path = self.get_download_path()
+        ds_dict = self.database.get_dataset_dict(dataset_id)
+        if self.jobs is not None:
+            for res_dict in ds_dict.get("resources", []):
+                cond_this = condensed
+                if condensed and not is_dc_resource_dict(res_dict):
+                    # Not a DC resource; cannot download condensed file!
+                    cond_this = False
+                self.jobs.new_job(res_dict["id"], dl_path, cond_this)
+
     @QtCore.pyqtSlot(str, bool)
     def download_resource(self, resource_id, condensed=False):
-        fallback = QStandardPaths.writableLocation(
-                      QStandardPaths.StandardLocation.DownloadLocation)
-        dl_path = self.settings.value("downloads/default path", fallback)
+        """Download a resource"""
+        dl_path = self.get_download_path()
         if self.jobs is not None:
             self.jobs.new_job(resource_id, dl_path, condensed)
         else:
@@ -98,6 +139,11 @@ def download_resource(self, resource_id, condensed=False):
                 f"We cannot connect to '{api.server}'. Please make "
                 f"sure you are connected to the network.")
 
+    def get_download_path(self):
+        fallback = QStandardPaths.writableLocation(
+                      QStandardPaths.StandardLocation.DownloadLocation)
+        return self.settings.value("downloads/default path", fallback)
+
     def prepare_quit(self):
         """Should be called before the application quits"""
         self.init_timer.stop()
@@ -106,6 +152,9 @@ def prepare_quit(self):
         if self.jobs is not None:
             self.jobs.__del__()
 
+    def set_database(self, database):
+        self.database = database
+
 
 class DownloadTableWidget(QtWidgets.QTableWidget):
     download_finished = QtCore.pyqtSignal()

dcoraid/gui/panel_find_data/widget_find_data.py

@@ -12,7 +12,7 @@
 
 
 class WidgetFindData(QtWidgets.QWidget):
-    request_download = QtCore.pyqtSignal(str, bool)
+    download_item = QtCore.pyqtSignal(str, str, bool)
 
     def __init__(self, *args, **kwargs):
         """Browse DCOR data"""
@@ -32,8 +32,7 @@ def __init__(self, *args, **kwargs):
         # Signals for data browser
         self.pushButton_search.clicked.connect(self.on_search)
         self.pushButton_update_db.clicked.connect(self.on_update_db)
-        self.public_filter_chain.download_resource.connect(
-            self.request_download)
+        self.public_filter_chain.download_item.connect(self.download_item)
 
     @QtCore.pyqtSlot()
     def on_search(self):

dcoraid/gui/panel_my_data/widget_my_data.py

@@ -16,7 +16,7 @@
 
 
 class WidgetMyData(QtWidgets.QWidget):
-    request_download = QtCore.pyqtSignal(str, bool)
+    download_item = QtCore.pyqtSignal(str, str, bool)
 
     def __init__(self, *args, **kwargs):
         """Browse public DCOR data"""
@@ -29,8 +29,7 @@ def __init__(self, *args, **kwargs):
         # Signals for user datasets (my data)
         self.pushButton_user_refresh.clicked.connect(
             self.on_refresh_private_data)
-        self.user_filter_chain.download_resource.connect(
-            self.request_download)
+        self.user_filter_chain.download_item.connect(self.download_item)
         self.user_filter_chain.added_datasets_to_collection.connect(
             self.on_added_datasets_to_collection)