fix: check for existence of DC resources before starting uploads (close #85)

Author: paulmueller

CHANGELOG

@@ -1,6 +1,8 @@
 0.17.2
+ - fix: check for existence of DC resources before starting uploads (#85)
  - enh: sort resources according to internal basin dependencies (#79)
  - ref: use more f-strings
+ - ref: add `common.is_dc_file` for consistent resource checks
 0.17.1
  - fix: found culprit of regular segmentation faults (#14)
  - reg: invalid use PyQt6 namespace after migration from PyQt5

dcoraid/common.py

@@ -3,6 +3,7 @@
 import pathlib
 import weakref
 
+import dclab
 import requests
 
 from .api import s3_api
@@ -59,6 +60,27 @@ def etagsum(path):
     return etag
 
 
+@functools.lru_cache(maxsize=2000)
+def is_dc_file(path, test_open=True):
+    """Return True when the file is a valid DC dataset
+
+    Performs a path suffix check.
+    If `test_open` is True (default), attempt to open the file with dclab.
+    """
+    is_dc = False
+    path = pathlib.Path(path)
+    if path.suffix.lower() in [".rtdc", ".dc"]:
+        if test_open:
+            try:
+                with dclab.new_dataset(path):
+                    is_dc = True
+            except BaseException:
+                pass
+        else:
+            is_dc = True
+    return is_dc
+
+
 @functools.lru_cache(maxsize=2000)
 def sha256sum(path):
     """Compute the SHA256 hash of a file"""

dcoraid/gui/upload/dlg_upload.py

@@ -7,6 +7,7 @@
 
 
 from ...api import dataset_create
+from ...common import is_dc_file
 from ...upload import job
 
 from ..tools import ShowWaitCursor
@@ -297,11 +298,22 @@ def on_proceed(self):
         This will first trigger a creation of the draft dataset
         on DCOR. Then, the job is enqueued in the parent
         """
-        # We should only proceed if we have resources
+        # We should only proceed when we have resources
         if self.rvmodel.rowCount() == 0:
             QtWidgets.QMessageBox.critical(self, "No resources selected",
                                            "Please add at least one resource.")
             return
+
+        # In addition, we should only proceed when we have at least one
+        # DC resource.
+        paths = self.rvmodel.get_file_list()
+        if sum([is_dc_file(pp, test_open=False) for pp in paths]) == 0:
+            QtWidgets.QMessageBox.critical(
+                self, "No DC resources specified",
+                "Please add at least one deformability cytometry "
+                "resource.")
+            return
+
         # Checking for duplicate resources is the responsibility of
         # DCOR-Aid, because we are skipping existing resources in
         # dcoraid.upload.job.UploadJob.task_upload_resources.

dcoraid/gui/upload/resources_model.py

@@ -6,6 +6,7 @@
 
 from PyQt6 import QtCore, QtGui
 
+from ...common import is_dc_file
 from ...upload import job
 
 
@@ -61,10 +62,7 @@ def filenames_were_edited(self):
     def index_is_dc(self, index):
         """Does the given index instance belong to an RT-DC file?"""
         path = pathlib.Path(self.get_file_list()[index.row()])
-        if path.suffix in [".dc", ".rtdc"]:
-            return True
-        else:
-            return False
+        return is_dc_file(path, test_open=False)
 
     def index_has_edits(self, index):
         """Is there a modification of the list entry of this index instance?"""

dcoraid/upload/job.py

@@ -12,7 +12,12 @@
 from dclab.cli import compress
 
 from ..api import dataset_activate, resource_add, resource_exists
-from ..common import sha256sum
+from ..common import is_dc_file, sha256sum
+
+
+class AtLeastOneDCResourceRequiredPerDatasetError(BaseException):
+    """Raised when an upload does not contain at least one DC resource"""
+    pass
 
 
 logger = logging.getLogger(__name__)
@@ -76,10 +81,28 @@ def __init__(self, api, dataset_id, resource_paths,
             Multiple upload jobs may share the same cache dir,
             since each job creates its own subdirectory.
         """
+        # Ensure resource_paths is a list in case somebody passed an iterator.
+        resource_paths = list(resource_paths)
+
         self.api = api.copy()  # create a copy of the API
         self.dataset_id = dataset_id
+
+        # Check whether at least one DC resource is present in the list.
+        # This is a hard DCOR requirement.
+        for pp in resource_paths:
+            if is_dc_file(pp):
+                break
+        else:
+            raise AtLeastOneDCResourceRequiredPerDatasetError(
+                f"DCOR requires at least one valid deformability cytometry "
+                f"file per dataset upload. Please make sure that this "
+                f"condition is met and all of these files exist: "
+                f"{resource_paths}")
+
         # make sure the dataset_id is valid
         self.api.get("package_show", id=self.dataset_id, timeout=500)
+
+        # resolve paths and set resource names
         resolved_paths = [pathlib.Path(pp).resolve() for pp in resource_paths]
         if resource_names is None:
             resource_names = [pp.name for pp in resolved_paths]
@@ -192,18 +215,19 @@ def sort_resources_according_to_basin_hierarchy(paths) -> list[int]:
         paths_basins = {}
         for pp in path_strings:
             basins = []
-            try:
-                with dclab.new_dataset(pp) as ds:
-                    for bn in ds.basins:
-                        if (bn.basin_type == "file"
-                                and bn.basin_format == "hdf5"):
-                            bpath = str(bn.location.resolve())
-                            if bpath in path_strings:
-                                basins.append(bpath)
-            except BaseException:
-                logger.error(f"Failed to get basin info for {pp}. Traceback"
-                             f"follows.")
-                logger.error(traceback.format_exc())
+            if is_dc_file(pp, test_open=False):
+                try:
+                    with dclab.new_dataset(pp) as ds:
+                        for bn in ds.basins:
+                            if (bn.basin_type == "file"
+                                    and bn.basin_format == "hdf5"):
+                                bpath = str(bn.location.resolve())
+                                if bpath in path_strings:
+                                    basins.append(bpath)
+                except BaseException:
+                    logger.error(f"Failed to get basin info for {pp}. "
+                                 f"Traceback follows.")
+                    logger.error(traceback.format_exc())
             paths_basins[pp] = basins
 
         # edit path_strings in-place and populate paths_sort_order
@@ -418,14 +442,12 @@ def task_compress_resources(self):
         Data are stored in the user's cache directory and
         deleted after upload is complete.
         """
-        # make sure that we have .rtdc or .dc files
-        dc_files = [pp for pp in self.paths if pp.suffix in [".rtdc", ".dc"]]
-        if not dc_files:
-            raise ValueError("There are no RT-DC files in this dataset!")
         self.set_state("compress")
         ds_dict = self.api.get("package_show", id=self.dataset_id, timeout=500)
         for ii, path in enumerate(self.paths):
-            if path.suffix in [".rtdc", ".dc"]:  # do we have a DC file?
+            # We check EVERY DC file. So `test_open=False`. Integrity checker
+            # will open it and make sure it is a valid DC instance.
+            if is_dc_file(path, test_open=False):
                 if resource_exists(
                         dataset_id=self.dataset_id,
                         resource_name=self.resource_names[ii],
@@ -434,7 +456,7 @@ def task_compress_resources(self):
                         dataset_dict=ds_dict
                         ):
                     # There is no need to compress resources that have
-                    # already been upladed to DCOR. The same check is done
+                    # already been uploaded to DCOR. The same check is done
                     # in task_upload_resources, so there is no danger that
                     # an uncompressed resource would be uploaded.
                     continue

dcoraid/upload/task.py

@@ -18,6 +18,7 @@
 import uuid
 
 from ..api import dataset_create, APINotFoundError
+from ..common import is_dc_file
 
 from .job import UploadJob
 
@@ -271,7 +272,7 @@ def load_task(path, api, dataset_kwargs=None, map_task_to_dataset_id=None,
             raise ValueError("Number of resource paths does not match number "
                              f"of resource supplements in '{path}'!")
         for ii, pp in enumerate(uj_state["resource_paths"]):
-            if (not str(pp).endswith(".rtdc")
+            if (not is_dc_file(pp, test_open=False)
                     and uj_state["resource_supplements"][ii]):
                 raise ValueError("Resource supplements must be empty for "
                                  f"non-RT-DC datasets in '{path}'!")

tests/test_upload_job.py

@@ -24,6 +24,56 @@
               ]
 
 
+def test_check_existence_for_dc_resource_1(tmp_path):
+    (tmp_path / "test.csv").write_text("This is not a DC resource")
+    (tmp_path / "test.txt").write_text("This is not a DC resource as well")
+
+    api = common.get_api()
+
+    # create some metadata
+    bare_dict = common.make_dataset_dict(hint="create-with-resource")
+    # create dataset (to get the "id")
+    dataset_dict = dataset_create(dataset_dict=bare_dict, api=api)
+    with pytest.raises(job.AtLeastOneDCResourceRequiredPerDatasetError,
+                       match="test.csv"):
+        job.UploadJob(api=api,
+                      dataset_id=dataset_dict["id"],
+                      resource_paths=list(tmp_path.glob("*")))
+
+
+def test_check_existence_for_dc_resource_2(tmp_path):
+    (tmp_path / "test.rtdc").write_text("This is not a DC resource")
+
+    api = common.get_api()
+
+    # create some metadata
+    bare_dict = common.make_dataset_dict(hint="create-with-resource")
+    # create dataset (to get the "id")
+    dataset_dict = dataset_create(dataset_dict=bare_dict, api=api)
+    with pytest.raises(job.AtLeastOneDCResourceRequiredPerDatasetError):
+        job.UploadJob(api=api,
+                      dataset_id=dataset_dict["id"],
+                      resource_paths=list(tmp_path.glob("*")))
+
+
+def test_check_existence_for_dc_resource_control(tmp_path):
+    (tmp_path / "test.rtdc").write_text(
+        "This is not a DC resource and running the upload job would fail,"
+        "but we are only checking UploadJob instantiation here.")
+    shutil.copy2(rtdc_paths[0], tmp_path / "test2.rtdc")
+
+    api = common.get_api()
+
+    # create some metadata
+    bare_dict = common.make_dataset_dict(hint="create-with-resource")
+    # create dataset (to get the "id")
+    dataset_dict = dataset_create(dataset_dict=bare_dict, api=api)
+    uj = job.UploadJob(api=api,
+                       dataset_id=dataset_dict["id"],
+                       resource_paths=list(tmp_path.glob("*")))
+    assert len(uj.paths) == 2
+
+
 def test_resource_name_characters():
     assert re.match(job.VALID_RESOURCE_REGEXP, job.VALID_RESOURCE_CHARS)
     assert re.match(job.VALID_RESOURCE_REGEXP, job.VALID_RESOURCE_CHARS)