enh: allow specifying collections in upload task files and via the API

Author: paulmueller

CHANGELOG

@@ -1,4 +1,5 @@
 0.17.3
+ - enh: allow specifying collections in upload task files and via the API
  - docs: increase warning threshold for max number of resources to 1000
    (CKAN 2.11.3 handles datasets with many resources much better)
 0.17.2

dcoraid/api/ckan_api.py

@@ -430,3 +430,23 @@ def post(self, api_call, data, dump_json=True, headers=None,
                             timeout=timeout)
         resp = self.handle_response(req, api_call)
         return resp["result"]
+
+    def require_collection(self,
+                           name: str,
+                           title: str = None):
+        """Return a collection dict with the given name and optional title
+
+        The collection is created if it does not exist already. If the
+        collection already exists, `title` is ignored.
+        """
+        # check whether the group exists
+        try:
+            col_dict = self.get("group_show", id=name)
+        except APINotFoundError:
+            # Create non-existent group
+            self.post("group_create",
+                      data={"name": name,
+                            "title": title or name,
+                            })
+            col_dict = self.get("group_show", id=name)
+        return col_dict

dcoraid/upload/job.py

@@ -11,7 +11,7 @@
 from dclab.rtdc_dataset.check import IntegrityChecker
 from dclab.cli import compress
 
-from ..api import dataset_activate, resource_add, resource_exists
+from ..api import CKANAPI, dataset_activate, resource_add, resource_exists
 from ..common import is_dc_file, sha256sum
 
 
@@ -48,9 +48,15 @@ class AtLeastOneDCResourceRequiredPerDatasetError(BaseException):
 
 
 class UploadJob:
-    def __init__(self, api, dataset_id, resource_paths,
-                 resource_names=None, resource_supplements=None,
-                 task_id=None, cache_dir=None):
+    def __init__(self,
+                 api: CKANAPI,
+                 dataset_id: str,
+                 resource_paths: list[str | pathlib.Path],
+                 resource_names: list[str] = None,
+                 resource_supplements: list[dict] = None,
+                 collections: list[str] = None,
+                 task_id: str = None,
+                 cache_dir: str | pathlib.Path = None):
         """Wrapper for resource uploads
 
         This job is meant to be run from a separate thread.
@@ -73,6 +79,9 @@ def __init__(self, api, dataset_id, resource_paths,
             on DCOR
         resource_supplements: list of dict
             Supplementary resource information
+        collections: list of strings
+            List of unique identifiers of collections that this
+            dataset should be appended to
         task_id: str
             Unique task ID (used for identifying jobs uploaded already)
         cache_dir: str or pathlib.Path
@@ -86,6 +95,16 @@ def __init__(self, api, dataset_id, resource_paths,
 
         self.api = api.copy()  # create a copy of the API
         self.dataset_id = dataset_id
+        self.collections = []
+        # add dataset to collections
+        for col in collections or []:
+            col_dict = self.api.require_collection(col)
+            self.collections.append(col_dict)
+        if self.collections:
+            revise_dict = {
+                "match": {"id": dataset_id},
+                "update": {"groups": self.collections}}
+            api.post("package_revise", revise_dict)
 
         # Check whether at least one DC resource is present in the list.
         # This is a hard DCOR requirement.

tests/common.py

@@ -88,6 +88,7 @@ def make_upload_task(task_id=True,  # tester may pass `None` to disable
                      resource_paths=None,
                      resource_names=None,
                      resource_supplements=None,
+                     collections=None,
                      ):
     """Return path to example task file"""
     if resource_paths is None:
@@ -100,6 +101,8 @@ def make_upload_task(task_id=True,  # tester may pass `None` to disable
         dataset_dict = make_dataset_dict(hint="task_test")
     if dataset_dict and dataset_id is None:
         dataset_id = dataset_dict.get("id")
+    if collections is None:
+        collections = []
     td = pathlib.Path(tempfile.mkdtemp(prefix="task_"))
     # copy resources there
     new_resource_paths = []
@@ -117,6 +120,7 @@ def make_upload_task(task_id=True,  # tester may pass `None` to disable
         "resource_paths": [str(pp) for pp in new_resource_paths],
         "resource_names": resource_names,
         "resource_supplements": resource_supplements,
+        "collections": collections,
     }
     data = {"upload_job": uj_state}
     if dataset_dict:

tests/test_api_base.py

@@ -1,4 +1,5 @@
 import time
+import uuid
 
 import pytest
 
@@ -37,6 +38,23 @@ def test_api_requests_cache_with_parameters():
     assert (t1 - t0) > (t2 - t1)
 
 
+def test_require_collection():
+    """Test creating a collection"""
+    api = common.get_api()
+    name = f"test-collection-{uuid.uuid4()}"
+    title = f"Test Collection {name[-5:]}"
+
+    # create collection
+    col_dict = api.require_collection(name=name, title=title)
+    assert col_dict["name"] == name
+    assert col_dict["title"] == title
+
+    # retrieve collection, title is ignored
+    col_dict2 = api.require_collection(name=name, title="RANDOM")
+    assert col_dict2["name"] == name
+    assert col_dict2["title"] == title
+
+
 @pytest.mark.parametrize("server,api_server", [
     ("http://localhost:5000", "http://localhost:5000"),
     ("https://localhost:5000", "https://localhost:5000"),

tests/test_upload_job.py

@@ -5,6 +5,7 @@
 import tempfile
 import time
 from unittest import mock
+import uuid
 import warnings
 
 import pytest
@@ -74,6 +75,31 @@ def test_check_existence_for_dc_resource_control(tmp_path):
     assert len(uj.paths) == 2
 
 
+def test_collection():
+    api = common.get_api()
+    # create some metadata
+    collection = f"test-collection-{uuid.uuid4()}"
+    bare_dict = common.make_dataset_dict(hint="create-with-resource"
+                                         )
+    # create dataset (to get the "id")
+    dataset_dict = dataset_create(dataset_dict=bare_dict, api=api)
+    uj = job.UploadJob(api=api,
+                       dataset_id=dataset_dict["id"],
+                       resource_paths=rtdc_paths,
+                       collections=[collection],
+                       )
+    assert uj.state == "init"
+    uj.task_compress_resources()
+    assert uj.state == "parcel"
+    uj.task_upload_resources()
+    assert uj.state == "online"
+
+    common.wait_for_job_no_queue(uj)
+
+    ds_dict = api.get("package_show", id=uj.dataset_id)
+    assert ds_dict["groups"][0]["name"] == collection
+
+
 def test_resource_name_characters():
     assert re.match(job.VALID_RESOURCE_REGEXP, job.VALID_RESOURCE_CHARS)
     assert re.match(job.VALID_RESOURCE_REGEXP, job.VALID_RESOURCE_CHARS)

tests/test_upload_task.py

@@ -59,6 +59,30 @@ def test_create_task():
     assert ddict["resources"][1]["size"] == 6
 
 
+def test_collection():
+    api = common.get_api()
+    collection = f"test-collection-{uuid.uuid4()}"
+    # post dataset creation request
+    task_path = common.make_upload_task(
+        resource_paths=[str(dpath)],
+        resource_names=[dpath.name],
+        collections=[collection],
+    )
+    uj = task.load_task(task_path, api=api)
+    uj.task_compress_resources()
+    uj.task_upload_resources()
+    uj.task_verify_resources()
+    common.wait_for_job_no_queue(uj)
+
+    # now make sure the collection was set correctly
+    ds_dict = api.get("package_show", id=uj.dataset_id)
+    assert ds_dict["groups"][0]["name"] == collection
+    # TODO: There is a bug in CKAN that does not allow listing group datasets
+    # https://github.com/ckan/ckan/issues/9052
+    # col_dict = api.get("group_show", id=collection, include_datasets=True)
+    # assert col_dict["packages"][0]["id"] == ds_dict["id"]
+
+
 def test_custom_dataset_dict():
     api = common.get_api()
     # post dataset creation request
@@ -111,7 +135,7 @@ def test_dataset_id_already_exists_active_fails():
     uj.task_compress_resources()
     assert uj.state == "parcel"
     with pytest.raises(dcoraid.api.APIAuthorizationError,
-                       match=""):
+                       match=None):
         uj.task_upload_resources()