added rocrate_bucket and root_path to all required functions now

Author: douglowe

app/tasks/validation_tasks.py

@@ -29,12 +29,14 @@
 
 @celery.task
 def process_validation_task_by_id(
-    crate_id: str, profile_name: str | None, webhook_url: str | None
+    minio_bucket: str, crate_id: str, root_path: str, profile_name: str | None, webhook_url: str | None
 ) -> None:
     """
     Background task to process the RO-Crate validation by ID.
 
+    :param minio_bucket: The MinIO bucket containing the RO-Crate.
     :param crate_id: The ID of the RO-Crate to validate.
+    :param root_path: The root path containing the RO-Crate.
     :param profile_name: The name of the validation profile to use. Defaults to None.
     :param webhook_url: The webhook URL to send notifications to. Defaults to None.
     :raises Exception: If an error occurs during the validation process.
@@ -46,7 +48,7 @@ def process_validation_task_by_id(
 
     try:
         # Fetch the RO-Crate from MinIO using the provided ID:
-        file_path = fetch_ro_crate_from_minio(crate_id)
+        file_path = fetch_ro_crate_from_minio(minio_bucket, crate_id, root_path)
 
         logging.info(f"Processing validation task for {file_path}")
 
@@ -59,12 +61,12 @@ def process_validation_task_by_id(
             raise Exception(f"Validation failed: {validation_result}")
 
         if not validation_result.has_issues():
-            logging.info(f"RO Crate {file_path} is valid.")
+            logging.info(f"RO Crate {crate_id} is valid.")
         else:
-            logging.info(f"RO Crate {file_path} is invalid.")
+            logging.info(f"RO Crate {crate_id} is invalid.")
 
         # Update the validation status in MinIO:
-        update_validation_status_in_minio(crate_id, validation_result.to_json())
+        update_validation_status_in_minio(minio_bucket, crate_id, root_path, validation_result.to_json())
 
         # TODO: Prepare the data to send to the webhook, and send the webhook notification.
 
@@ -202,7 +204,7 @@ def check_ro_crate_exists(
     logging.info(f"Checking for existence of RO-Crate {crate_id}")
 
     minio_client = get_minio_client()
-    if find_rocrate_object_on_minio(crate_id, minio_client, bucket_name, storage_path=root_path):
+    if find_rocrate_object_on_minio(crate_id, minio_client, bucket_name, root_path):
         return True
     else:
         return False
@@ -225,7 +227,7 @@ def check_validation_exists(
     logging.info(f"Checking for existence of RO-Crate {crate_id}")
 
     minio_client = get_minio_client()
-    if find_validation_object_on_minio(crate_id, minio_client, bucket_name, storage_path=root_path):
+    if find_validation_object_on_minio(crate_id, minio_client, bucket_name, root_path):
         return True
     else:
         return False

app/utils/minio_utils.py

@@ -18,48 +18,49 @@
 logger = logging.getLogger(__name__)
 
 
-def fetch_ro_crate_from_minio(minio_bucket: str, crate_id: str) -> str:
+def fetch_ro_crate_from_minio(minio_bucket: str, crate_id: str, root_path: str) -> str:
     """
     Fetches an RO-Crate from MinIO based on the crate ID. Downloads the crate as a file and returns local file path.
 
     :param minio_bucket: The MinIO bucket containing the RO-Crate.
     :param crate_id: The ID of the RO-Crate to fetch from MinIO.
+    :param root_path: The root path containing the RO-Crate.
     :return: The local file path where the RO-Crate is saved.
     """
 
     minio_client = get_minio_client()
 
-    rocrate_object = find_rocrate_object_on_minio(crate_id, minio_client, minio_bucket)
+    rocrate_object = find_rocrate_object_on_minio(crate_id, minio_client, minio_bucket, root_path)
 
-    rocrate_path = rocrate_object.object_name
-    rocrate_name = rocrate_path.split('/')[-1]
+    rocrate_minio_path = rocrate_object.object_name
+    rocrate_name = rocrate_minio_path.split('/')[-1]
 
     temp_dir = tempfile.mkdtemp()
-    root_path = os.path.join(temp_dir, rocrate_name)
+    local_root_path = os.path.join(temp_dir, rocrate_name)
 
     logging.info(
-        f"Fetching RO-Crate {rocrate_name} from MinIO bucket {minio_bucket}. File path {root_path}"
+        f"Fetching RO-Crate {rocrate_name} from MinIO bucket {minio_bucket}. File path {local_root_path}"
     )
 
     if rocrate_object.is_dir:
-        os.makedirs(os.path.dirname(root_path), exist_ok=True)
+        os.makedirs(os.path.dirname(local_root_path), exist_ok=True)
 
-        objects_list = get_minio_object_list(rocrate_path, minio_client, minio_bucket, recursive=True)
+        objects_list = get_minio_object_list(rocrate_minio_path, minio_client, minio_bucket, recursive=True)
         for obj in objects_list:
-            relative_path = obj.object_name[len(rocrate_path):].lstrip("/")
-            local_file_path = os.path.join(root_path, relative_path)
+            relative_path = obj.object_name[len(rocrate_minio_path):].lstrip("/")
+            local_file_path = os.path.join(local_root_path, relative_path)
             os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
             download_file_from_minio(minio_client, minio_bucket, obj.object_name, local_file_path)
 
     else:
-        file_path = root_path
-        download_file_from_minio(minio_client, minio_bucket, rocrate_path, file_path)
+        file_path = local_root_path
+        download_file_from_minio(minio_client, minio_bucket, rocrate_minio_path, file_path)
 
     logging.info(
-        f"RO-Crate {rocrate_name} fetched successfully and saved to {root_path}."
+        f"RO-Crate {rocrate_name} fetched successfully and saved to {local_root_path}."
     )
 
-    return root_path
+    return local_root_path
 
 
 def update_validation_status_in_minio(minio_bucket: str, crate_id: str, validation_status: str) -> None:
@@ -185,15 +186,15 @@ def download_file_from_minio(minio_client: object, minio_bucket: str, object_pat
         raise InvalidAPIUsage(f"Unknown Error: {e}", 500)
 
 
-def find_validation_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, storage_path: str) -> object:
+def find_validation_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, root_path: str) -> object:
     """
     Checks that the requested object exists on the MinIO instance.
 
     If it does not exist then a False value is returned.
     If it does exist then the minio.datatypes.Object is returned.
 
     :param rocrate_id: string containing the name of ro-crate
-    :param storage_path: string containing the path within which the ro-crate should be
+    :param root_path: string containing the path within which the ro-crate should be
     :param minio_client: minio object
     :param minio_bucket: string containing bucket on minio
     :return return_object: rocrate object we require
@@ -202,8 +203,8 @@ def find_validation_object_on_minio(rocrate_id: str, minio_client, minio_bucket:
 
     logging.info(f"Finding Validation result: {rocrate_id}_validation/validation_status.txt")
 
-    if storage_path:
-        file_path = f"{storage_path}/{rocrate_id}_validation/validation_status.txt"
+    if root_path:
+        file_path = f"{root_path}/{rocrate_id}_validation/validation_status.txt"
     else:
         file_path = f"{rocrate_id}_validation/validation_status.txt"
 
@@ -222,15 +223,15 @@ def find_validation_object_on_minio(rocrate_id: str, minio_client, minio_bucket:
         return return_object
 
 
-def find_rocrate_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, storage_path: str) -> object | bool:
+def find_rocrate_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, root_path: str) -> object | bool:
     """
     Checks that the requested object exists on the MinIO instance.
 
     If it does not exist then a False value is returned.
     If it does exist then the minio.datatypes.Object is returned.
 
     :param rocrate_id: string containing the name of ro-crate
-    :param storage_path: string containing the path within which the ro-crate should be
+    :param root_path: string containing the path within which the ro-crate should be
     :param minio_client: minio object
     :param minio_bucket: string containing bucket on minio
     :return return_object or False: rocrate object we require, or False result
@@ -239,8 +240,8 @@ def find_rocrate_object_on_minio(rocrate_id: str, minio_client, minio_bucket: st
 
     logging.info(f"Finding RO-Crate: {rocrate_id}")
 
-    if storage_path:
-        rocrate_path = f"{storage_path}/{rocrate_id}"
+    if root_path:
+        rocrate_path = f"{root_path}/{rocrate_id}"
     else:
         rocrate_path = rocrate_id