Merge branch 'develop' into dependabot/pip/werkzeug-3.1.6

Author: douglowe

app/ro_crates/routes/post_routes.py

@@ -111,4 +111,6 @@ def validate_ro_crate_metadata(json_data) -> tuple[Response, int]:
     else:
         profile_name = None
 
-    return queue_ro_crate_metadata_validation_task(crate_json, profile_name)
+    profiles_path = current_app.config["PROFILES_PATH"]
+
+    return queue_ro_crate_metadata_validation_task(crate_json, profile_name, profiles_path=profiles_path)

app/services/validation_service.py

@@ -61,14 +61,15 @@ def queue_ro_crate_validation_task(
 
 
 def queue_ro_crate_metadata_validation_task(
-    crate_json: str, profile_name=None, webhook_url=None
+    crate_json: str, profile_name=None, webhook_url=None, profiles_path=None
 ) -> tuple[Response, int]:
     """
     Queues an RO-Crate for validation with Celery.
 
     :param crate_id: The ID of the RO-Crate to validate.
     :param profile_name: The profile to validate against.
     :param webhook_url: The URL to POST the validation results to.
+    :param profiles_path: A path to the profile definition directory.
     :return: A tuple containing a JSON response and an HTTP status code.
     :raises: Exception: If an error occurs whilst queueing the task.
     """
@@ -90,7 +91,8 @@ def queue_ro_crate_metadata_validation_task(
         result = process_validation_task_by_metadata.delay(
                                                      crate_json,
                                                      profile_name,
-                                                     webhook_url
+                                                     webhook_url,
+                                                     profiles_path
                 )
         if webhook_url:
             return jsonify({"message": "Validation in progress"}), 202

app/tasks/validation_tasks.py

@@ -7,6 +7,7 @@
 import logging
 import os
 import shutil
+import json
 from typing import Optional
 
 from rocrate_validator import services
@@ -22,7 +23,6 @@
     find_validation_object_on_minio
 )
 from app.utils.webhook_utils import send_webhook_notification
-from app.utils.file_utils import build_metadata_only_rocrate
 
 logger = logging.getLogger(__name__)
 
@@ -98,32 +98,27 @@ def process_validation_task_by_id(
 
 @celery.task
 def process_validation_task_by_metadata(
-    crate_json: str, profile_name: str | None, webhook_url: str | None
+    crate_json: str, profile_name: str | None, webhook_url: str | None, profiles_path: Optional[str] = None
 ) -> ValidationResult | str:
     """
     Background task to process the RO-Crate validation for a given json metadata string.
 
     :param crate_json: A string containing the RO-Crate JSON metadata to validate.
     :param profile_name: The name of the validation profile to use. Defaults to None.
     :param webhook_url: The webhook URL to send notifications to. Defaults to None.
+    :param profiles_path: The path to the profiles definition directory. Defaults to None.
     :raises Exception: If an error occurs during the validation process.
 
     :todo: Replace the Crate ID with a more comprehensive system, and replace profile name with URI.
     """
 
-    skip_checks_list = ['ro-crate-1.1_12.1']
-    file_path = None
-
     try:
-        # Fetch the RO-Crate from MinIO using the provided ID:
-        file_path = build_metadata_only_rocrate(crate_json)
-
-        logging.info(f"Processing validation task for {file_path}")
+        logging.info("Processing validation task for provided metadata string")
 
         # Perform validation:
-        validation_result = perform_ro_crate_validation(file_path,
+        validation_result = perform_metadata_validation(crate_json,
                                                         profile_name,
-                                                        skip_checks_list
+                                                        profiles_path
                                                         )
 
         if isinstance(validation_result, str):
@@ -132,9 +127,9 @@ def process_validation_task_by_metadata(
             raise Exception(f"Validation failed: {validation_result}")
 
         if not validation_result.has_issues():
-            logging.info(f"RO Crate {file_path} is valid.")
+            logging.info("RO Crate metadata is valid.")
         else:
-            logging.info(f"RO Crate {file_path} is invalid.")
+            logging.info("RO Crate metadata is invalid.")
 
         if webhook_url:
             send_webhook_notification(webhook_url, validation_result.to_json())
@@ -148,10 +143,6 @@ def process_validation_task_by_metadata(
             send_webhook_notification(webhook_url, error_data)
 
     finally:
-        # Clean up the temporary file if it was created:
-        if file_path and os.path.exists(file_path):
-            shutil.rmtree(file_path)
-
         if isinstance(validation_result, str):
             return validation_result
         else:
@@ -196,6 +187,39 @@ def perform_ro_crate_validation(
         return str(e)
 
 
+def perform_metadata_validation(
+    crate_json: str, profile_name: str | None, skip_checks_list: Optional[list] = None, profiles_path: Optional[str] = None
+) -> ValidationResult | str:
+    """
+    Validates only RO-Crate metadata provided as a json string.
+
+    :param crate_json: The JSON string containing the metadata
+    :param profile_name: The name of the validation profile to use. Defaults to None. If None, the CRS4 validator will
+        attempt to determine the profile.
+    :param profiles_path: The path to the profiles definition directory
+    :param skip_checks_list: A list of checks to skip, if needed
+    :return: The validation result.
+    :raises Exception: If an error occurs during the validation process.
+    """
+
+    try:
+        logging.info(f"Validating ro-crate metadata with profile {profile_name}")
+
+        settings = services.ValidationSettings(
+            **({"metadata_only": True}),
+            **({"metadata_dict": json.loads(crate_json)}),
+            **({"profile_identifier": profile_name} if profile_name else {}),
+            **({"skip_checks": skip_checks_list} if skip_checks_list else {}),
+            **({"profiles_path": profiles_path} if profiles_path else {})
+        )
+
+        return services.validate(settings)
+
+    except Exception as e:
+        logging.error(f"Unexpected error during validation: {e}")
+        return str(e)
+
+
 def check_ro_crate_exists(
         minio_client: object,
         bucket_name: str,

app/utils/file_utils.py

@@ -1,9 +1,9 @@
 celery==5.6.2
 minio==7.2.20
 requests==2.32.5
-Flask==3.1.2
+Flask==3.1.3
 Werkzeug==3.1.6
-redis==7.1.1
+redis==7.2.0
 python-dotenv==1.2.1
 apiflask==3.0.2
 roc-validator==0.8

requirements.in

@@ -59,7 +59,7 @@ email-validator==2.3.0
     # via pydantic
 enum-tools==0.12.0
     # via roc-validator
-flask==3.1.2
+flask==3.1.3
     # via
     #   -r requirements.in
     #   apiflask
@@ -145,7 +145,7 @@ rdflib[html]==7.1.4
     #   owlrl
     #   pyshacl
     #   roc-validator
-redis==7.1.1
+redis==7.2.0
     # via -r requirements.in
 requests==2.32.5
     # via

requirements.txt

@@ -141,24 +141,26 @@ def test_validate_fails_missing_elements(client: FlaskClient, crate_id: str, pay
 
 # Test POST API: /v1/ro_crates/validate_metadata
 
+# TODO: Write tests for profiles_path environment variable. This will require a refactoring of the create_app function.
 @pytest.mark.parametrize(
-    "payload, status_code, response_json",
+    "payload, status_code, response_json, profiles_path",
     [
         (
             {
                 "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}',
                 "profile_name": "default"
-            }, 200, {"status": "success"}
+            }, 200, {"status": "success"}, None
         ),
         (
             {
                 "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}',
-            }, 200, {"status": "success"}
+            }, 200, {"status": "success"}, None
         ),
     ],
     ids=["success_with_all_fields", "success_without_profile_name"]
 )
-def test_validate_metadata_success(client: FlaskClient, payload: dict, status_code: int, response_json: dict):
+def test_validate_metadata_success(client: FlaskClient, payload: dict, status_code: int,
+                                   response_json: dict, profiles_path: str):
     with patch("app.ro_crates.routes.post_routes.queue_ro_crate_metadata_validation_task") as mock_queue:
         mock_queue.return_value = (response_json, status_code)
 
@@ -167,7 +169,7 @@ def test_validate_metadata_success(client: FlaskClient, payload: dict, status_co
         crate_json = payload["crate_json"] if "crate_json" in payload else None
         profile_name = payload["profile_name"] if "profile_name" in payload else None
 
-        mock_queue.assert_called_once_with(crate_json, profile_name)
+        mock_queue.assert_called_once_with(crate_json, profile_name, profiles_path=profiles_path)
         assert response.status_code == status_code
         assert response.json == response_json
 

tests/test_api_routes.py

@@ -141,32 +141,32 @@ def test_queue_ro_crate_validation_task_failure(
 # Test function: queue_ro_crate_metadata_validation_task
 
 @pytest.mark.parametrize(
-        "crate_json, profile, webhook, status_code, return_value, response_json, delay_side_effect",
+        "crate_json, profile, webhook, status_code, return_value, response_json, delay_side_effect, profiles_path",
         [
             (
                 '{"@context": "https://w3id.org/ro/crate/1.1/context"}',
                 "default", "http://webhook",
                 202, None, {"message": "Validation in progress"},
-                None
+                None, None
             ),
             (
                 '{"@context": "https://w3id.org/ro/crate/1.1/context"}',
                 "default", None,
                 200, {"status": "ok"}, {"result": {"status": "ok"}},
-                None
+                None, None
             ),
             (
                 '{"@context": "https://w3id.org/ro/crate/1.1/context"}',
                 "default", "http://webhook",
                 500, None, {"error": "Celery error"},
-                Exception("Celery error")
+                Exception("Celery error"), None
             ),
         ],
         ids=["success_with_webhook", "success_without_webhook", "failure_celery_error"]
 )
 def test_queue_metadata(flask_app, crate_json: dict, profile: str, webhook: str,
                         status_code: int, return_value: dict, response_json: dict,
-                        delay_side_effect: Exception):
+                        delay_side_effect: Exception, profiles_path: str):
     with patch("app.services.validation_service.process_validation_task_by_metadata.delay",
                side_effect=delay_side_effect) as mock_delay:
         mock_result = MagicMock()
@@ -175,9 +175,9 @@ def test_queue_metadata(flask_app, crate_json: dict, profile: str, webhook: str,
         if delay_side_effect is None:
             mock_delay.return_value = mock_result
 
-        response, status = queue_ro_crate_metadata_validation_task(crate_json, profile, webhook)
+        response, status = queue_ro_crate_metadata_validation_task(crate_json, profile, webhook, profiles_path)
 
-        mock_delay.assert_called_once_with(crate_json, profile, webhook)
+        mock_delay.assert_called_once_with(crate_json, profile, webhook, profiles_path)
         assert status == status_code
         assert response.json == response_json
 
@@ -197,7 +197,8 @@ def test_queue_metadata(flask_app, crate_json: dict, profile: str, webhook: str,
                 "{}",
                 422, "Required parameter crate_json is empty"
             ),
-        ]
+        ],
+        ids=["missing_crate_json","invalid_json","empty_json"]
 )
 def test_queue_metadata_json_errors(flask_app, crate_json: str, status_code: int, response_error: str):
     response, status = queue_ro_crate_metadata_validation_task(crate_json)