API for validating RO-Crate metadata only

.github/workflows/test_docker.yml

@@ -0,0 +1,35 @@
+name: Integration Tests
+
+on:
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest requests
+
+      - name: Build Docker Compose Containers
+        run: |
+          cp example.env .env
+          docker compose -f docker-compose-develop.yml build 
+
+      - name: Spin Up Docker Compose and Run Tests
+        run: pytest tests/test_integration.py -v
+
+      - name: Ensure that Docker Compose is Shutdown
+        if: always()
+        run: docker compose down

app/__init__.py

@@ -9,7 +9,7 @@
 from apiflask import APIFlask
 
 from app.celery_worker import make_celery, celery
-from app.ro_crates.routes import ro_crates_post_bp, ro_crates_get_bp
+from app.ro_crates.routes import v1_post_bp, v1_get_bp
 from app.utils.config import DevelopmentConfig, ProductionConfig, make_celery
 
 
@@ -20,8 +20,8 @@ def create_app() -> APIFlask:
     :return: Flask: A configured Flask application instance.
     """
     app = APIFlask(__name__)
-    app.register_blueprint(ro_crates_post_bp, url_prefix="/ro_crates")
-    app.register_blueprint(ro_crates_get_bp, url_prefix="/ro_crates")
+    app.register_blueprint(v1_post_bp, url_prefix="/v1/ro_crates")
+    app.register_blueprint(v1_get_bp, url_prefix="/v1/ro_crates")
 
     # Load configuration:
     if os.getenv("FLASK_ENV") == "production":

app/ro_crates/routes/__init__.py

@@ -4,14 +4,8 @@
 # License: MIT
 # Copyright (c) 2025 eScience Lab, The University of Manchester
 
-from apiflask import APIBlueprint
-
 from app.ro_crates.routes.post_routes import post_routes_bp
 from app.ro_crates.routes.get_routes import get_routes_bp
 
-#ro_crates_bp = APIBlueprint("ro_crates", __name__)
-
-#ro_crates_bp.register_blueprint(post_routes_bp, url_prefix="/post")
-
-ro_crates_post_bp = post_routes_bp
-ro_crates_get_bp = get_routes_bp
+v1_post_bp = post_routes_bp
+v1_get_bp = get_routes_bp

app/ro_crates/routes/post_routes.py

@@ -8,18 +8,25 @@
 from apiflask.fields import Integer, String
 from flask import request, Response
 
-from app.services.validation_service import queue_ro_crate_validation_task
+from app.services.validation_service import (
+    queue_ro_crate_validation_task, 
+    queue_ro_crate_metadata_validation_task
+)
 
 post_routes_bp = APIBlueprint("post_routes", __name__)
 
-class validate_data(Schema):
+class ValidateData(Schema):
     crate_id = String(required=True)
     profile_name = String(required=False)
     webhook_url = String(required=False)
 
+class ValidateJSON(Schema):
+    crate_json = String(required=True)
+    profile_name = String(required=False)
+
 
 @post_routes_bp.post("/validate_by_id")
-@post_routes_bp.input(validate_data(partial=True), location='json')
+@post_routes_bp.input(ValidateData(partial=True), location='json')
 def validate_ro_crate_from_id(json_data) -> tuple[Response, int]:
     """
     Endpoint to validate an RO-Crate using its ID from MinIO.
@@ -53,7 +60,7 @@ def validate_ro_crate_from_id(json_data) -> tuple[Response, int]:
     return queue_ro_crate_validation_task(crate_id, profile_name, webhook_url)
 
 @post_routes_bp.post("/validate_by_id_no_webhook")
-@post_routes_bp.input(validate_data(partial=True), location='json') # -> json_data
+@post_routes_bp.input(ValidateData(partial=True), location='json') # -> json_data
 def validate_ro_crate_from_id_no_webhook(json_data) -> tuple[Response, int]:
     """
     Endpoint to validate an RO-Crate using its ID from MinIO.
@@ -80,3 +87,34 @@ def validate_ro_crate_from_id_no_webhook(json_data) -> tuple[Response, int]:
         profile_name = None
 
     return queue_ro_crate_validation_task(crate_id, profile_name)
+
+
+@post_routes_bp.post("/validate_metadata")
+@post_routes_bp.input(ValidateJSON(partial=True), location='json') # -> json_data
+def validate_ro_crate_metadata(json_data) -> tuple[Response, int]:
+    """
+    Endpoint to validate an RO-Crate JSON file uploaded to the Service.
+
+    Parameters:
+    - **crate_json**: The RO-Crate JSON-LD, as a string. _Required_
+    - **profile_name**: The profile name for validation. _Optional_.
+
+    Returns:
+    - A tuple containing the validation task's response and an HTTP status code.
+
+    Raises:
+    - KeyError: If required parameters (`crate_json`) are missing.
+    """
+
+    try:
+        crate_json = json_data['crate_json']
+    except:
+        raise KeyError("Missing required parameter: 'crate_json'")
+
+    try:
+        profile_name = json_data['profile_name']
+    except:
+        profile_name = None
+
+    return queue_ro_crate_metadata_validation_task(crate_json, profile_name)
+

app/services/validation_service.py

@@ -10,6 +10,7 @@
 
 from app.tasks.validation_tasks import (
     process_validation_task_by_id,
+    process_validation_task_by_metadata,
     return_ro_crate_validation
     )
 
@@ -42,6 +43,39 @@ def queue_ro_crate_validation_task(
         return jsonify({"error": str(e)}), 500
 
 
+def queue_ro_crate_metadata_validation_task(
+    crate_json, profile_name=None, webhook_url=None
+) -> tuple[Response, int]:
+    """
+    Queues an RO-Crate for validation with Celery.
+
+    :param crate_id: The ID of the RO-Crate to validate.
+    :param profile_name: The profile to validate against.
+    :param webhook_url: The URL to POST the validation results to.
+    :return: A tuple containing a JSON response and an HTTP status code.
+    :raises: Exception: If an error occurs whilst queueing the task.
+    """
+
+    logging.info(f"Processing: {crate_json}, {profile_name}, {webhook_url}")
+
+    if not crate_json:
+        return jsonify({"error": "Missing required parameter: crate_json"}), 400
+
+    try:
+        result = process_validation_task_by_metadata.delay(
+                                                     crate_json,
+                                                     profile_name,
+                                                     webhook_url
+                )
+        if webhook_url:
+            return jsonify({"message": "Validation in progress"}), 202
+        else:
+            return jsonify({"result": result.get()}), 200
+
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+
 def get_ro_crate_validation_task(
     crate_id
 ) -> tuple[Response, int]:

app/tasks/validation_tasks.py

@@ -6,6 +6,8 @@
 
 import logging
 import os
+import shutil
+from typing import Optional
 
 from rocrate_validator import services
 from rocrate_validator.models import ValidationResult
@@ -17,6 +19,7 @@
     get_validation_status_from_minio
 )
 from app.utils.webhook_utils import send_webhook_notification
+from app.utils.file_utils import build_metadata_only_rocrate
 
 logger = logging.getLogger(__name__)
 
@@ -78,15 +81,75 @@ def process_validation_task_by_id(
             os.remove(file_path)
 
 
+@celery.task
+def process_validation_task_by_metadata(
+    crate_json: str, profile_name: str | None, webhook_url: str | None
+) -> ValidationResult | str:
+    """
+    Background task to process the RO-Crate validation for a given json metadata string.
+
+    :param crate_json: A string containing the RO-Crate JSON metadata to validate.
+    :param profile_name: The name of the validation profile to use. Defaults to None.
+    :param webhook_url: The webhook URL to send notifications to. Defaults to None.
+    :raises Exception: If an error occurs during the validation process.
+
+    :todo: Replace the Crate ID with a more comprehensive system, and replace profile name with URI.
+    """
+
+    skip_checks_list = ['ro-crate-1.1_12.1']
+    file_path = None
+
+    try:
+        # Fetch the RO-Crate from MinIO using the provided ID:
+        file_path = build_metadata_only_rocrate(crate_json)
+
+        logging.info(f"Processing validation task for {file_path}")
+
+        # Perform validation:
+        validation_result = perform_ro_crate_validation(file_path,
+                                                        profile_name,
+                                                        skip_checks_list
+                            )
+
+        if isinstance(validation_result, str):
+            logging.error(f"Validation failed: {validation_result}")
+            # TODO: Send webhook with failure notification
+            raise Exception(f"Validation failed: {validation_result}")
+
+        if not validation_result.has_issues():
+            logging.info(f"RO Crate {file_path} is valid.")
+        else:
+            logging.info(f"RO Crate {file_path} is invalid.")
+
+        if webhook_url:
+            send_webhook_notification(webhook_url, validation_result.to_json())
+
+    except Exception as e:
+        logging.error(f"Error processing validation task: {e}")
+
+        # Send failure notification via webhook
+        error_data = {"profile_name": profile_name, "error": str(e)}
+        send_webhook_notification(webhook_url, error_data)
+
+    finally:
+        # Clean up the temporary file if it was created:
+        if file_path and os.path.exists(file_path):
+            shutil.rmtree(file_path)
+
+        return validation_result.to_json()
+
+
+
 def perform_ro_crate_validation(
-    file_path: str, profile_name: str | None
+    file_path: str, profile_name: str | None, skip_checks_list: Optional[list] = None
 ) -> ValidationResult | str:
     """
     Validates an RO-Crate using the provided file path and profile name.
 
     :param file_path: The path to the RO-Crate file to validate
     :param profile_name: The name of the validation profile to use. Defaults to None. If None, the CRS4 validator will
         attempt to determine the profile.
+    :param skip_checks_list: A list of checks to skip, if needed
     :return: The validation result.
     :raises Exception: If an error occurs during the validation process.
     """
@@ -102,8 +165,8 @@ def perform_ro_crate_validation(
         )
         settings = services.ValidationSettings(
             rocrate_uri=full_file_path,
-            # Only include profile_identifier if the profile_name is provided:
             **({"profile_identifier": profile_name} if profile_name else {}),
+            **({"skip_checks": skip_checks_list} if skip_checks_list else {})
         )
 
         return services.validate(settings)

app/utils/file_utils.py

@@ -0,0 +1,54 @@
+"""Utility methods for interacting with the File System."""
+
+# Author: Douglas Lowe, Alexander Hambley
+# License: MIT
+# Copyright (c) 2025 eScience Lab, The University of Manchester
+
+import json
+import logging
+import os
+import tempfile
+
+from dotenv import load_dotenv
+
+
+logger = logging.getLogger(__name__)
+
+
+def build_metadata_only_rocrate(crate_json: str) -> str:
+    """
+    Creates a temporary directory for an empty RO-Crate,
+    and saves the JSON string as a metadata file.
+
+    :param crate_json: The metadata string.
+    :return: The local file path where the RO-Crate is saved.
+    :raises ValueError: If the required environment variables are not set.
+    :raises Exception: If an unexpected error occurs during the operation.
+    """
+
+    load_dotenv()
+
+    try:
+        # Prepare temporary file path to store RO Crate for validation:
+        temp_dir = tempfile.mkdtemp()
+        file_path = os.path.join(temp_dir, 'ro-crate-metadata.json')
+
+        logging.info(
+            f"Creating RO-Crate Metadata file. File path: {file_path}"
+        )
+        with open(file_path, 'w') as f:
+            f.write(crate_json)
+        logging.info(
+            f"RO-Crate metadata successfully saved to {file_path}."
+        )
+
+        return temp_dir
+
+    except ValueError as value_error:
+        logging.error(f"Configuration Error: {value_error}")
+        raise
+
+    except Exception as e:
+        logging.error(f"Unexpected error creating RO-Crate metadata: {e}")
+        raise
+

requirements.testing.txt

@@ -0,0 +1,10 @@
+celery~=5.4.0
+minio~=7.2.10
+requests~=2.32.3
+Flask~=3.0.3
+Werkzeug~=3.0.4
+redis~=5.2.0
+python-dotenv~=1.0.1
+apiflask~=2.4.0
+roc-validator~=0.6.5
+pytest~=8.4.1