diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..6a7695c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml new file mode 100644 index 0000000..9af7b23 --- /dev/null +++ b/.github/workflows/test_docker.yml @@ -0,0 +1,35 @@ +name: Integration Tests + +on: + pull_request: + branches: [ develop ] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest requests minio docker + + - name: Build Docker Compose Containers + run: | + cp example.env .env + docker compose -f docker-compose-develop.yml build + + - name: Spin Up Docker Compose and Run Tests + run: pytest -s -v tests/test_integration.py + + - name: Ensure that Docker Compose is Shutdown + if: always() + run: docker compose down diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml new file mode 100644 index 0000000..7c0a996 --- /dev/null +++ b/.github/workflows/unit_tests.yml @@ -0,0 +1,28 @@ +name: Unit Tests + +on: + pull_request: + branches: [ develop ] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest pytest-mock + + - name: Run tests (excluding integration tests) + run: | + python -m pytest --ignore=tests/test_integration.py diff --git a/README.md b/README.md index 36a89a0..b1de0e9 100644 --- a/README.md +++ b/README.md @@ -70,12 +70,20 @@ For testing locally developed containers use the alternate Docker Compose file: ## Example Usage -``` +Submission of validation of RO-Crate with the ID of `ro_crate_1`. No webhook is used here: +```bash curl -X 'POST' \ - 'http://localhost:5001/ro_crates/validate_by_id_no_webhook' \ + 'http://localhost:5001/ro_crates/v1/ro_crate_1/validation' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{}' +``` + +Retrieval of validation result for RO-Crate `ro_crate_1`: +```bash +curl -X 'GET' \ + 'http://localhost:5001/ro_crates/v1/ro_crate_1/validation' \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ - -d '{ - "crate_id": "1" -}' + -d '{}' ``` diff --git a/app/__init__.py b/app/__init__.py index cee8c91..95a3d98 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -8,9 +8,9 @@ from apiflask import APIFlask -from app.celery_worker import make_celery, celery -from app.ro_crates.routes import ro_crates_post_bp, ro_crates_get_bp -from app.utils.config import DevelopmentConfig, ProductionConfig, make_celery +from app.ro_crates.routes import v1_post_bp, v1_get_bp +from app.utils.config import DevelopmentConfig, ProductionConfig, InvalidAPIUsage, make_celery +from flask import jsonify def create_app() -> APIFlask: @@ -20,8 +20,12 @@ def create_app() -> APIFlask: :return: Flask: A configured Flask application instance. """ app = APIFlask(__name__) - app.register_blueprint(ro_crates_post_bp, url_prefix="/ro_crates") - app.register_blueprint(ro_crates_get_bp, url_prefix="/ro_crates") + app.register_blueprint(v1_post_bp, url_prefix="/v1/ro_crates") + app.register_blueprint(v1_get_bp, url_prefix="/v1/ro_crates") + + @app.errorhandler(InvalidAPIUsage) + def invalid_api_usage(e): + return jsonify(e.to_dict()), e.status_code # Load configuration: if os.getenv("FLASK_ENV") == "production": diff --git a/app/celery_worker.py b/app/celery_worker.py index d3be21a..778407c 100644 --- a/app/celery_worker.py +++ b/app/celery_worker.py @@ -5,42 +5,6 @@ # Copyright (c) 2025 eScience Lab, The University of Manchester from celery import Celery -from flask import Flask - - -def make_celery(app: Flask = None) -> Celery: - """ - Create and configure a Celery instance using Flask configuration. - - :param app: The Flask application. Configuration will be used to initialise Celery. - :return: Celery: A configured Celery instance. - :raises ValueError: If the Flask configuration values are not provided. - """ - if not app: - raise ValueError( - "A Flask application instance must be provided to configure Celery." - ) - - if "CELERY_RESULT_BACKEND" not in app.config: - raise ValueError( - "Missing configuration: 'CELERY_RESULT_BACKEND' is not defined in the Flask app config." - ) - - if "CELERY_BROKER_URL" not in app.config: - raise ValueError( - "Missing configuration: 'CELERY_BROKER_URL' is not defined in the Flask app config." - ) - - celery_instance = Celery( - app.import_name, - backend=app.config["CELERY_RESULT_BACKEND"], - broker=app.config["CELERY_BROKER_URL"], - ) - - if app: - celery_instance.conf.update(app.config) - - return celery_instance celery = Celery() diff --git a/app/ro_crates/routes/__init__.py b/app/ro_crates/routes/__init__.py index 0a20b53..da8a903 100644 --- a/app/ro_crates/routes/__init__.py +++ b/app/ro_crates/routes/__init__.py @@ -4,14 +4,8 @@ # License: MIT # Copyright (c) 2025 eScience Lab, The University of Manchester -from apiflask import APIBlueprint - from app.ro_crates.routes.post_routes import post_routes_bp from app.ro_crates.routes.get_routes import get_routes_bp -#ro_crates_bp = APIBlueprint("ro_crates", __name__) - -#ro_crates_bp.register_blueprint(post_routes_bp, url_prefix="/post") - -ro_crates_post_bp = post_routes_bp -ro_crates_get_bp = get_routes_bp \ No newline at end of file +v1_post_bp = post_routes_bp +v1_get_bp = get_routes_bp diff --git a/app/ro_crates/routes/get_routes.py b/app/ro_crates/routes/get_routes.py index 39d6064..d6b23c6 100644 --- a/app/ro_crates/routes/get_routes.py +++ b/app/ro_crates/routes/get_routes.py @@ -5,25 +5,45 @@ # Copyright (c) 2025 eScience Lab, The University of Manchester from apiflask import APIBlueprint, Schema -from apiflask.fields import Integer, String -from flask import request, Response +from apiflask.fields import String, Boolean +from marshmallow.fields import Nested +from flask import Response from app.services.validation_service import get_ro_crate_validation_task get_routes_bp = APIBlueprint("get_routes", __name__) -class validate_data(Schema): - crate_id = String(required=True) +class MinioConfig(Schema): + endpoint = String(required=True) + accesskey = String(required=True) + secret = String(required=True) + ssl = Boolean(required=True) + bucket = String(required=True) -@get_routes_bp.get("/get_validation_by_id") -@get_routes_bp.input(validate_data(partial=True), location='json') -def get_ro_crate_validation_by_id(json_data) -> tuple[Response, int]: + +class ValidateResult(Schema): + minio_config = Nested(MinioConfig, required=True) + root_path = String(required=False) + + +@get_routes_bp.get("/validation") +@get_routes_bp.input(ValidateResult(partial=False), location='json') +def get_ro_crate_validation_by_id(json_data, crate_id) -> tuple[Response, int]: """ Endpoint to obtain an RO-Crate validation result using its ID from MinIO. - Parameters: - - **crate_id**: The ID of the RO-Crate to validate. _Required_. + Path Parameters: + - **crate_id**: The RO-Crate ID. _Required_. + + Request Body Parameters: + - **minio_config**: The MinIO bucket containing the RO-Crate. _Required_ + - **endpoint**: Endpoint, e.g. 'localhost:9000' + - **accesskey**: Access key / username + - **secret**: Secret / password + - **ssl**: Use SSL encryption? True/False + - **bucket**: The MinIO bucket to access + - **root_path**: The root path containing the RO-Crate. _Optional_ Returns: - A tuple containing the validation result and an HTTP status code. @@ -32,9 +52,11 @@ def get_ro_crate_validation_by_id(json_data) -> tuple[Response, int]: - KeyError: If required parameters (`crate_id`) are missing. """ - try: - crate_id = json_data["crate_id"] - except: - raise KeyError("Missing required parameter: 'crate_id'") + minio_config = json_data["minio_config"] + + if "root_path" in json_data: + root_path = json_data["root_path"] + else: + root_path = None - return get_ro_crate_validation_task(crate_id) + return get_ro_crate_validation_task(minio_config, crate_id, root_path) diff --git a/app/ro_crates/routes/post_routes.py b/app/ro_crates/routes/post_routes.py index 12f849b..c1ebcdb 100644 --- a/app/ro_crates/routes/post_routes.py +++ b/app/ro_crates/routes/post_routes.py @@ -5,29 +5,57 @@ # Copyright (c) 2025 eScience Lab, The University of Manchester from apiflask import APIBlueprint, Schema -from apiflask.fields import Integer, String -from flask import request, Response +from apiflask.fields import String, Boolean +from marshmallow.fields import Nested +from flask import Response -from app.services.validation_service import queue_ro_crate_validation_task +from app.services.validation_service import ( + queue_ro_crate_validation_task, + queue_ro_crate_metadata_validation_task +) post_routes_bp = APIBlueprint("post_routes", __name__) -class validate_data(Schema): - crate_id = String(required=True) + +class MinioConfig(Schema): + endpoint = String(required=True) + accesskey = String(required=True) + secret = String(required=True) + ssl = Boolean(required=True) + bucket = String(required=True) + + +class ValidateCrate(Schema): + minio_config = Nested(MinioConfig, required=True) + root_path = String(required=False) profile_name = String(required=False) webhook_url = String(required=False) -@post_routes_bp.post("/validate_by_id") -@post_routes_bp.input(validate_data(partial=True), location='json') -def validate_ro_crate_from_id(json_data) -> tuple[Response, int]: +class ValidateJSON(Schema): + crate_json = String(required=True) + profile_name = String(required=False) + + +@post_routes_bp.post("/validation") +@post_routes_bp.input(ValidateCrate(partial=False), location='json') +def validate_ro_crate_via_id(json_data, crate_id) -> tuple[Response, int]: """ Endpoint to validate an RO-Crate using its ID from MinIO. - Parameters: - - **crate_id**: The ID of the RO-Crate to validate. _Required_. + Path Parameters: + - **crate_id**: The RO-Crate ID. _Required_. + + Request Body Parameters: + - **minio_config**: The MinIO bucket containing the RO-Crate. _Required_ + - **endpoint**: Endpoint, e.g. 'localhost:9000' + - **accesskey**: Access key / username + - **secret**: Secret / password + - **ssl**: Use SSL encryption? True/False + - **bucket**: The MinIO bucket to access + - **root_path**: The root path containing the RO-Crate. _Optional_ - **profile_name**: The profile name for validation. _Optional_. - - **webhook_url**: The webhook URL where validation results will be sent. _Required_. + - **webhook_url**: The webhook URL where validation results will be sent. _Optional_. Returns: - A tuple containing the validation task's response and an HTTP status code. @@ -36,47 +64,48 @@ def validate_ro_crate_from_id(json_data) -> tuple[Response, int]: - KeyError: If required parameters (`crate_id` or `webhook_url`) are missing. """ - try: - crate_id = json_data["crate_id"] - except: - raise KeyError("Missing required parameter: 'crate_id'") - try: + minio_config = json_data["minio_config"] + + if "root_path" in json_data: + root_path = json_data["root_path"] + else: + root_path = None + + if "webhook_url" in json_data: webhook_url = json_data["webhook_url"] - except: - raise KeyError("Missing required parameter: 'webhook_url'") + else: + webhook_url = None - try: + if "profile_name" in json_data: profile_name = json_data["profile_name"] - except: + else: profile_name = None - return queue_ro_crate_validation_task(crate_id, profile_name, webhook_url) + return queue_ro_crate_validation_task(minio_config, crate_id, root_path, profile_name, webhook_url) + -@post_routes_bp.post("/validate_by_id_no_webhook") -@post_routes_bp.input(validate_data(partial=True), location='json') # -> json_data -def validate_ro_crate_from_id_no_webhook(json_data) -> tuple[Response, int]: +@post_routes_bp.post("/validate_metadata") +@post_routes_bp.input(ValidateJSON(partial=False), location='json') # -> json_data +def validate_ro_crate_metadata(json_data) -> tuple[Response, int]: """ - Endpoint to validate an RO-Crate using its ID from MinIO. + Endpoint to validate an RO-Crate JSON file uploaded to the Service. - Parameters: - - **crate_id**: The ID of the RO-Crate to validate. _Required_. + Request Body Parameters: + - **crate_json**: The RO-Crate JSON-LD, as a string. _Required_ - **profile_name**: The profile name for validation. _Optional_. Returns: - A tuple containing the validation task's response and an HTTP status code. Raises: - - KeyError: If required parameters (`crate_id`) are missing. + - KeyError: If required parameters (`crate_json`) are missing. """ - try: - crate_id = json_data['crate_id'] - except: - raise KeyError("Missing required parameter: 'id'") + crate_json = json_data["crate_json"] - try: - profile_name = json_data['profile_name'] - except: + if "profile_name" in json_data: + profile_name = json_data["profile_name"] + else: profile_name = None - return queue_ro_crate_validation_task(crate_id, profile_name) + return queue_ro_crate_metadata_validation_task(crate_json, profile_name) diff --git a/app/services/validation_service.py b/app/services/validation_service.py index 669ab38..67dde94 100644 --- a/app/services/validation_service.py +++ b/app/services/validation_service.py @@ -5,24 +5,34 @@ # Copyright (c) 2025 eScience Lab, The University of Manchester import logging +import json from flask import jsonify, Response from app.tasks.validation_tasks import ( process_validation_task_by_id, - return_ro_crate_validation + process_validation_task_by_metadata, + return_ro_crate_validation, + check_ro_crate_exists, + check_validation_exists ) +from app.utils.config import InvalidAPIUsage +from app.utils.minio_utils import get_minio_client + + logger = logging.getLogger(__name__) def queue_ro_crate_validation_task( - crate_id, profile_name=None, webhook_url=None + minio_config, crate_id, root_path=None, profile_name=None, webhook_url=None ) -> tuple[Response, int]: """ Queues an RO-Crate for validation with Celery. + :param minio_config: Access settings for Minio instance containing the RO-Crate. :param crate_id: The ID of the RO-Crate to validate. + :param root_path: The root path containing the RO-Crate. :param profile_name: The profile to validate against. :param webhook_url: The URL to POST the validation results to. :return: A tuple containing a JSON response and an HTTP status code. @@ -30,34 +40,93 @@ def queue_ro_crate_validation_task( """ logging.info(f"Processing: {crate_id}, {profile_name}, {webhook_url}") + logging.info(f"Minio Bucket: {minio_config['bucket']}; Root path: {root_path}") - if not crate_id: - return jsonify({"error": "Missing required parameter: crate_id"}), 400 + minio_client = get_minio_client(minio_config) + + if check_ro_crate_exists(minio_client, minio_config["bucket"], crate_id, root_path): + logging.info("RO-Crate exists") + else: + logging.info("RO-Crate does not exist") + raise InvalidAPIUsage(f"No RO-Crate with prefix: {crate_id}", 400) try: - process_validation_task_by_id.delay(crate_id, profile_name, webhook_url) + process_validation_task_by_id.delay(minio_config, crate_id, root_path, profile_name, webhook_url) return jsonify({"message": "Validation in progress"}), 202 except Exception as e: return jsonify({"error": str(e)}), 500 +def queue_ro_crate_metadata_validation_task( + crate_json: str, profile_name=None, webhook_url=None +) -> tuple[Response, int]: + """ + Queues an RO-Crate for validation with Celery. + + :param crate_id: The ID of the RO-Crate to validate. + :param profile_name: The profile to validate against. + :param webhook_url: The URL to POST the validation results to. + :return: A tuple containing a JSON response and an HTTP status code. + :raises: Exception: If an error occurs whilst queueing the task. + """ + + logging.info(f"Processing: {crate_json}, {profile_name}, {webhook_url}") + + if not crate_json: + return jsonify({"error": "Missing required parameter: crate_json"}), 422 + + try: + json_dict = json.loads(crate_json) + except json.decoder.JSONDecodeError as err: + return jsonify({"error": f"Required parameter crate_json is not valid JSON: {err}"}), 422 + else: + if len(json_dict) == 0: + return jsonify({"error": "Required parameter crate_json is empty"}), 422 + + try: + result = process_validation_task_by_metadata.delay( + crate_json, + profile_name, + webhook_url + ) + if webhook_url: + return jsonify({"message": "Validation in progress"}), 202 + else: + return jsonify({"result": result.get()}), 200 + + except Exception as e: + return jsonify({"error": str(e)}), 500 + + def get_ro_crate_validation_task( - crate_id + minio_config: dict, + crate_id: str, + root_path: str, ) -> tuple[Response, int]: """ Retrieves an RO-Crate validation result. + :param minio_config: Access settings for Minio instance containing the RO-Crate. :param crate_id: The ID of the RO-Crate to validate. + :param root_path: The root path containing the RO-Crate. :return: A tuple containing a JSON response and an HTTP status code. - :raises Exception: If an error occurs whilst retrieving RO-Crate + :raises Exception: If an error occurs whilst retreiving validation result """ logging.info(f"Retrieving validation for: {crate_id}") - if not crate_id: - return jsonify({"error": "Missing required parameter: crate_id"}), 400 + minio_client = get_minio_client(minio_config) - try: - return return_ro_crate_validation(crate_id), 200 - except Exception as e: - return jsonify({"service error": str(e)}), 500 + if check_ro_crate_exists(minio_client, minio_config["bucket"], crate_id, root_path): + logging.info("RO-Crate exists") + else: + logging.info("RO-Crate does not exist") + raise InvalidAPIUsage(f"No RO-Crate with prefix: {crate_id}", 400) + + if check_validation_exists(minio_client, minio_config["bucket"], crate_id, root_path): + logging.info("Validation result exists") + else: + logging.info("Validation does not exist") + raise InvalidAPIUsage(f"No validation result yet for RO-Crate: {crate_id}", 400) + + return return_ro_crate_validation(minio_client, minio_config["bucket"], crate_id, root_path), 200 diff --git a/app/tasks/validation_tasks.py b/app/tasks/validation_tasks.py index c58799e..0a62b55 100644 --- a/app/tasks/validation_tasks.py +++ b/app/tasks/validation_tasks.py @@ -6,6 +6,8 @@ import logging import os +import shutil +from typing import Optional from rocrate_validator import services from rocrate_validator.models import ValidationResult @@ -14,33 +16,42 @@ from app.utils.minio_utils import ( fetch_ro_crate_from_minio, update_validation_status_in_minio, - get_validation_status_from_minio + get_validation_status_from_minio, + get_minio_client, + find_rocrate_object_on_minio, + find_validation_object_on_minio ) from app.utils.webhook_utils import send_webhook_notification +from app.utils.file_utils import build_metadata_only_rocrate logger = logging.getLogger(__name__) @celery.task def process_validation_task_by_id( - crate_id: str, profile_name: str | None, webhook_url: str | None + minio_config: dict, crate_id: str, root_path: str, profile_name: str | None, webhook_url: str | None ) -> None: """ Background task to process the RO-Crate validation by ID. + :param minio_config: The MinIO configuration. :param crate_id: The ID of the RO-Crate to validate. + :param root_path: The root path containing the RO-Crate. :param profile_name: The name of the validation profile to use. Defaults to None. :param webhook_url: The webhook URL to send notifications to. Defaults to None. :raises Exception: If an error occurs during the validation process. - :todo: Replace the Crate ID with a more comprehensive system, and replace profile name with URI. """ + # TODO: Split try statements: (1) fetch and validate; (2) write to minio; (3) webhook + + minio_client = get_minio_client(minio_config) + file_path = None try: # Fetch the RO-Crate from MinIO using the provided ID: - file_path = fetch_ro_crate_from_minio(crate_id) + file_path = fetch_ro_crate_from_minio(minio_client, minio_config["bucket"], crate_id, root_path) logging.info(f"Processing validation task for {file_path}") @@ -53,33 +64,101 @@ def process_validation_task_by_id( raise Exception(f"Validation failed: {validation_result}") if not validation_result.has_issues(): - logging.info(f"RO Crate {file_path} is valid.") + logging.info(f"RO Crate {crate_id} is valid.") else: - logging.info(f"RO Crate {file_path} is invalid.") + logging.info(f"RO Crate {crate_id} is invalid.") # Update the validation status in MinIO: - update_validation_status_in_minio(crate_id, validation_result.to_json()) + update_validation_status_in_minio(minio_client, minio_config["bucket"], crate_id, root_path, validation_result.to_json()) # TODO: Prepare the data to send to the webhook, and send the webhook notification. if webhook_url: send_webhook_notification(webhook_url, validation_result.to_json()) + except Exception as e: + logging.error(f"Error processing validation task: {e}") + + # TODO: Should we write error messages to the minio instance too? + + # Send failure notification via webhook + if webhook_url: + error_data = {"profile_name": profile_name, "error": str(e)} + send_webhook_notification(webhook_url, error_data) + + finally: + # Clean up the temporary file if it was created: + if file_path and os.path.exists(file_path): + if os.path.isfile(file_path): + os.remove(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + + +@celery.task +def process_validation_task_by_metadata( + crate_json: str, profile_name: str | None, webhook_url: str | None +) -> ValidationResult | str: + """ + Background task to process the RO-Crate validation for a given json metadata string. + + :param crate_json: A string containing the RO-Crate JSON metadata to validate. + :param profile_name: The name of the validation profile to use. Defaults to None. + :param webhook_url: The webhook URL to send notifications to. Defaults to None. + :raises Exception: If an error occurs during the validation process. + + :todo: Replace the Crate ID with a more comprehensive system, and replace profile name with URI. + """ + + skip_checks_list = ['ro-crate-1.1_12.1'] + file_path = None + + try: + # Fetch the RO-Crate from MinIO using the provided ID: + file_path = build_metadata_only_rocrate(crate_json) + + logging.info(f"Processing validation task for {file_path}") + + # Perform validation: + validation_result = perform_ro_crate_validation(file_path, + profile_name, + skip_checks_list + ) + + if isinstance(validation_result, str): + logging.error(f"Validation failed: {validation_result}") + # TODO: Send webhook with failure notification + raise Exception(f"Validation failed: {validation_result}") + + if not validation_result.has_issues(): + logging.info(f"RO Crate {file_path} is valid.") + else: + logging.info(f"RO Crate {file_path} is invalid.") + + if webhook_url: + send_webhook_notification(webhook_url, validation_result.to_json()) + except Exception as e: logging.error(f"Error processing validation task: {e}") # Send failure notification via webhook error_data = {"profile_name": profile_name, "error": str(e)} - send_webhook_notification(webhook_url, error_data) + if webhook_url: + send_webhook_notification(webhook_url, error_data) finally: # Clean up the temporary file if it was created: if file_path and os.path.exists(file_path): - os.remove(file_path) + shutil.rmtree(file_path) + + if isinstance(validation_result, str): + return validation_result + else: + return validation_result.to_json() def perform_ro_crate_validation( - file_path: str, profile_name: str | None + file_path: str, profile_name: str | None, skip_checks_list: Optional[list] = None ) -> ValidationResult | str: """ Validates an RO-Crate using the provided file path and profile name. @@ -87,6 +166,7 @@ def perform_ro_crate_validation( :param file_path: The path to the RO-Crate file to validate :param profile_name: The name of the validation profile to use. Defaults to None. If None, the CRS4 validator will attempt to determine the profile. + :param skip_checks_list: A list of checks to skip, if needed :return: The validation result. :raises Exception: If an error occurs during the validation process. """ @@ -102,8 +182,8 @@ def perform_ro_crate_validation( ) settings = services.ValidationSettings( rocrate_uri=full_file_path, - # Only include profile_identifier if the profile_name is provided: **({"profile_identifier": profile_name} if profile_name else {}), + **({"skip_checks": skip_checks_list} if skip_checks_list else {}) ) return services.validate(settings) @@ -113,22 +193,68 @@ def perform_ro_crate_validation( return str(e) +def check_ro_crate_exists( + minio_client: object, + bucket_name: str, + crate_id: str, + root_path: str, +) -> bool: + """ + Checks for the existence of an RO-Crate using the provided Crate ID. + + :param minio_client: The MinIO client + :param bucket_name: The MinIO bucket containing the RO-Crate. + :param crate_id: The ID of the RO-Crate to validate. + :param root_path: The root path containing the RO-Crate. + :return: Boolean indicating existence + """ + + logging.info(f"Checking for existence of RO-Crate {crate_id}") + + if find_rocrate_object_on_minio(crate_id, minio_client, bucket_name, root_path): + return True + else: + return False + + +def check_validation_exists( + minio_client: object, + bucket_name: str, + crate_id: str, + root_path: str, +) -> bool: + """ + Checks for the existence of a validation result using the provided Crate ID. + + :param minio_client: The MinIO client + :param minio_bucket: The MinIO bucket containing the RO-Crate. + :param crate_id: The ID of the RO-Crate to validate. + :param root_path: The root path containing the RO-Crate. + :return: Boolean indicating existence + """ + + logging.info(f"Checking for existence of RO-Crate {crate_id}") + + if find_validation_object_on_minio(crate_id, minio_client, bucket_name, root_path): + return True + else: + return False + + def return_ro_crate_validation( - crate_id: str, + minio_client: object, + bucket_name: str, + crate_id: str, + root_path: str, ) -> dict | str: """ Retrieves the validation result for an RO-Crate using the provided Crate ID. + :param minio_client: The MinIO client :param crate_id: The ID of the RO-Crate that has been validated :return: The validation result - :raises Exception: If an error occurs in the retrieving the validation result """ - try: - logging.info(f"Fetching validation result for RO-Crate {crate_id}") - - return get_validation_status_from_minio(crate_id) + logging.info(f"Fetching validation result for RO-Crate {crate_id}") - except Exception as e: - logging.error(f"Unexpected error when retrieving validation: {e}") - return str(e) + return get_validation_status_from_minio(minio_client, bucket_name, crate_id, root_path) diff --git a/app/utils/config.py b/app/utils/config.py index 2258931..a57b63f 100644 --- a/app/utils/config.py +++ b/app/utils/config.py @@ -40,6 +40,22 @@ class ProductionConfig(Config): ENV = "production" +class InvalidAPIUsage(Exception): + status_code = 400 + + def __init__(self, message, status_code=None, payload=None): + super().__init__() + self.message = message + if status_code is not None: + self.status_code = status_code + self.payload = payload + + def to_dict(self): + rv = dict(self.payload or ()) + rv['message'] = self.message + return rv + + def make_celery(app: Flask = None) -> Celery: """ Initialises and configures a Celery instance with the Flask application. diff --git a/app/utils/file_utils.py b/app/utils/file_utils.py new file mode 100644 index 0000000..15c16e4 --- /dev/null +++ b/app/utils/file_utils.py @@ -0,0 +1,53 @@ +"""Utility methods for interacting with the File System.""" + +# Author: Douglas Lowe, Alexander Hambley +# License: MIT +# Copyright (c) 2025 eScience Lab, The University of Manchester + +import json +import logging +import os +import tempfile + +from dotenv import load_dotenv + + +logger = logging.getLogger(__name__) + + +def build_metadata_only_rocrate(crate_json: str) -> str: + """ + Creates a temporary directory for an empty RO-Crate, + and saves the JSON string as a metadata file. + + :param crate_json: The metadata string. + :return: The local file path where the RO-Crate is saved. + :raises ValueError: If the required environment variables are not set. + :raises Exception: If an unexpected error occurs during the operation. + """ + + load_dotenv() + + try: + # Prepare temporary file path to store RO Crate for validation: + temp_dir = tempfile.mkdtemp() + file_path = os.path.join(temp_dir, 'ro-crate-metadata.json') + + logging.info( + f"Creating RO-Crate Metadata file. File path: {file_path}" + ) + with open(file_path, 'w') as f: + f.write(crate_json) + logging.info( + f"RO-Crate metadata successfully saved to {file_path}." + ) + + return temp_dir + + except ValueError as value_error: + logging.error(f"Configuration Error: {value_error}") + raise + + except Exception as e: + logging.error(f"Unexpected error creating RO-Crate metadata: {e}") + raise diff --git a/app/utils/minio_utils.py b/app/utils/minio_utils.py index a4ac42a..1612f90 100644 --- a/app/utils/minio_utils.py +++ b/app/utils/minio_utils.py @@ -9,63 +9,64 @@ import os import tempfile -from dotenv import load_dotenv from io import BytesIO from minio import Minio, S3Error +from app.utils.config import InvalidAPIUsage logger = logging.getLogger(__name__) -def fetch_ro_crate_from_minio(crate_id: str) -> str: +def fetch_ro_crate_from_minio(minio_client: object, minio_bucket: str, crate_id: str, root_path: str) -> str: """ Fetches an RO-Crate from MinIO based on the crate ID. Downloads the crate as a file and returns local file path. + :param minio_client: The MinIO client + :param minio_bucket: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate to fetch from MinIO. + :param root_path: The root path containing the RO-Crate. :return: The local file path where the RO-Crate is saved. - :raises S3Error: If an error occurs during the MinIO operation. - :raises ValueError: If the required environment variables are not set. - :raises Exception: If an unexpected error occurs during the operation. """ - load_dotenv() + rocrate_object = find_rocrate_object_on_minio(crate_id, minio_client, minio_bucket, root_path) - try: - minio_client, bucket_name = get_minio_client_and_bucket() + rocrate_minio_path = rocrate_object.object_name + rocrate_name = rocrate_minio_path.split('/')[-1] - object_name = f"{crate_id}.zip" + temp_dir = tempfile.mkdtemp() + local_root_path = os.path.join(temp_dir, rocrate_name) - # Prepare temporary file path to store RO Crate for validation: - temp_dir = tempfile.mkdtemp() - file_path = os.path.join(temp_dir, object_name) + logging.info( + f"Fetching RO-Crate {rocrate_name} from MinIO bucket {minio_bucket}. File path {local_root_path}" + ) - logging.info( - f"Fetching RO-Crate {object_name} from MinIO bucket {bucket_name}. File path: {file_path}" - ) - minio_client.fget_object(bucket_name, object_name, file_path) - logging.info( - f"RO-Crate {object_name} fetched successfully and saved to {file_path}." - ) + if rocrate_object.is_dir: + os.makedirs(os.path.dirname(local_root_path), exist_ok=True) - return file_path + objects_list = get_minio_object_list(rocrate_minio_path, minio_client, minio_bucket, recursive=True) + for obj in objects_list: + relative_path = obj.object_name[len(rocrate_minio_path):].lstrip("/") + local_file_path = os.path.join(local_root_path, relative_path) + os.makedirs(os.path.dirname(local_file_path), exist_ok=True) + download_file_from_minio(minio_client, minio_bucket, obj.object_name, local_file_path) - except S3Error as s3_error: - logging.error(f"MinIO S3 Error: {s3_error}") - raise + else: + file_path = local_root_path + download_file_from_minio(minio_client, minio_bucket, rocrate_minio_path, file_path) - except ValueError as value_error: - logging.error(f"Configuration Error: {value_error}") - raise + logging.info( + f"RO-Crate {rocrate_name} fetched successfully and saved to {local_root_path}." + ) - except Exception as e: - logging.error(f"Unexpected error fetching RO-Crate from MinIO: {e}") - raise + return local_root_path -def update_validation_status_in_minio(crate_id: str, validation_status: str) -> None: +def update_validation_status_in_minio(minio_client: object, minio_bucket: str, crate_id: str, root_path: str, validation_status: str) -> None: """ Uploads the validation status to the MinIO bucket. + :param minio_client: The MinIO client + :param minio_bucket: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate in MinIO :param validation_status: The validation result to upload :raises S3Error: If an error occurs during the MinIO operation @@ -73,65 +74,64 @@ def update_validation_status_in_minio(crate_id: str, validation_status: str) -> :raises Exception: If an unexpected error occurs """ - load_dotenv() - - try: - minio_client, bucket_name = get_minio_client_and_bucket() + # The object in MinIO is _validation/validation_status.txt + if root_path: + object_name = f"{root_path}/{crate_id}_validation/validation_status.txt" + else: + object_name = f"{crate_id}_validation/validation_status.txt" - # The object in MinIO is /validation_status.txt - object_name = f"{crate_id}/validation_status.txt" - - # convert pretty string to dictionary, then back to plain utf-8 encoded string - validation_string = json.dumps(json.loads(validation_status), indent=None).encode("utf-8") + # convert pretty string to dictionary, then back to plain utf-8 encoded string + validation_string = json.dumps(json.loads(validation_status), indent=None).encode("utf-8") + try: minio_client.put_object( - bucket_name, + minio_bucket, object_name, data=BytesIO(validation_string), length=len(validation_string), content_type="application/json", ) - logging.info( - f"Validation status file uploaded to {bucket_name}/{object_name} successfully." - ) - except S3Error as s3_error: logging.error(f"MinIO S3 Error: {s3_error}") - raise + raise InvalidAPIUsage(f"MinIO S3 Error: {s3_error}", 500) except ValueError as value_error: logging.error(f"Configuration Error: {value_error}") - raise + raise InvalidAPIUsage(f"Configuration Error: {value_error}", 500) except Exception as e: logging.error(f"Unexpected error updating validation status in MinIO: {e}") - raise + raise InvalidAPIUsage(f"Unknown Error: {e}", 500) + logging.info( + f"Validation status file uploaded to {minio_bucket}/{object_name} successfully." + ) -def get_validation_status_from_minio(crate_id: str) -> dict | str: + +def get_validation_status_from_minio(minio_client: object, minio_bucket: str, crate_id: str, root_path: str) -> dict: """ Checks for the existence of a validation report for the given RO-Crate in the MinIO bucket. Returns validation message if it exists, or notification that it is missing if not. + :param minio_client: The MinIO client + :param minio_bucket: The MinIO bucket containing the RO-Crate. :param crate_id: The ID of the RO-Crate in MinIO :return validation_status: Either the validation status, or note that this does not exist - :raises S3Error: If an error occurs during the MinIO operation - :raises ValueError: If the required environment variables are not set - :raises Exception: If an unexpected error occurs """ - try: - minio_client, bucket_name = get_minio_client_and_bucket() - - # The object in MinIO is /validation_status.txt - object_name = f"{crate_id}/validation_status.txt" + # The object in MinIO is _validation/validation_status.txt + if root_path: + object_name = f"{root_path}/{crate_id}_validation/validation_status.txt" + else: + object_name = f"{crate_id}_validation/validation_status.txt" - logging.info(f"Getting object {object_name}") + logging.info(f"Getting object {object_name}") + try: response = minio_client.get_object( - bucket_name, + minio_bucket, object_name, ) @@ -141,40 +141,183 @@ def get_validation_status_from_minio(crate_id: str) -> dict | str: except S3Error as s3_error: logging.error(f"MinIO S3 Error: {s3_error}") - raise + raise InvalidAPIUsage(f"MinIO S3 Error: {s3_error}", 500) except ValueError as value_error: logging.error(f"Configuration Error: {value_error}") - raise + raise InvalidAPIUsage(f"Configuration Error: {value_error}", 500) except Exception as e: logging.error(f"Unexpected error retrieving validation status from MinIO: {e}") - raise + raise InvalidAPIUsage(f"Unknown Error: {e}", 500) else: return validation_message -def get_minio_client_and_bucket() -> [Minio, str]: +def download_file_from_minio(minio_client: object, minio_bucket: str, object_path: str, file_path: str) -> None: + """ + Downloads a file from MinIO + + :param minio_client: MinIO object + :param minio_bucket: name of MinIO bucket, string + :param object_path: path to object on MinIO, string + :param file_path: local path, string + :raises S3Error: If an error occurs during the MinIO operation + :raises ValueError: If the required environment variables are not set + :raises Exception: If an unexpected error occurs + """ + + try: + minio_client.fget_object(minio_bucket, object_path, file_path) + + except S3Error as s3_error: + logging.error(f"MinIO S3 Error: {s3_error}") + raise InvalidAPIUsage(f"MinIO S3 Error: {s3_error}", 500) + + except ValueError as value_error: + logging.error(f"Configuration Error: {value_error}") + raise InvalidAPIUsage(f"Configuration Error: {value_error}", 500) + + except Exception as e: + logging.error(f"Unexpected error retrieving file from MinIO: {e}") + raise InvalidAPIUsage(f"Unknown Error: {e}", 500) + + +def find_validation_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, root_path: str) -> object: + """ + Checks that the requested object exists on the MinIO instance. + + If it does not exist then a False value is returned. + If it does exist then the minio.datatypes.Object is returned. + + :param rocrate_id: string containing the name of ro-crate + :param root_path: string containing the path within which the ro-crate should be + :param minio_client: minio object + :param minio_bucket: string containing bucket on minio + :return return_object: rocrate object we require + :raise Exception: If validation result can't be found, 400 + """ + + logging.info(f"Finding Validation result: {rocrate_id}_validation/validation_status.txt") + + if root_path: + file_path = f"{root_path}/{rocrate_id}_validation/validation_status.txt" + else: + file_path = f"{rocrate_id}_validation/validation_status.txt" + + file_list = get_minio_object_list(file_path, minio_client, minio_bucket) + + return_object = False + for obj in file_list: + if obj.object_name == file_path: + return_object = obj + break + + if not return_object: + logging.error(f"No validation result yet for RO-Crate: {rocrate_id}") + return False + else: + return return_object + + +def find_rocrate_object_on_minio(rocrate_id: str, minio_client, minio_bucket: str, root_path: str) -> object | bool: """ - Initialises the MinIO client and retrieves the bucket name from environment variables. + Checks that the requested object exists on the MinIO instance. + + If it does not exist then a False value is returned. + If it does exist then the minio.datatypes.Object is returned. + + :param rocrate_id: string containing the name of ro-crate + :param root_path: string containing the path within which the ro-crate should be + :param minio_client: minio object + :param minio_bucket: string containing bucket on minio + :return return_object or False: rocrate object we require, or False result + :raise Exception: If RO-Crate can't be found, 400 + """ + + logging.info(f"Finding RO-Crate: {rocrate_id}") + + if root_path: + rocrate_path = f"{root_path}/{rocrate_id}" + else: + rocrate_path = rocrate_id + + rocrate_list = get_minio_object_list(rocrate_path, minio_client, minio_bucket) + + return_object = False + for obj in rocrate_list: + # TODO: We should be checking here for the existence of the ro-crate metadata file within this object too + if (obj.object_name == f"{rocrate_path}/" and obj.is_dir) or obj.object_name == f"{rocrate_path}.zip": + return_object = obj + break + + if not return_object: + logging.error(f"No RO-Crate with prefix: {rocrate_path}") + return False + else: + return return_object + - :return: A tuple containing the MinIO client and the bucket name. +def get_minio_object_list(object_path: str, minio_client, minio_bucket: str, recursive: bool = False) -> list: + """ + Creates a list of objects which match the object_id and path_prefix + + :param object_path: The object ID, string + :param path_prefix: Path prefix, string, optional + :param minio_client: MinIO client object + :param minio_bucket: string + :param recursive: boolean, default = False + :return object_list: List containing objects of type minio.datatypes.Object + :raises S3Error: If an error occurs during the MinIO operation, 500 + :raises ValueError: If the required environment variables are not set, 500 + :raises Exception: If an unexpected error occurs, 500 + """ + + try: + response = minio_client.list_objects( + minio_bucket, + object_path, + recursive=recursive + ) + object_list = [obj for obj in response] + + response.close() + + except S3Error as s3_error: + logging.error(f"MinIO S3 Error: {s3_error}") + raise InvalidAPIUsage(f"MinIO S3 Error: {s3_error}", 500) + + except ValueError as value_error: + logging.error(f"Configuration Error: {value_error}") + raise InvalidAPIUsage(f"Configuration Error: {value_error}", 500) + + except Exception as e: + logging.error(f"Unexpected error getting object list from MinIO: {e}") + raise InvalidAPIUsage(f"Unknown Error: {e}", 500) + + else: + return object_list + + +def get_minio_client(minio_config: dict) -> Minio: + """ + Initialises the MinIO client from provided settings. + + :param minio_config: A dictionary containing the below parameters + :param endpoint: A string containing host and port. E.g. 'localhost:9000' + :param access_key: A string containing the access key / username + :param secret_key: A string containing the secret key / password + :param use_ssl: Boolean defining if SSL connection should be used or not + :return: The MinIO client. :raises ValueError: If required environment variables are not set. """ - load_dotenv() minio_client = Minio( - endpoint=os.environ.get("MINIO_ENDPOINT"), - access_key=os.environ.get("MINIO_ROOT_USER"), - secret_key=os.environ.get("MINIO_ROOT_PASSWORD"), - secure=False, + endpoint=minio_config["endpoint"], + access_key=minio_config["accesskey"], + secret_key=minio_config["secret"], + secure=minio_config["ssl"], ) - bucket_name = os.environ.get("MINIO_BUCKET_NAME") - if not bucket_name: - raise ValueError( - "RO Crate MINIO_BUCKET_NAME is not set in the environment variables." - ) - - return minio_client, bucket_name + return minio_client diff --git a/docker-compose.yml b/docker-compose.yml index d4e88c8..8605c0c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,8 @@ version: '3.8' services: flask: - image: "ghcr.io/esciencelab/cratey-validator:latest" + platform: linux/x86_64 + image: "ghcr.io/esciencelab/cratey-validator:0.1" ports: - "5001:5000" environment: @@ -19,7 +20,8 @@ services: - minio celery_worker: - image: "ghcr.io/esciencelab/cratey-validator:latest" + platform: linux/x86_64 + image: "ghcr.io/esciencelab/cratey-validator:0.1" command: celery -A app.celery_worker.celery worker --loglevel=info -E environment: - CELERY_BROKER_URL=redis://redis:6379/0 diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..96735eb --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +log_format = %(asctime)s %(levelname)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..26116dc --- /dev/null +++ b/requirements.in @@ -0,0 +1,9 @@ +celery==5.5.3 +minio==7.2.16 +requests==2.32.4 +Flask==3.0.3 +Werkzeug==3.0.6 +redis==5.2.0 +python-dotenv==1.1.1 +apiflask==2.4.0 +roc-validator==0.7.3 diff --git a/requirements.txt b/requirements.txt index f828262..760e034 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,188 @@ -celery~=5.4.0 -minio~=7.2.10 -requests~=2.32.3 -Flask~=3.0.3 -Werkzeug~=3.0.4 -redis~=5.2.0 -python-dotenv~=1.0.1 -apiflask~=2.4.0 -roc-validator~=0.6.5 \ No newline at end of file +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --output-file=requirements.txt requirements.in +# +amqp==5.3.1 + # via kombu +apiflask==2.4.0 + # via -r requirements.in +apispec==6.8.2 + # via apiflask +argon2-cffi==25.1.0 + # via minio +argon2-cffi-bindings==25.1.0 + # via argon2-cffi +attrs==25.3.0 + # via + # cattrs + # requests-cache +billiard==4.2.1 + # via celery +blinker==1.9.0 + # via flask +cattrs==25.1.1 + # via requests-cache +celery==5.5.3 + # via -r requirements.in +certifi==2025.8.3 + # via + # minio + # requests +cffi==1.17.1 + # via argon2-cffi-bindings +charset-normalizer==3.4.2 + # via requests +click==8.2.1 + # via + # celery + # click-didyoumean + # click-plugins + # click-repl + # flask + # rich-click + # roc-validator +click-didyoumean==0.3.1 + # via celery +click-plugins==1.1.1.2 + # via celery +click-repl==0.3.0 + # via celery +colorlog==6.9.0 + # via roc-validator +enum-tools==0.12.0 + # via roc-validator +flask==3.0.3 + # via + # -r requirements.in + # apiflask + # flask-httpauth + # flask-marshmallow +flask-httpauth==4.8.0 + # via apiflask +flask-marshmallow==1.3.0 + # via apiflask +html5rdf==1.2.1 + # via rdflib +idna==3.10 + # via + # requests + # url-normalize +importlib-metadata==8.7.0 + # via pyshacl +inquirerpy==0.3.4 + # via roc-validator +itsdangerous==2.2.0 + # via flask +jinja2==3.1.6 + # via flask +kombu==5.5.4 + # via celery +markdown-it-py==3.0.0 + # via rich +markupsafe==3.0.2 + # via + # jinja2 + # werkzeug +marshmallow==4.0.0 + # via + # apiflask + # flask-marshmallow + # webargs +mdurl==0.1.2 + # via markdown-it-py +minio==7.2.16 + # via -r requirements.in +owlrl==7.1.4 + # via pyshacl +packaging==25.0 + # via + # apispec + # kombu + # pyshacl + # webargs +pfzy==0.3.4 + # via inquirerpy +platformdirs==4.3.8 + # via requests-cache +prettytable==3.16.0 + # via pyshacl +prompt-toolkit==3.0.51 + # via + # click-repl + # inquirerpy +pycparser==2.22 + # via cffi +pycryptodome==3.23.0 + # via minio +pygments==2.19.2 + # via + # enum-tools + # rich +pyparsing==3.2.3 + # via rdflib +pyshacl==0.30.1 + # via roc-validator +python-dateutil==2.9.0.post0 + # via celery +python-dotenv==1.1.1 + # via -r requirements.in +rdflib[html]==7.1.4 + # via + # owlrl + # pyshacl + # roc-validator +redis==5.2.0 + # via -r requirements.in +requests==2.32.4 + # via + # -r requirements.in + # requests-cache + # roc-validator +requests-cache==1.2.1 + # via roc-validator +rich==13.9.4 + # via + # rich-click + # roc-validator +rich-click==1.8.9 + # via roc-validator +roc-validator==0.7.3 + # via -r requirements.in +six==1.17.0 + # via python-dateutil +toml==0.10.2 + # via roc-validator +typing-extensions==4.14.1 + # via + # cattrs + # enum-tools + # minio + # rich-click +tzdata==2025.2 + # via kombu +url-normalize==2.2.1 + # via requests-cache +urllib3==2.5.0 + # via + # minio + # requests + # requests-cache +vine==5.1.0 + # via + # amqp + # celery + # kombu +wcwidth==0.2.13 + # via + # prettytable + # prompt-toolkit +webargs==8.7.0 + # via apiflask +werkzeug==3.0.6 + # via + # -r requirements.in + # flask +zipp==3.23.0 + # via importlib-metadata diff --git a/tests/data/ro-crate-metadata.json b/tests/data/ro-crate-metadata.json new file mode 100644 index 0000000..957ddea --- /dev/null +++ b/tests/data/ro-crate-metadata.json @@ -0,0 +1,101 @@ +{ + "@context": "https://w3id.org/ro/crate/1.1/context", + "@graph": [ + { + "@id": "./", + "@type": "Dataset", + "name": "example dataset", + "description": "crate that should conform with validator requirements", + "datePublished": "2024-03-17T15:55:54+00:00", + "license": { + "@id": "http://spdx.org/licenses/MIT" + }, + "hasPart": [ + { + "@id": "paper.pdf" + }, + { + "@id": "results.csv" + }, + { + "@id": "images/figure.svg" + }, + { + "@id": "logs/" + } + ] + }, + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": { + "@id": "./" + }, + "conformsTo": { + "@id": "https://w3id.org/ro/crate/1.1" + } + }, + { + "@id": "paper.pdf", + "@type": "File", + "author": [ + { + "@id": "https://orcid.org/0000-0000-0000-0000" + }, + { + "@id": "https://orcid.org/0000-0000-0000-0001" + } + ], + "encodingFormat": "application/pdf", + "name": "manuscript" + }, + { + "@id": "results.csv", + "@type": "File", + "author": { + "@id": "https://orcid.org/0000-0000-0000-0000" + }, + "encodingFormat": "text/csv", + "name": "experimental data" + }, + { + "@id": "images/figure.svg", + "@type": "File", + "author": { + "@id": "https://orcid.org/0000-0000-0000-0001" + }, + "encodingFormat": "image/svg+xml", + "name": "bar chart" + }, + { + "@id": "logs/", + "@type": "Dataset" + }, + { + "@id": "https://orcid.org/0000-0000-0000-0000", + "@type": "Person", + "affiliation": "University of Flatland", + "name": "Alice Doe" + }, + { + "@id": "https://orcid.org/0000-0000-0000-0001", + "@type": "Person", + "affiliation": "University of Flatland", + "name": "Bob Doe" + }, + { + "@id": "#hdr2024", + "@type": "Project Meeting", + "endDate": "2024-04-16", + "location": "Swansea, UK", + "name": "HDR-UK 2024", + "startDate": "2024-04-15" + }, + { + "@id": "https://spdx.org/licenses/MIT", + "@type": "CreativeWork", + "name": "MIT Licence", + "identifier": "MIT" + } + ] +} \ No newline at end of file diff --git a/tests/data/ro_crates/project_a/ro_crate_4.zip b/tests/data/ro_crates/project_a/ro_crate_4.zip new file mode 100644 index 0000000..c574f20 Binary files /dev/null and b/tests/data/ro_crates/project_a/ro_crate_4.zip differ diff --git a/tests/data/ro_crates/project_a/ro_crate_5/ro-crate-metadata.json b/tests/data/ro_crates/project_a/ro_crate_5/ro-crate-metadata.json new file mode 100644 index 0000000..7f490d6 --- /dev/null +++ b/tests/data/ro_crates/project_a/ro_crate_5/ro-crate-metadata.json @@ -0,0 +1,112 @@ +{ + "@context": "https://w3id.org/ro/crate/1.1/context", + "@graph": [ + { + "@id": "./", + "@type": "Dataset", + "datePublished": "2024-04-17T13:39:44+00:00", + "hasPart": [ + { + "@id": "sort-and-change-case.ga" + }, + { + "@id": "sort-and-change-case.cwl" + }, + { + "@id": "blank.png" + }, + { + "@id": "README.md" + } + ], + "mainEntity": { + "@id": "sort-and-change-case.ga" + } + }, + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": { + "@id": "./" + }, + "conformsTo": [ + { + "@id": "https://w3id.org/ro/crate/1.1" + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.0" + } + ] + }, + { + "@id": "sort-and-change-case.ga", + "@type": [ + "File", + "SoftwareSourceCode", + "ComputationalWorkflow" + ], + "description": "sort lines and change text to upper case", + "image": { + "@id": "blank.png" + }, + "license": "https://spdx.org/licenses/MIT.html", + "name": "sort-and-change-case", + "programmingLanguage": { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#galaxy" + }, + "subjectOf": { + "@id": "sort-and-change-case.cwl" + } + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#galaxy", + "@type": "ComputerLanguage", + "identifier": { + "@id": "https://galaxyproject.org/" + }, + "name": "Galaxy", + "url": { + "@id": "https://galaxyproject.org/" + } + }, + { + "@id": "sort-and-change-case.cwl", + "@type": [ + "File", + "SoftwareSourceCode", + "HowTo" + ], + "name": "sort-and-change-case", + "programmingLanguage": { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#cwl" + } + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#cwl", + "@type": "ComputerLanguage", + "alternateName": "CWL", + "identifier": { + "@id": "https://w3id.org/cwl/" + }, + "name": "Common Workflow Language", + "url": { + "@id": "https://www.commonwl.org/" + } + }, + { + "@id": "blank.png", + "@type": [ + "File", + "ImageObject" + ] + }, + { + "@id": "README.md", + "@type": "File", + "about": { + "@id": "./" + }, + "encodingFormat": "text/markdown" + } + ] +} \ No newline at end of file diff --git a/tests/data/ro_crates/project_a/ro_crate_5/sort-and-change-case.ga b/tests/data/ro_crates/project_a/ro_crate_5/sort-and-change-case.ga new file mode 100644 index 0000000..5a19996 --- /dev/null +++ b/tests/data/ro_crates/project_a/ro_crate_5/sort-and-change-case.ga @@ -0,0 +1,118 @@ +{ + "uuid": "e2a8566c-c025-4181-9e90-7ed29d4e4df1", + "tags": [], + "format-version": "0.1", + "name": "sort-and-change-case", + "version": 0, + "steps": { + "0": { + "tool_id": null, + "tool_version": null, + "outputs": [], + "workflow_outputs": [], + "input_connections": {}, + "tool_state": "{}", + "id": 0, + "uuid": "5a36fad2-66c7-4b9e-8759-0fbcae9b8541", + "errors": null, + "name": "Input dataset", + "label": "bed_input", + "inputs": [], + "position": { + "top": 200, + "left": 200 + }, + "annotation": "", + "content_id": null, + "type": "data_input" + }, + "1": { + "tool_id": "sort1", + "tool_version": "1.1.0", + "outputs": [ + { + "type": "input", + "name": "out_file1" + } + ], + "workflow_outputs": [ + { + "output_name": "out_file1", + "uuid": "8237f71a-bc2a-494e-a63c-09c1e65ef7c8", + "label": "sorted_bed" + } + ], + "input_connections": { + "input": { + "output_name": "output", + "id": 0 + } + }, + "tool_state": "{\"__page__\": null, \"style\": \"\\\"alpha\\\"\", \"column\": \"\\\"1\\\"\", \"__rerun_remap_job_id__\": null, \"column_set\": \"[]\", \"input\": \"{\\\"__class__\\\": \\\"RuntimeValue\\\"}\", \"header_lines\": \"\\\"0\\\"\", \"order\": \"\\\"ASC\\\"\"}", + "id": 1, + "uuid": "0b6b3cda-c75f-452b-85b1-8ae4f3302ba4", + "errors": null, + "name": "Sort", + "post_job_actions": {}, + "label": "sort", + "inputs": [ + { + "name": "input", + "description": "runtime parameter for tool Sort" + } + ], + "position": { + "top": 200, + "left": 420 + }, + "annotation": "", + "content_id": "sort1", + "type": "tool" + }, + "2": { + "tool_id": "ChangeCase", + "tool_version": "1.0.0", + "outputs": [ + { + "type": "tabular", + "name": "out_file1" + } + ], + "workflow_outputs": [ + { + "output_name": "out_file1", + "uuid": "c31cd733-dab6-4d50-9fec-b644d162397b", + "label": "uppercase_bed" + } + ], + "input_connections": { + "input": { + "output_name": "out_file1", + "id": 1 + } + }, + "tool_state": "{\"__page__\": null, \"casing\": \"\\\"up\\\"\", \"__rerun_remap_job_id__\": null, \"cols\": \"\\\"c1\\\"\", \"delimiter\": \"\\\"TAB\\\"\", \"input\": \"{\\\"__class__\\\": \\\"RuntimeValue\\\"}\"}", + "id": 2, + "uuid": "9698bcde-0729-48fe-b88d-ccfb6f6153b4", + "errors": null, + "name": "Change Case", + "post_job_actions": {}, + "label": "change_case", + "inputs": [ + { + "name": "input", + "description": "runtime parameter for tool Change Case" + } + ], + "position": { + "top": 200, + "left": 640 + }, + "annotation": "", + "content_id": "ChangeCase", + "type": "tool" + } + }, + "annotation": "", + "a_galaxy_workflow": "true" +} diff --git a/tests/data/ro_crates/ro_crate_1.zip b/tests/data/ro_crates/ro_crate_1.zip new file mode 100644 index 0000000..c574f20 Binary files /dev/null and b/tests/data/ro_crates/ro_crate_1.zip differ diff --git a/tests/data/ro_crates/ro_crate_2/ro-crate-metadata.json b/tests/data/ro_crates/ro_crate_2/ro-crate-metadata.json new file mode 100644 index 0000000..7f490d6 --- /dev/null +++ b/tests/data/ro_crates/ro_crate_2/ro-crate-metadata.json @@ -0,0 +1,112 @@ +{ + "@context": "https://w3id.org/ro/crate/1.1/context", + "@graph": [ + { + "@id": "./", + "@type": "Dataset", + "datePublished": "2024-04-17T13:39:44+00:00", + "hasPart": [ + { + "@id": "sort-and-change-case.ga" + }, + { + "@id": "sort-and-change-case.cwl" + }, + { + "@id": "blank.png" + }, + { + "@id": "README.md" + } + ], + "mainEntity": { + "@id": "sort-and-change-case.ga" + } + }, + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": { + "@id": "./" + }, + "conformsTo": [ + { + "@id": "https://w3id.org/ro/crate/1.1" + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.0" + } + ] + }, + { + "@id": "sort-and-change-case.ga", + "@type": [ + "File", + "SoftwareSourceCode", + "ComputationalWorkflow" + ], + "description": "sort lines and change text to upper case", + "image": { + "@id": "blank.png" + }, + "license": "https://spdx.org/licenses/MIT.html", + "name": "sort-and-change-case", + "programmingLanguage": { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#galaxy" + }, + "subjectOf": { + "@id": "sort-and-change-case.cwl" + } + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#galaxy", + "@type": "ComputerLanguage", + "identifier": { + "@id": "https://galaxyproject.org/" + }, + "name": "Galaxy", + "url": { + "@id": "https://galaxyproject.org/" + } + }, + { + "@id": "sort-and-change-case.cwl", + "@type": [ + "File", + "SoftwareSourceCode", + "HowTo" + ], + "name": "sort-and-change-case", + "programmingLanguage": { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#cwl" + } + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate#cwl", + "@type": "ComputerLanguage", + "alternateName": "CWL", + "identifier": { + "@id": "https://w3id.org/cwl/" + }, + "name": "Common Workflow Language", + "url": { + "@id": "https://www.commonwl.org/" + } + }, + { + "@id": "blank.png", + "@type": [ + "File", + "ImageObject" + ] + }, + { + "@id": "README.md", + "@type": "File", + "about": { + "@id": "./" + }, + "encodingFormat": "text/markdown" + } + ] +} \ No newline at end of file diff --git a/tests/data/ro_crates/ro_crate_2/sort-and-change-case.ga b/tests/data/ro_crates/ro_crate_2/sort-and-change-case.ga new file mode 100644 index 0000000..5a19996 --- /dev/null +++ b/tests/data/ro_crates/ro_crate_2/sort-and-change-case.ga @@ -0,0 +1,118 @@ +{ + "uuid": "e2a8566c-c025-4181-9e90-7ed29d4e4df1", + "tags": [], + "format-version": "0.1", + "name": "sort-and-change-case", + "version": 0, + "steps": { + "0": { + "tool_id": null, + "tool_version": null, + "outputs": [], + "workflow_outputs": [], + "input_connections": {}, + "tool_state": "{}", + "id": 0, + "uuid": "5a36fad2-66c7-4b9e-8759-0fbcae9b8541", + "errors": null, + "name": "Input dataset", + "label": "bed_input", + "inputs": [], + "position": { + "top": 200, + "left": 200 + }, + "annotation": "", + "content_id": null, + "type": "data_input" + }, + "1": { + "tool_id": "sort1", + "tool_version": "1.1.0", + "outputs": [ + { + "type": "input", + "name": "out_file1" + } + ], + "workflow_outputs": [ + { + "output_name": "out_file1", + "uuid": "8237f71a-bc2a-494e-a63c-09c1e65ef7c8", + "label": "sorted_bed" + } + ], + "input_connections": { + "input": { + "output_name": "output", + "id": 0 + } + }, + "tool_state": "{\"__page__\": null, \"style\": \"\\\"alpha\\\"\", \"column\": \"\\\"1\\\"\", \"__rerun_remap_job_id__\": null, \"column_set\": \"[]\", \"input\": \"{\\\"__class__\\\": \\\"RuntimeValue\\\"}\", \"header_lines\": \"\\\"0\\\"\", \"order\": \"\\\"ASC\\\"\"}", + "id": 1, + "uuid": "0b6b3cda-c75f-452b-85b1-8ae4f3302ba4", + "errors": null, + "name": "Sort", + "post_job_actions": {}, + "label": "sort", + "inputs": [ + { + "name": "input", + "description": "runtime parameter for tool Sort" + } + ], + "position": { + "top": 200, + "left": 420 + }, + "annotation": "", + "content_id": "sort1", + "type": "tool" + }, + "2": { + "tool_id": "ChangeCase", + "tool_version": "1.0.0", + "outputs": [ + { + "type": "tabular", + "name": "out_file1" + } + ], + "workflow_outputs": [ + { + "output_name": "out_file1", + "uuid": "c31cd733-dab6-4d50-9fec-b644d162397b", + "label": "uppercase_bed" + } + ], + "input_connections": { + "input": { + "output_name": "out_file1", + "id": 1 + } + }, + "tool_state": "{\"__page__\": null, \"casing\": \"\\\"up\\\"\", \"__rerun_remap_job_id__\": null, \"cols\": \"\\\"c1\\\"\", \"delimiter\": \"\\\"TAB\\\"\", \"input\": \"{\\\"__class__\\\": \\\"RuntimeValue\\\"}\"}", + "id": 2, + "uuid": "9698bcde-0729-48fe-b88d-ccfb6f6153b4", + "errors": null, + "name": "Change Case", + "post_job_actions": {}, + "label": "change_case", + "inputs": [ + { + "name": "input", + "description": "runtime parameter for tool Change Case" + } + ], + "position": { + "top": 200, + "left": 640 + }, + "annotation": "", + "content_id": "ChangeCase", + "type": "tool" + } + }, + "annotation": "", + "a_galaxy_workflow": "true" +} diff --git a/tests/data/ro_crates/ro_crate_3.zip b/tests/data/ro_crates/ro_crate_3.zip new file mode 100644 index 0000000..c574f20 Binary files /dev/null and b/tests/data/ro_crates/ro_crate_3.zip differ diff --git a/tests/data/ro_crates/ro_crate_3_validation/validation_status.txt b/tests/data/ro_crates/ro_crate_3_validation/validation_status.txt new file mode 100644 index 0000000..40b627e --- /dev/null +++ b/tests/data/ro_crates/ro_crate_3_validation/validation_status.txt @@ -0,0 +1 @@ +{"meta": {"version": "0.2"}, "validation_settings": {"profile_identifier": "ro-crate-1.1", "enable_profile_inheritance": true, "abort_on_first": false, "requirement_severity": "REQUIRED", "rocrate_validator_version": "0.6.5"}, "passed": false, "issues": [{"severity": "REQUIRED", "message": "RO-Crate file descriptor \"ro-crate-metadata.json\" is not in the correct format", "violatingEntity": null, "violatingProperty": null, "violatingPropertyValue": null, "check": {"identifier": "ro-crate-1.1_1.1", "label": "REQUIRED 1.1", "order": 1, "name": "File Descriptor JSON format", "description": "Check if the file descriptor is in the correct format", "severity": "REQUIRED", "requirement": {"identifier": "ro-crate-1.1_1", "name": "File Descriptor JSON format", "description": "The file descriptor MUST be a valid JSON file", "order": 1, "profile": {"identifier": "ro-crate-1.1", "uri": "https://w3id.org/ro/crate/1.1", "name": "RO-Crate Metadata Specification 1.1", "description": "RO-Crate Metadata Specification."}}}}, {"severity": "REQUIRED", "message": "Unexpected error: File not found: ro-crate-metadata.json", "violatingEntity": null, "violatingProperty": null, "violatingPropertyValue": null, "check": {"identifier": "ro-crate-1.1_2.1", "label": "REQUIRED 2.1", "order": 1, "name": "Validation of the compaction format of the file descriptor", "description": "Check if the file descriptor is in the **compacted** JSON-LD format", "severity": "REQUIRED", "requirement": {"identifier": "ro-crate-1.1_2", "name": "File Descriptor JSON-LD format", "description": "The file descriptor MUST be a valid JSON-LD file", "order": 2, "profile": {"identifier": "ro-crate-1.1", "uri": "https://w3id.org/ro/crate/1.1", "name": "RO-Crate Metadata Specification 1.1", "description": "RO-Crate Metadata Specification."}}}}, {"severity": "REQUIRED", "message": "RO-Crate \"ro-crate-metadata.json\" file descriptor is empty", "violatingEntity": null, "violatingProperty": null, "violatingPropertyValue": null, "check": {"identifier": "ro-crate-1.1_3.2", "label": "REQUIRED 3.2", "order": 2, "name": "File Descriptor size check", "description": "Check if the file descriptor is not empty", "severity": "REQUIRED", "requirement": {"identifier": "ro-crate-1.1_3", "name": "File Descriptor existence", "description": "The file descriptor MUST be present in the RO-Crate and MUST not be empty.", "order": 3, "profile": {"identifier": "ro-crate-1.1", "uri": "https://w3id.org/ro/crate/1.1", "name": "RO-Crate Metadata Specification 1.1", "description": "RO-Crate Metadata Specification."}}}}]} \ No newline at end of file diff --git a/tests/data/ro_crates/ro_crate_not_validated.zip b/tests/data/ro_crates/ro_crate_not_validated.zip new file mode 100644 index 0000000..c574f20 Binary files /dev/null and b/tests/data/ro_crates/ro_crate_not_validated.zip differ diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py new file mode 100644 index 0000000..f527501 --- /dev/null +++ b/tests/test_api_routes.py @@ -0,0 +1,275 @@ +from flask.testing import FlaskClient +import pytest +from unittest.mock import patch +from app import create_app + + +@pytest.fixture +def client(): + app = create_app() + return app.test_client() + + +# Test POST API: /v1/ro_crates/{crate_id}/validation + +@pytest.mark.parametrize( + "crate_id, payload, status_code, response_json", + [ + ( + "crate-123", { + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + }, 202, {"message": "Validation in progress"} + ), + ( + "crate-123", { + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + }, 202, {"message": "Validation in progress"} + ), + ( + "crate-123", { + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "root_path": "base_path", + "profile_name": "default" + }, 202, {"message": "Validation in progress"} + ), + ( + "crate-123", { + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + }, 202, {"message": "Validation in progress"} + ), + ( + "crate-123", { + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + }, 202, {"message": "Validation in progress"} + ), + ], + ids=["validate_by_id", "validate_with_missing_profile_name", + "validate_with_missing_webhook_url", "validate_with_missing_root_path", + "validate_with_missing_root_path_and_profile_name_and_webhook_url"] +) +def test_validate_by_id_success(client: FlaskClient, crate_id: str, payload: dict, status_code: int, response_json: dict): + with patch("app.ro_crates.routes.post_routes.queue_ro_crate_validation_task") as mock_queue: + mock_queue.return_value = (response_json, status_code) + + response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) + + minio_config = payload["minio_config"] if "minio_config" in payload else None + root_path = payload["root_path"] if "root_path" in payload else None + profile_name = payload["profile_name"] if "profile_name" in payload else None + webhook_url = payload["webhook_url"] if "webhook_url" in payload else None + assert response.status_code == status_code + assert response.json == response_json + mock_queue.assert_called_once_with(minio_config, crate_id, root_path, profile_name, webhook_url) + + +@pytest.mark.parametrize( + "crate_id, payload, status_code", + [ + ( + "", { + "minio_bucket": "test_bucket", + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + }, 404 + ), + ( + "crate-123", { + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + }, 422 + ), + ], + ids=[ + "missing_crate_id_returns_404", + "missing_minio_bucket_returns_422" + ] +) +def test_validate_fails_missing_elements(client: FlaskClient, crate_id: str, payload: dict, status_code: int): + response = client.post(f"/v1/ro_crates/{crate_id}/validation", json=payload) + assert response.status_code == status_code + + +# Test POST API: /v1/ro_crates/validate_metadata + +@pytest.mark.parametrize( + "payload, status_code, response_json", + [ + ( + { + "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}', + "profile_name": "default" + }, 200, {"status": "success"} + ), + ( + { + "crate_json": '{"@context": "https://w3id.org/ro/crate/1.1/context"}', + }, 200, {"status": "success"} + ), + ], + ids=["success_with_all_fields", "success_without_profile_name"] +) +def test_validate_metadata_success(client: FlaskClient, payload: dict, status_code: int, response_json: dict): + with patch("app.ro_crates.routes.post_routes.queue_ro_crate_metadata_validation_task") as mock_queue: + mock_queue.return_value = (response_json, status_code) + + response = client.post("/v1/ro_crates/validate_metadata", json=payload) + + crate_json = payload["crate_json"] if "crate_json" in payload else None + profile_name = payload["profile_name"] if "profile_name" in payload else None + + mock_queue.assert_called_once_with(crate_json, profile_name) + assert response.status_code == status_code + assert response.json == response_json + + +@pytest.mark.parametrize( + "payload, status_code, response_text", + [ + ( + { + "profile_name": "default" + }, 422, "Missing data for required field" + ), + ( + { + "crate_json": '', + }, 422, "Missing required parameter" + ), + ( + { + "crate_json": '{', + }, 422, "not valid JSON" + ), + ( + { + "crate_json": '{}', + }, 422, "Required parameter crate_json is empty" + ), + ], + ids=["failure_missing_crate", "failure_empty_crate", + "failure_malformed_crate", "failure_empty_crate"] +) +def test_validate_metadata_failure(client: FlaskClient, payload: dict, status_code: int, response_text: str): + response = client.post("/v1/ro_crates/validate_metadata", json=payload) + assert response.status_code == status_code + assert response_text in response.get_data(as_text=True) + + +# Test GET API: /v1/ro_crates/{crate_id}/validation + +@pytest.mark.parametrize( + "crate_id, payload, status_code", + [ + ( + "", { + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "root_path": "base_path" + }, 404 + ), + ( + "crate-123", { + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + }, + "root_path": "base_path" + }, 422 + ), + ], + ids=["failure_missing_crate_id", "failure_missing_minio_bucket"] +) +def test_get_validation_by_id_failures(client: FlaskClient, crate_id: str, payload: dict, status_code: int): + response = client.get(f"/v1/ro_crates/{crate_id}/validation", json=payload) + assert response.status_code == status_code + + +def test_get_validation_by_id_success(client): + crate_id = "crate-123" + payload = { + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "root_path": "base_path" + } + + with patch("app.ro_crates.routes.get_routes.get_ro_crate_validation_task") as mock_get: + mock_get.return_value = ({"status": "valid"}, 200) + + response = client.get(f"/v1/ro_crates/{crate_id}/validation", json=payload) + + assert response.status_code == 200 + assert response.json == {"status": "valid"} + mock_get.assert_called_once_with(payload["minio_config"], "crate-123", "base_path") + + +def test_get_validation_by_id_missing_root_path(client): + crate_id = "crate-123" + payload = { + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + } + } + + with patch("app.ro_crates.routes.get_routes.get_ro_crate_validation_task") as mock_get: + mock_get.return_value = ({"status": "valid"}, 200) + + response = client.get(f"/v1/ro_crates/{crate_id}/validation", json=payload) + + assert response.status_code == 200 + assert response.json == {"status": "valid"} + mock_get.assert_called_once_with(payload["minio_config"], "crate-123", None) diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..2b4b9a7 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,510 @@ +import pytest +import subprocess +import time +import requests +import json +import os +import docker +from minio import Minio + + +@pytest.fixture(scope="session") +def docker_client(): + return docker.from_env() + + +@pytest.fixture(scope="session", autouse=True) +def docker_compose(docker_client): + """Start Docker Compose before tests, shut down after.""" + print("Starting Docker Compose...") + subprocess.run( + ["docker", "compose", "-f", "docker-compose-develop.yml", "up", "-d"], + check=True + ) + time.sleep(10) # Wait for services to start — adjust as needed + + load_test_data_into_minio() + + yield # Run the tests + + for container in docker_client.containers.list(): + if "cratey-validator" in container.name: + logs = container.logs().decode("utf-8") + + print(f"\n======= Logs from {container.name} container =======") + print(logs) + + print("Stopping Docker Compose...") + subprocess.run(["docker", "compose", "down"], check=True) + + +def load_test_data_into_minio(): + """Connect to MinIO and upload test files.""" + minio_client = Minio( + endpoint="localhost:9000", + access_key="minioadmin", + secret_key="minioadmin", + secure=False + ) + + bucket_name = "ro-crates" + test_data_dir = "tests/data/ro_crates" + + # Ensure bucket exists + if not minio_client.bucket_exists(bucket_name): + minio_client.make_bucket(bucket_name) + + # Walk and upload files + for root, _, files in os.walk(test_data_dir): + for file_name in files: + file_path = os.path.join(root, file_name) + object_name = os.path.relpath(file_path, test_data_dir) + + print(f"Uploading {file_path} as {object_name} to bucket {bucket_name}") + minio_client.fput_object(bucket_name, object_name, file_path) + + +def test_validate_metadata(): + url = "http://localhost:5001/v1/ro_crates/validate_metadata" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # Load the JSON from file + filepath = os.path.join("tests/data", "ro-crate-metadata.json") + with open(filepath, "r", encoding="utf-8") as f: + crate_json_data = json.load(f) + + # The API expects the JSON to be passed as a string + payload = { + "crate_json": json.dumps(crate_json_data) + } + + response = requests.post(url, json=payload, headers=headers) + + response_result = json.loads(response.json()['result']) + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions — update based on expected API behavior + assert response.status_code == 200 + assert response_result['passed'] is True + + +def test_no_rocrate_for_validation(): + ro_crate = "ro_crate_10" + url = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = { + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } + } + + response = requests.post(url, json=payload, headers=headers) + + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions — update based on expected API behavior + assert response.status_code == 400 + assert response_result['message'] == f"No RO-Crate with prefix: {ro_crate}" + + +def test_no_validation_result_for_missing_crate(): + ro_crate = "ro_crate_10" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = { + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } + } + + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 400 + assert response_result['message'] == f"No RO-Crate with prefix: {ro_crate}" + + +def test_get_existing_validation_result(): + ro_crate = "ro_crate_3" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = { + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } + } + + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 200 + assert response_result["passed"] is False + + +def test_rocrate_not_validated_yet(): + ro_crate = "ro_crate_not_validated" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = { + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } + } + + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 400 + assert response_result['message'] == f"No validation result yet for RO-Crate: {ro_crate}" + + +def test_zipped_rocrate_validation(): + ro_crate = "ro_crate_1" + url_post = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = { + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } + } + + # POST action and tests + response = requests.post(url_post, json=payload, headers=headers) + response_result = response.json()['message'] + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 202 + assert response_result == "Validation in progress" + + # wait for ro-crate to be validated + time.sleep(10) + + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + start_time = time.time() + while response.status_code == 400: + time.sleep(10) + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + elapsed = time.time() - start_time + if elapsed > 60: + print("60 seconds passed. Exiting loop") + break + + # Assertions + assert response.status_code == 200 + assert response_result["passed"] is False + + +def test_directory_rocrate_validation(): + ro_crate = "ro_crate_2" + url_post = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = { + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } + } + + # POST action and tests + response = requests.post(url_post, json=payload, headers=headers) + response_result = response.json()['message'] + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 202 + assert response_result == "Validation in progress" + + # wait for ro-crate to be validated + time.sleep(10) + + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + start_time = time.time() + while response.status_code == 400: + time.sleep(10) + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + elapsed = time.time() - start_time + if elapsed > 60: + print("60 seconds passed. Exiting loop") + break + + # Assertions + assert response.status_code == 200 + assert response_result["passed"] is False + + +def test_ignore_rocrates_not_on_basepath(): + ro_crate = "ro_crate_4" + url_post = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = { + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + } + } + + # POST action and tests + response = requests.post(url_post, json=payload, headers=headers) + response_result = response.json()['message'] + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 400 + assert response_result == "No RO-Crate with prefix: ro_crate_4" + + +def test_zipped_rocrate_in_subdirectory_validation(): + ro_crate = "ro_crate_4" + subdir_path = "project_a" + url_post = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = { + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + }, + "root_path" : subdir_path + } + + # POST action and tests + response = requests.post(url_post, json=payload, headers=headers) + response_result = response.json()['message'] + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 202 + assert response_result == "Validation in progress" + + # wait for ro-crate to be validated + time.sleep(10) + + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + start_time = time.time() + while response.status_code == 400: + time.sleep(10) + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + elapsed = time.time() - start_time + if elapsed > 60: + print("60 seconds passed. Exiting loop") + break + + # Assertions + assert response.status_code == 200 + assert response_result["passed"] is False + + +def test_directory_rocrate_in_subdirectory_validation(): + ro_crate = "ro_crate_5" + subdir_path = "project_a" + url_post = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + url_get = f"http://localhost:5001/v1/ro_crates/{ro_crate}/validation" + headers = { + "accept": "application/json", + "Content-Type": "application/json" + } + + # The API expects the JSON to be passed as a string + payload = { + "minio_config": { + "endpoint": "minio:9000", + "accesskey": "minioadmin", + "secret": "minioadmin", + "ssl": False, + "bucket": "ro-crates" + }, + "root_path" : subdir_path + } + + # POST action and tests + response = requests.post(url_post, json=payload, headers=headers) + response_result = response.json()['message'] + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + # Assertions + assert response.status_code == 202 + assert response_result == "Validation in progress" + + # wait for ro-crate to be validated + time.sleep(10) + + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + start_time = time.time() + while response.status_code == 400: + time.sleep(10) + # GET action and tests + response = requests.get(url_get, json=payload, headers=headers) + response_result = response.json() + # Print response for debugging + print("Status Code:", response.status_code) + print("Response JSON:", response_result) + + elapsed = time.time() - start_time + if elapsed > 60: + print("60 seconds passed. Exiting loop") + break + + # Assertions + assert response.status_code == 200 + assert response_result["passed"] is False diff --git a/tests/test_minio.py b/tests/test_minio.py new file mode 100644 index 0000000..426d901 --- /dev/null +++ b/tests/test_minio.py @@ -0,0 +1,506 @@ +import json +import pytest +from io import BytesIO +from minio import Minio +from minio.error import S3Error +from unittest.mock import MagicMock, patch +from unittest import mock + + +@pytest.fixture +def mock_minio_response(): + response = MagicMock() + response.data.decode.return_value = json.dumps({"status": "valid"}) + return response + + +class DummyObject: + def __init__(self, name, is_dir=False): + self.object_name = name + self.is_dir = is_dir + + +# Testing function: get_minio_client + +@pytest.mark.parametrize( + "minio_config", + [ + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False + }, + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "ignore_this" + } + ], + ids=["base_case", "ignore_extra_items"] +) +def test_get_minio_client_success(minio_config: dict): + + from app.utils.minio_utils import get_minio_client + client = get_minio_client(minio_config) + + assert isinstance(client, Minio) + assert client._base_url.host == "localhost:9000" + + +# Testing function: get_minio_object_list + +def test_get_minio_object_list_success(): + # Setup mock response + mock_response = MagicMock() + mock_objects = [DummyObject("file1.txt"), DummyObject("file2.txt")] + mock_response.__iter__.return_value = iter(mock_objects) + + # Patch minio_client + mock_minio_client = MagicMock() + mock_minio_client.list_objects.return_value = mock_response + + # Call function + from app.utils.minio_utils import get_minio_object_list + result = get_minio_object_list("path/", mock_minio_client, "my-bucket", recursive=True) + + # Assert + assert result == mock_objects + mock_minio_client.list_objects.assert_called_once_with("my-bucket", "path/", recursive=True) + mock_response.close.assert_called_once() + + +@pytest.mark.parametrize( + "bucket, path, status_code, list_side_effect, error_check", + [ + ( + "my-bucket", "path/rocrate.zip", 500, + S3Error(code="S3 error", + message=None, + resource=None, + request_id=None, + host_id=None, + response=None), + "MinIO S3 Error" + ), + ( + "my-bucket", "path/rocrate.zip", 500, + ValueError("Missing config"), + "Configuration Error" + ), + ( + "my-bucket", "path/rocrate.zip", 500, + RuntimeError("Something went wrong"), + "Unknown Error" + ), + ], + ids=["s3error", "value_error", "unexpected_error"] +) +def test_get_minio_object_list_errors(bucket: str, path: str, status_code: int, list_side_effect, error_check: str): + mock_minio_client = MagicMock() + mock_minio_client.list_objects.side_effect = list_side_effect + + from app.utils.minio_utils import get_minio_object_list, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: + get_minio_object_list(path, mock_minio_client, bucket) + + assert exc.value.status_code == status_code + assert error_check in str(exc.value.message) + + +# Testing function: find_rocrate_object_on_minio + + +@pytest.mark.parametrize( + "rocrate_object, crateid, bucket, root_path", + [ + ( + DummyObject("my/path/rocrate123/", is_dir=True), + "rocrate123", "bucket", "my/path" + ), + ( + DummyObject("my/path/rocrate123.zip"), + "rocrate123", "bucket", "my/path" + ), + ( + DummyObject("rocrate123.zip"), + "rocrate123", "bucket", None + ), + ], + ids=["rocrate_directory", "rocrate_zip", "rootpath_none"] +) +@patch("app.utils.minio_utils.get_minio_object_list") +def test_finding_rocrate_on_minio( + mock_get_list, + rocrate_object: DummyObject, crateid: str, bucket: str, root_path: str): + # Simulate a directory object match + mock_get_list.return_value = [rocrate_object] + minio_client = MagicMock() + + from app.utils.minio_utils import find_rocrate_object_on_minio + result = find_rocrate_object_on_minio(crateid, minio_client, bucket, root_path) + assert result == rocrate_object + + +@patch("app.utils.minio_utils.get_minio_object_list") +def test_rocrate_not_found(mock_get_list): + # Simulate no matching object + mock_get_list.return_value = [ + DummyObject("something_else"), + DummyObject("another_dir", is_dir=True) + ] + minio_client = MagicMock() + + from app.utils.minio_utils import find_rocrate_object_on_minio + result = find_rocrate_object_on_minio("rocrate123", minio_client, "bucket", None) + + mock_get_list.assert_called_once() + assert not result + + +# Testing function: find_validation_object_on_minio + +@pytest.mark.parametrize( + "object_path, crateid, bucket, root_path", + [ + ( + "my/storage/rocrate123_validation/validation_status.txt", + "rocrate123", "bucket", "my/storage" + ), + ( + "rocrate123_validation/validation_status.txt", + "rocrate123", "bucket", None + ), + ], + ids=["with_storage_path", "without_storage_path"] +) +@patch("app.utils.minio_utils.get_minio_object_list") +def test_validation_object_found_with_storage_path( + mock_get_list, + object_path: str, crateid: str, bucket: str, root_path: str): + # Setup + obj = DummyObject(object_path) + mock_get_list.return_value = [obj] + + from app.utils.minio_utils import find_validation_object_on_minio + # Execute + result = find_validation_object_on_minio(crateid, MagicMock(), bucket, root_path) + + # Assert + assert result == obj + mock_get_list.assert_called_once_with(object_path, mock.ANY, bucket) + + +@pytest.mark.parametrize( + "object_list, crateid, bucket, root_path", + [ + ( + [DummyObject("some/other/object.txt")], + "rocrate999", "bucket", None + ), + ( + [], + "rocrate999", "bucket", None + ), + ], + ids=["other_objects", "empty_list"] +) +@patch("app.utils.minio_utils.get_minio_object_list") +def test_validation_object_not_found( + mock_get_list, + object_list: list, crateid: str, bucket: str, root_path: str): + # Setup: no objects returned + mock_get_list.return_value = object_list + + from app.utils.minio_utils import find_validation_object_on_minio + result = find_validation_object_on_minio(crateid, MagicMock(), bucket, root_path) + + assert result is False + + +# Testing function: download_file_from_minio + +@patch("app.utils.minio_utils.logging") +def test_download_success(mock_logging): + mock_minio = MagicMock() + + from app.utils.minio_utils import download_file_from_minio + # No exceptions raised + download_file_from_minio(mock_minio, "bucket", "remote/path.txt", "local/path.txt") + + mock_minio.fget_object.assert_called_once_with("bucket", "remote/path.txt", "local/path.txt") + mock_logging.error.assert_not_called() + + +@pytest.mark.parametrize( + "bucket, remotepath, localpath, status_code, get_side_effect, error_check", + [ + ( + "my-bucket", "remote/path.txt", "local/path.txt", 500, + S3Error(code="S3 error", + message=None, + resource=None, + request_id=None, + host_id=None, + response=None), + "MinIO S3 Error" + ), + ( + "my-bucket", "remote/path.txt", "local/path.txt", 500, + ValueError("Missing config"), + "Configuration Error" + ), + ( + "my-bucket", "remote/path.txt", "local/path.txt", 500, + RuntimeError("Something went wrong"), + "Unknown Error" + ), + ], + ids=["s3error", "value_error", "unexpected_error"] +) +@patch("app.utils.minio_utils.logging") +def test_download_s3error( + mock_logging, + bucket: str, remotepath: str, localpath: str, status_code: int, + get_side_effect, error_check: str +): + mock_minio = MagicMock() + mock_minio.fget_object.side_effect = get_side_effect + + from app.utils.minio_utils import download_file_from_minio, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: + download_file_from_minio(mock_minio, bucket, remotepath, localpath) + + assert exc.value.status_code == status_code + assert error_check in str(exc.value.message) + mock_logging.error.assert_called_once() + + +# Testing function: get_validation_status_from_minio + +def test_successful_retrieval(mocker, mock_minio_response): + mock_client = MagicMock() + mock_client.get_object.return_value = mock_minio_response + + from app.utils.minio_utils import get_validation_status_from_minio + result = get_validation_status_from_minio(mock_client, "test_bucket", "crate123", None) + + assert result == {"status": "valid"} + mock_minio_response.close.assert_called_once() + mock_minio_response.release_conn.assert_called_once() + + +@pytest.mark.parametrize( + "bucket, crateid, root_path, status_code, get_side_effect, error_check", + [ + ( + "my-bucket", "crate123", None, 500, + S3Error(code="S3 error", + message=None, + resource=None, + request_id=None, + host_id=None, + response=None), + "MinIO S3 Error" + ), + ( + "my-bucket", "crate123", None, 500, + ValueError("Missing env var"), + "Configuration Error" + ), + ( + "my-bucket", "crate123", None, 500, + RuntimeError("Unexpected failure"), + "Unknown Error" + ), + ], + ids=["s3error", "value_error", "unexpected_error"] +) +def test_get_validation_error_raised( + mocker, + bucket: str, crateid: str, root_path: str, status_code: int, get_side_effect, error_check: str +): + mock_client = MagicMock() + mock_client.get_object.side_effect = get_side_effect + + from app.utils.minio_utils import get_validation_status_from_minio, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: + get_validation_status_from_minio(mock_client, bucket, crateid, root_path) + + assert exc.value.status_code == status_code + assert error_check in str(exc.value.message) + + +# Testing function: update_validation_status_in_minio + +def test_update_validation_status_success(): + mock_minio_client = mock.Mock() + + crate_id = "crate123" + validation_status = json.dumps({"status": "valid", "errors": []}) + + from app.utils.minio_utils import update_validation_status_in_minio + update_validation_status_in_minio(mock_minio_client, "test_bucket", crate_id, "", validation_status) + + expected_object_name = f"{crate_id}_validation/validation_status.txt" + expected_data = json.dumps(json.loads(validation_status), indent=None).encode("utf-8") + + mock_minio_client.put_object.assert_called_once() + args, kwargs = mock_minio_client.put_object.call_args + + # FIXME: Original suggested test expected 4 values in args, but returned only 2. + # Solution was to check both args and kwargs for the 'data' and 'length' objects. + # Do we need to chose one format of call_args for our tests, or is this ambiguity okay? + bucket_name = args[0] if args else kwargs["bucket_name"] + object_name = args[1] if len(args) > 1 else kwargs["object_name"] + actual_data_stream = args[2] if len(args) > 2 else kwargs["data"] + length = args[3] if len(args) > 3 else kwargs["length"] + + assert bucket_name == "test_bucket" + assert object_name == expected_object_name + assert isinstance(actual_data_stream, BytesIO) + actual_data_stream.seek(0) + assert actual_data_stream.read() == expected_data + assert length == len(expected_data) + assert kwargs["content_type"] == "application/json" + + +@pytest.mark.parametrize( + "bucket, crateid, root_path, validation_result, put_side_effect, error_check, status_code", + [ + ( + "my-bucket", "crate123", None, + {"status": "valid"}, + S3Error(code="S3 error", + message=None, + resource=None, + request_id=None, + host_id=None, + response=None), + "MinIO S3 Error", 500 + ), + ( + "my-bucket", "crate123", None, + {"status": "valid"}, + ValueError("Missing env vars"), + "Configuration Error", 500 + ), + ( + "my-bucket", "crate123", None, + {"status": "valid"}, + RuntimeError("Unexpected failure"), + "Unknown Error", 500 + ), + ], + ids=["s3error", "value_error", "unexpected_error"] +) +def test_update_validation_status_erro( + bucket: str, crateid: str, root_path: str, validation_result: dict, + put_side_effect, error_check: str, status_code: int +): + mock_minio_client = mock.Mock() + mock_minio_client.put_object.side_effect = put_side_effect + + from app.utils.minio_utils import update_validation_status_in_minio, InvalidAPIUsage + with pytest.raises(InvalidAPIUsage) as exc: + update_validation_status_in_minio(mock_minio_client, bucket, crateid, root_path, json.dumps(validation_result)) + + assert exc.value.status_code == status_code + assert error_check in str(exc.value.message) + + +# Testing function: fetch_ro_crate_from_minio + +@patch("app.utils.minio_utils.download_file_from_minio") +@patch("app.utils.minio_utils.get_minio_object_list") +@patch("app.utils.minio_utils.find_rocrate_object_on_minio") +def test_fetch_rocrate_zip( + mock_find_object, + mock_get_list, + mock_download, + tmp_path, +): + # Setup mocks + minio_client = "minio_client" + rocrate_obj = DummyObject("some/path/rocrate123.zip", is_dir=False) + mock_find_object.return_value = rocrate_obj + + from app.utils.minio_utils import fetch_ro_crate_from_minio + + with patch("app.utils.minio_utils.tempfile.mkdtemp", return_value=str(tmp_path)): + # Execute + result = fetch_ro_crate_from_minio(minio_client, "test_bucket", "rocrate123", "some/path") + + # Assert + expected_path = tmp_path / "rocrate123.zip" + assert result == str(expected_path) + mock_download.assert_called_once_with( + "minio_client", "test_bucket", + "some/path/rocrate123.zip", str(expected_path)) + + +@patch("app.utils.minio_utils.download_file_from_minio") +@patch("app.utils.minio_utils.get_minio_object_list") +@patch("app.utils.minio_utils.find_rocrate_object_on_minio") +def test_fetch_rocrate_directory( + mock_find_object, + mock_get_list, + mock_download, + tmp_path, +): + # Setup mocks + minio_client = "minio_client" + rocrate_obj = DummyObject("rocrates/rocrate124", is_dir=True) + mock_find_object.return_value = rocrate_obj + + from app.utils.minio_utils import fetch_ro_crate_from_minio + + with patch("app.utils.minio_utils.tempfile.mkdtemp", return_value=str(tmp_path)): + # Objects inside the RO-Crate + mock_get_list.return_value = [ + DummyObject("rocrates/rocrate124/metadata.json"), + DummyObject("rocrates/rocrate124/data/file1.txt"), + ] + + # Execute + result = fetch_ro_crate_from_minio(minio_client, "test_bucket", "rocrate124", "rocrates") + + # Assert + expected_root = tmp_path / "rocrate124" + assert result == str(expected_root) + mock_download.assert_any_call( + "minio_client", "test_bucket", + "rocrates/rocrate124/metadata.json", + str(expected_root / "metadata.json") + ) + mock_download.assert_any_call( + "minio_client", "test_bucket", + "rocrates/rocrate124/data/file1.txt", + str(expected_root / "data/file1.txt") + ) + + +@patch("app.utils.minio_utils.download_file_from_minio") +@patch("app.utils.minio_utils.get_minio_object_list") +@patch("app.utils.minio_utils.find_rocrate_object_on_minio") +def test_fetch_rocrate_handles_empty_dir( + mock_find_object, + mock_get_list, + mock_download, + tmp_path, +): + minio_client = "minio_client" + rocrate_obj = DummyObject("rocrate456", is_dir=True) + mock_find_object.return_value = rocrate_obj + mock_get_list.return_value = [] + + from app.utils.minio_utils import fetch_ro_crate_from_minio + + with patch("app.utils.minio_utils.tempfile.mkdtemp", return_value=str(tmp_path)): + result = fetch_ro_crate_from_minio(minio_client, "test_bucket", "rocrate456", "") + + expected_root = tmp_path / "rocrate456" + assert result == str(expected_root) + mock_download.assert_not_called() diff --git a/tests/test_services.py b/tests/test_services.py new file mode 100644 index 0000000..c7d50c3 --- /dev/null +++ b/tests/test_services.py @@ -0,0 +1,286 @@ +import pytest +from unittest.mock import patch, MagicMock +from flask import Flask +from flask.testing import FlaskClient + +from app.services.validation_service import ( + queue_ro_crate_validation_task, + queue_ro_crate_metadata_validation_task, + get_ro_crate_validation_task +) + +from app.utils.minio_utils import InvalidAPIUsage + + +@pytest.fixture +def flask_app(): + app = Flask(__name__) + with app.app_context(): + yield app + + +# Test function: queue_ro_crate_validation_task + +@pytest.mark.parametrize( + "crate_id, rocrate_exists, minio_client, delay_side_effects, payload, status_code, response_dict", + [ + ( + "crate123", True, "minio_client", None, + { + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + }, 202, {"message": "Validation in progress"} + ), + ( + "crate123", True, "minio_client", Exception("Celery down"), + { + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + }, 500, {"error": "Celery down"} + ), + ], + ids=["successful_queue", "celery_server_down"] +) +@patch("app.services.validation_service.process_validation_task_by_id.delay") +@patch("app.services.validation_service.check_ro_crate_exists") +@patch("app.services.validation_service.get_minio_client") +def test_queue_ro_crate_validation_task( + mock_client, + mock_exists, + mock_delay, + flask_app: FlaskClient, crate_id: str, rocrate_exists: bool, minio_client: str, + delay_side_effects: Exception, payload: dict, status_code: int, response_dict: dict +): + mock_delay.side_effect = delay_side_effects + mock_exists.return_value = rocrate_exists + mock_client.return_value = minio_client + + minio_config = payload["minio_config"] if "minio_config" in payload else None + root_path = payload["root_path"] if "root_path" in payload else None + profile_name = payload["profile_name"] if "profile_name" in payload else None + webhook_url = payload["webhook_url"] if "webhook_url" in payload else None + + response, status_code = queue_ro_crate_validation_task(minio_config, crate_id, root_path, profile_name, webhook_url) + + mock_client.assert_called_once_with(minio_config) + mock_exists.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, root_path) + mock_delay.assert_called_once_with(minio_config, crate_id, root_path, profile_name, webhook_url) + assert status_code == status_code + assert response.json == response_dict + + +@pytest.mark.parametrize( + "crate_id, rocrate_exists, minio_client, payload, iau_message", + [ + ( + "crate12z", False, "minio_client", + { + "minio_config": { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "root_path": "base_path", + "webhook_url": "https://webhook.example.com", + "profile_name": "default" + }, "No RO-Crate with prefix: crate12z" + ), + ], + ids=["no_rocrate_exists"] +) +@patch("app.services.validation_service.process_validation_task_by_id.delay") +@patch("app.services.validation_service.check_ro_crate_exists") +@patch("app.services.validation_service.get_minio_client") +def test_queue_ro_crate_validation_task_failure( + mock_client, + mock_exists, + mock_delay, + flask_app: FlaskClient, crate_id: str, rocrate_exists: bool, + minio_client: str, payload: dict, iau_message: str +): + mock_exists.return_value = rocrate_exists + mock_client.return_value = minio_client + + minio_config = payload["minio_config"] if "minio_config" in payload else None + root_path = payload["root_path"] if "root_path" in payload else None + profile_name = payload["profile_name"] if "profile_name" in payload else None + webhook_url = payload["webhook_url"] if "webhook_url" in payload else None + + with pytest.raises(InvalidAPIUsage) as exc_info: + queue_ro_crate_validation_task(minio_config, crate_id, root_path, profile_name, webhook_url) + + assert iau_message in str(exc_info.value.message) + mock_client.assert_called_once_with(minio_config) + mock_exists.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, root_path) + mock_delay.assert_not_called() + + +# Test function: queue_ro_crate_metadata_validation_task + +@pytest.mark.parametrize( + "crate_json, profile, webhook, status_code, return_value, response_json, delay_side_effect", + [ + ( + '{"@context": "https://w3id.org/ro/crate/1.1/context"}', + "default", "http://webhook", + 202, None, {"message": "Validation in progress"}, + None + ), + ( + '{"@context": "https://w3id.org/ro/crate/1.1/context"}', + "default", None, + 200, {"status": "ok"}, {"result": {"status": "ok"}}, + None + ), + ( + '{"@context": "https://w3id.org/ro/crate/1.1/context"}', + "default", "http://webhook", + 500, None, {"error": "Celery error"}, + Exception("Celery error") + ), + ], + ids=["success_with_webhook", "success_without_webhook", "failure_celery_error"] +) +def test_queue_metadata(flask_app, crate_json: dict, profile: str, webhook: str, + status_code: int, return_value: dict, response_json: dict, + delay_side_effect: Exception): + with patch("app.services.validation_service.process_validation_task_by_metadata.delay", + side_effect=delay_side_effect) as mock_delay: + mock_result = MagicMock() + if return_value is not None: + mock_result.get.return_value = return_value + if delay_side_effect is None: + mock_delay.return_value = mock_result + + response, status = queue_ro_crate_metadata_validation_task(crate_json, profile, webhook) + + mock_delay.assert_called_once_with(crate_json, profile, webhook) + assert status == status_code + assert response.json == response_json + + +@pytest.mark.parametrize( + "crate_json, status_code, response_error", + [ + ( + None, + 422, "Missing required parameter: crate_json" + ), + ( + "{", + 422, "not valid JSON" + ), + ( + "{}", + 422, "Required parameter crate_json is empty" + ), + ] +) +def test_queue_metadata_json_errors(flask_app, crate_json: str, status_code: int, response_error: str): + response, status = queue_ro_crate_metadata_validation_task(crate_json) + assert status == status_code + assert response_error in response.json["error"] + + +# Test function: get_ro_crate_validation_task + +@pytest.mark.parametrize( + "minio_config, crate_id, crate_exists, validation_exists, " + + "validation_value, status_code, error_message, minio_client", + [ + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", True, True, {"status": "valid"}, 200, None, + "minio_client" + ), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", False, False, None, 400, "No RO-Crate with prefix: crate123", + "minio_client" + ), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", True, False, None, 400, "No validation result yet for RO-Crate: crate123", + "minio_client" + ), + ], + ids=["validation_exists", "rocrate_missing", "validation_missing"] +) +@patch("app.services.validation_service.check_ro_crate_exists") +@patch("app.services.validation_service.check_validation_exists") +@patch("app.services.validation_service.return_ro_crate_validation") +@patch("app.services.validation_service.get_minio_client") +def test_get_validation( + mock_client, + mock_return, + mock_validation, + mock_rocrate, + flask_app, minio_config: dict, crate_id: str, crate_exists: bool, + validation_exists: bool, validation_value: dict, + status_code: int, error_message: str, minio_client: str +): + mock_client.return_value = minio_client + mock_rocrate.return_value = crate_exists + mock_validation.return_value = validation_exists + mock_return.return_value = validation_value + + if crate_exists and validation_exists: + response, status = get_ro_crate_validation_task(minio_config, crate_id, "base_path") + + mock_client.assert_called_once_with(minio_config) + mock_return.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "base_path") + mock_rocrate.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "base_path") + mock_validation.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "base_path") + + assert status == status_code + assert response == validation_value + + else: + with pytest.raises(InvalidAPIUsage) as exc_info: + get_ro_crate_validation_task(minio_config, crate_id, "base_path") + + assert exc_info.value.status_code == status_code + assert error_message in str(exc_info.value.message) + + mock_rocrate.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "base_path") + if crate_exists: + mock_validation.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "base_path") + else: + mock_validation.assert_not_called() + mock_return.assert_not_called() diff --git a/tests/test_validation_tasks.py b/tests/test_validation_tasks.py new file mode 100644 index 0000000..afa11c2 --- /dev/null +++ b/tests/test_validation_tasks.py @@ -0,0 +1,462 @@ +from unittest import mock +import pytest + +from app.tasks.validation_tasks import ( + process_validation_task_by_id, + perform_ro_crate_validation, + return_ro_crate_validation, + process_validation_task_by_metadata, + check_ro_crate_exists, + check_validation_exists +) + +from app.utils.minio_utils import InvalidAPIUsage + + +# Test function: process_validation_task_by_id + +@pytest.mark.parametrize( + "minio_config, crate_id, os_path_exists, os_path_isfile, os_path_isdir, " + + "return_value, webhook, profile, val_success, val_result, minio_client", + [ + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", True, True, False, "/tmp/crate.zip", + "https://example.com/hook", "profileA", True, '{"status": "valid"}', + "minio_client" + ), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", True, False, True, "/tmp/crate123", + "https://example.com/hook", "profileA", True, '{"status": "valid"}', + "minio_client" + ), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", True, False, True, "/tmp/crate123", + None, "profileA", True, '{"status": "valid"}', + "minio_client" + ), + ], + ids=["successful_validation_zip", "successful_validation_dir", "successful_validation_nowebhook"] +) +@mock.patch("app.tasks.validation_tasks.get_minio_client") +@mock.patch("app.tasks.validation_tasks.shutil.rmtree") +@mock.patch("app.tasks.validation_tasks.os.remove") +@mock.patch("app.tasks.validation_tasks.os.path.exists") +@mock.patch("app.tasks.validation_tasks.os.path.isfile") +@mock.patch("app.tasks.validation_tasks.os.path.isdir") +@mock.patch("app.tasks.validation_tasks.send_webhook_notification") +@mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") +@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") +@mock.patch("app.tasks.validation_tasks.fetch_ro_crate_from_minio") +def test_process_validation( + mock_fetch, + mock_validate, + mock_update, + mock_webhook, + mock_isdir, + mock_isfile, + mock_exists, + mock_remove, + mock_rmtree, + mock_client, + minio_config: dict, crate_id: str, os_path_exists: bool, os_path_isfile: bool, os_path_isdir: bool, + return_value: str, webhook: str, profile: str, val_success: bool, val_result: str, minio_client: str +): + mock_exists.return_value = os_path_exists + mock_isfile.return_value = os_path_isfile + mock_isdir.return_value = os_path_isdir + mock_fetch.return_value = return_value + mock_client.return_value = minio_client + + mock_validation_result = mock.Mock() + mock_validation_result.has_issues.return_value = val_success + mock_validation_result.to_json.return_value = val_result + mock_validate.return_value = mock_validation_result + + process_validation_task_by_id(minio_config, crate_id, "", profile, webhook) + + mock_client.assert_called_once_with(minio_config) + mock_fetch.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "") + mock_validate.assert_called_once_with(return_value, profile) + mock_update.assert_called_once_with(minio_client, minio_config["bucket"], crate_id, "", val_result) + if webhook is not None: + mock_webhook.assert_called_once_with(webhook, val_result) + else: + mock_webhook.assert_not_called() + if os_path_exists and os_path_isfile: + mock_remove.assert_called_once_with(return_value) + mock_rmtree.assert_not_called() + elif os_path_exists and os_path_isdir: + mock_rmtree.assert_called_once_with(return_value) + mock_remove.assert_not_called() + + +@pytest.mark.parametrize( + "minio_config, crate_id, os_path_exists, os_path_isfile, os_path_isdir, return_fetch, " + + "webhook, profile, return_validate, validate_side_effect, fetch_side_effect, minio_client", + [ + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", True, True, False, "/tmp/crate.zip", + "https://example.com/hook", "profileA", "Validation failed", None, None, + "minio_client" + ), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", True, True, False, "/tmp/crate.zip", + "https://example.com/hook", "profileA", None, Exception("Unexpected error"), None, + "minio_client" + ), + ( + { + "endpoint": "localhost:9000", + "accesskey": "admin", + "secret": "password123", + "ssl": False, + "bucket": "test_bucket" + }, + "crate123", False, False, False, None, + "https://example.com/hook", "profileA", None, None, Exception("MinIO fetch failed"), + "minio_client" + ), + ], + ids=["validation_fails_with_message", "validation_fails_with_validation_exception", + "validation_fails_with_fetch_exception"] +) +@mock.patch("app.tasks.validation_tasks.get_minio_client") +@mock.patch("app.tasks.validation_tasks.shutil.rmtree") +@mock.patch("app.tasks.validation_tasks.os.remove") +@mock.patch("app.tasks.validation_tasks.os.path.exists") +@mock.patch("app.tasks.validation_tasks.os.path.isfile") +@mock.patch("app.tasks.validation_tasks.os.path.isdir") +@mock.patch("app.tasks.validation_tasks.send_webhook_notification") +@mock.patch("app.tasks.validation_tasks.update_validation_status_in_minio") +@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") +@mock.patch("app.tasks.validation_tasks.fetch_ro_crate_from_minio") +def test_process_validation_failure( + mock_fetch, + mock_validate, + mock_update, + mock_webhook, + mock_isdir, + mock_isfile, + mock_exists, + mock_remove, + mock_rmtree, + mock_client, + minio_config: dict, crate_id: str, os_path_exists: bool, os_path_isfile: bool, os_path_isdir: bool, + return_fetch: str, webhook: str, profile: str, return_validate: str, + validate_side_effect: Exception, fetch_side_effect: Exception, minio_client: str +): + mock_exists.return_value = os_path_exists + mock_isfile.return_value = os_path_isfile + mock_isdir.return_value = os_path_isdir + mock_client.return_value = minio_client + + if fetch_side_effect is None: + mock_fetch.return_value = return_fetch + else: + mock_fetch.side_effect = fetch_side_effect + + if validate_side_effect is None: + mock_validate.return_value = return_validate + else: + mock_validate.side_effect = validate_side_effect + + process_validation_task_by_id(minio_config, crate_id, "", profile, webhook) + + if fetch_side_effect is None: + mock_validate.assert_called_once_with(return_fetch, profile) + else: + mock_validate.assert_not_called() + + mock_update.assert_not_called() + mock_webhook.assert_called_once() + args, kwargs = mock_webhook.call_args + assert args[0] == webhook + if fetch_side_effect is not None: + assert fetch_side_effect.args[0] in args[1]["error"] + elif validate_side_effect is not None: + assert validate_side_effect.args[0] in args[1]["error"] + else: + assert return_validate in args[1]["error"] + + if not os_path_exists: + mock_remove.assert_not_called() + mock_rmtree.assert_not_called() + elif os_path_exists and os_path_isfile: + mock_remove.assert_called_once_with(return_fetch) + mock_rmtree.assert_not_called() + elif os_path_exists and os_path_isdir: + mock_rmtree.assert_called_once_with(return_fetch) + mock_remove.assert_not_called() + + +# Test function: process_validation_task_by_metadata + +@pytest.mark.parametrize( + "crate_json, profile_name, webhook_url, mock_path, validation_json, validation_value, os_path_exists", + [ + ( + '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}', + "test-profile", "https://example.com/webhook", "/tmp/crate", + '{"status": "valid"}', False, True + ), + ( + '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}', + "test-profile", "https://example.com/webhook", "/tmp/crate", + '{"status": "invalid"}', True, True + ) + ], + ids=["success_no_issues", "success_with_issues"] +) +@mock.patch("app.tasks.validation_tasks.shutil.rmtree") +@mock.patch("app.tasks.validation_tasks.os.path.exists") +@mock.patch("app.tasks.validation_tasks.send_webhook_notification") +@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") +@mock.patch("app.tasks.validation_tasks.build_metadata_only_rocrate") +def test_metadata_validation( + mock_build, mock_validate, mock_webhook, mock_exists, mock_rmtree, + crate_json: str, profile_name: str, webhook_url: str, mock_path: str, + validation_json: str, validation_value: bool, os_path_exists: bool +): + mock_exists.return_value = os_path_exists + mock_build.return_value = mock_path + + mock_result = mock.Mock() + mock_result.has_issues.return_value = validation_value + mock_result.to_json.return_value = validation_json + mock_validate.return_value = mock_result + + result = process_validation_task_by_metadata(crate_json, profile_name, webhook_url) + + assert result == validation_json + mock_build.assert_called_once_with(crate_json) + mock_validate.assert_called_once() + mock_webhook.assert_called_once_with(webhook_url, validation_json) + mock_rmtree.assert_called_once_with(mock_path) + + +@pytest.mark.parametrize( + "crate_json, profile_name, webhook_url, mock_path, validation_message, os_path_exists", + [ + ( + '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}', + "test-profile", "https://example.com/webhook", "/tmp/crate", + "Validation error", True + ), + ( + '{"@context": "https://w3id.org/ro/crate/1.1/context", "@graph": []}', + "test-profile", None, "/tmp/crate", + "Validation error", True + ) + ], + ids=["validation_fails", "validation_fails_no_webhook"] +) +@mock.patch("app.tasks.validation_tasks.shutil.rmtree") +@mock.patch("app.tasks.validation_tasks.os.path.exists", return_value=True) +@mock.patch("app.tasks.validation_tasks.send_webhook_notification") +@mock.patch("app.tasks.validation_tasks.perform_ro_crate_validation") +@mock.patch("app.tasks.validation_tasks.build_metadata_only_rocrate") +def test_validation_fails_and_sends_error_notification_to_webhook( + mock_build, mock_validate, mock_webhook, mock_exists, mock_rmtree, + crate_json: str, profile_name: str, webhook_url: str, mock_path: str, + validation_message: str, os_path_exists: bool +): + mock_build.return_value = mock_path + + mock_validate.return_value = validation_message + + result = process_validation_task_by_metadata(crate_json, profile_name, webhook_url) + + assert isinstance(result, str) + assert validation_message in result + + if webhook_url is not None: + # Error webhook should be sent + mock_webhook.assert_called_once() + args, kwargs = mock_webhook.call_args + assert kwargs is None or "error" in args[1] + else: + # Make sure webhook not sent + mock_webhook.assert_not_called() + + mock_rmtree.assert_called_once_with(mock_path) + + +# Test function: perform_ro_crate_validation + +@pytest.mark.parametrize( + "file_path, profile_name, skip_checks", + [ + ("crates/test_crate", "ro_profile", ["check1", "check2"]), + ("crates/test_crate", None, None) + ], + ids=["success_with_all_args", "success_with_only_crate"] +) +@mock.patch("app.tasks.validation_tasks.services.validate") +@mock.patch("app.tasks.validation_tasks.services.ValidationSettings") +def test_validation_success_with_all_args( + mock_validation_settings, mock_validate, + file_path: str, profile_name: str, skip_checks: list +): + mock_result = mock.Mock() + mock_validate.return_value = mock_result + + result = perform_ro_crate_validation(file_path, profile_name, skip_checks) + + # Assert that result was returned + assert result == mock_result + + # Validate proper construction of ValidationSettings + mock_validation_settings.assert_called_once() + args, kwargs = mock_validation_settings.call_args + assert kwargs["rocrate_uri"].endswith(file_path) + if profile_name is not None: + assert kwargs["profile_identifier"] == profile_name + else: + assert "profile_identifier" not in kwargs + if skip_checks is not None: + assert kwargs["skip_checks"] == skip_checks + else: + assert "skip_checks" not in kwargs + + mock_validate.assert_called_once_with(mock_validation_settings.return_value) + + +@mock.patch("app.tasks.validation_tasks.services.validate", side_effect=RuntimeError("Validation error")) +@mock.patch("app.tasks.validation_tasks.services.ValidationSettings") +def test_validation_raises_exception_and_returns_string(mock_validation_settings, mock_validate): + file_path = "crates/test_crate" + result = perform_ro_crate_validation(file_path, "profile", skip_checks_list=None) + + assert isinstance(result, str) + assert "Validation error" in result + mock_validate.assert_called_once() + + +@mock.patch("app.tasks.validation_tasks.services.validate") +@mock.patch("app.tasks.validation_tasks.services.ValidationSettings", side_effect=ValueError("Bad config")) +def test_validation_settings_error(mock_validation_settings, mock_validate): + file_path = "crates/test_crate" + result = perform_ro_crate_validation(file_path, None) + + assert isinstance(result, str) + assert "Bad config" in result + mock_validate.assert_not_called() + + +# Test function: return_ro_crate_validation + +@mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio") +def test_return_validation_returns_dict(mock_get_status): + # Simulate dict result + mock_get_status.return_value = {"status": "passed", "errors": []} + + result = return_ro_crate_validation("minio_client", "test_bucket", "crate123", None) + assert isinstance(result, dict) + assert result["status"] == "passed" + mock_get_status.assert_called_once_with("minio_client", "test_bucket", "crate123", None) + + +@mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio") +def test_return_validation_returns_string(mock_get_status): + # Simulate string result + mock_get_status.return_value = "Validation result: OK" + + result = return_ro_crate_validation("minio_client", "test_bucket", "crate456", None) + assert isinstance(result, str) + assert "OK" in result + mock_get_status.assert_called_once_with("minio_client", "test_bucket", "crate456", None) + + +@mock.patch("app.tasks.validation_tasks.get_validation_status_from_minio") +def test_return_validation_raises_error(mock_get_status): + # Simulate exception + mock_get_status.side_effect = InvalidAPIUsage("MinIO S3 Error: empty", 500) + + with pytest.raises(InvalidAPIUsage) as exc_info: + return_ro_crate_validation("minio_client", "test_bucket", "crate789", None) + + assert "MinIO S3 Error" in str(exc_info.value.message) + mock_get_status.assert_called_once_with("minio_client", "test_bucket", "crate789", None) + + +# Test function: check_ro_crate_exists + +@pytest.mark.parametrize( + "minio_client, bucket, crate_id, base_path, ro_object_return, rocrate_exists", + [ + ("minio_client", "test_bucket", "crate123", "base_path", "crate123", True), + ("minio_client", "test_bucket", "crate12z", "base_path", False, False) + ], + ids=["rocrate_exists", "rocrate_does_not_exist"] +) +@mock.patch("app.tasks.validation_tasks.find_rocrate_object_on_minio") +def test_ro_crate_exists( + mock_find_rocrate, + minio_client: str, bucket: str, crate_id: str, base_path: str, + ro_object_return: str, rocrate_exists: bool +): + mock_find_rocrate.return_value = ro_object_return + + result = check_ro_crate_exists(minio_client, bucket, crate_id, base_path) + + mock_find_rocrate.assert_called_once_with(crate_id, minio_client, bucket, base_path) + assert result is rocrate_exists + + +# Test function: check_validation_exists + +@pytest.mark.parametrize( + "minio_client, bucket, crate_id, base_path, val_object_return, validate_exists", + [ + ("minio_client", "test_bucket", "crate123", "base_path", "crate123", True), + ("minio_client", "test_bucket", "crate12z", "base_path", False, False) + ], + ids=["validation_exists", "validation_does_not_exist"] +) +@mock.patch("app.tasks.validation_tasks.find_validation_object_on_minio") +def test_validation_exists( + mock_find_validation, + minio_client: str, bucket: str, crate_id: str, base_path: str, + val_object_return: str, validate_exists: bool +): + mock_find_validation.return_value = val_object_return + + result = check_validation_exists(minio_client, bucket, crate_id, base_path) + + mock_find_validation.assert_called_once_with(crate_id, minio_client, bucket, base_path) + assert result is validate_exists